import re

from bs4 import BeautifulSoup

import urllib2

from mylog import MyLog as mylog

from save2excel import SavaBallDate



class DoubleColorBallItem(object):

    date = None

    order = None

    red1 = None

    red2 = None

    red3 = None

    red4 = None

    red5 = None

    red6 = None

    blue = None

    money = None

    firstPrize = None

    secondPrize = None



class GetDoubleColorBallNumber(object):

    '''这个类用于获取双色球中奖号码， 返回一个txt文件

    '''

    def __init__(self):

        self.urls = 

        self.log = mylog()

        self.getUrls()

        self.items = self.spider(self.urls)

        self.pipelines(self.items)

        self.log.info('beging save data to excel \r\n')

        SavaBallDate(self.items)

        self.log.info('save data to excel end ...\r\n')

      

        

    def getUrls(self):

        '''获取数据来源网页

        '''

        URL = r'http://kaijiang.zhcw.com/zhcw/html/ssq/list_1.html'

        htmlContent = self.getResponseContent(URL)

        soup = BeautifulSoup(htmlContent, 'lxml')

        tag = soup.find_all(re.compile('p'))[-1]

        pages = tag.strong.get_text()        

        for i in xrange(1, int(pages)+1):

            url = r'http://kaijiang.zhcw.com/zhcw/html/ssq/list_' + str(i) + '.html'

            self.urls.append(url)

            self.log.info(u'添加URL:%s 到URLS \r\n' %url)

            

    def getResponseContent(self, url):

        '''这里单独使用一个函数返回页面返回值，是为了后期方便的加入proxy和headers等

        '''

        try:

            response = urllib2.urlopen(url.encode('utf8'))

        except:

            self.log.error(u'Python 返回URL:%s  数据失败  \r\n' %url)

        else:

            self.log.info(u'Python 返回URUL:%s  数据成功 \r\n' %url)

            return response.read()

        

            

    def spider(self,urls):

        '''这个函数的作用是从获取的数据中过滤得到中奖信息

        '''

        items = 

        for url in urls:

            htmlContent = self.getResponseContent(url)

            soup = BeautifulSoup(htmlContent, 'lxml')

            tags = soup.find_all('tr', attrs={})

            for tag in tags:

                if tag.find('em'):

                    item = DoubleColorBallItem()

                    tagTd = tag.find_all('td')

                    item.date = tagTd[0].get_text()

                    item.order = tagTd[1].get_text()

                    tagEm = tagTd[2].find_all('em')

                    item.red1 = tagEm[0].get_text()

                    item.red2 = tagEm[1].get_text()

                    item.red3 = tagEm[2].get_text()

                    item.red4 = tagEm[3].get_text()

                    item.red5 = tagEm[4].get_text()

                    item.red6 = tagEm[5].get_text()

                    item.blue = tagEm[6].get_text()

                    item.money = tagTd[3].find('strong').get_text()

                    item.firstPrize = tagTd[4].find('strong').get_text()

                    item.secondPrize = tagTd[5].find('strong').get_text()

                    items.append(item)

                    self.log.info(u'获取日期为:%s 的数据成功' %(item.date))

        return items

    

    def pipelines(self,items):

        fileName = u'双色球.txt'.encode('GBK')

        with open(fileName, 'w') as fp:

            for item in items:

                fp.write('%s %s \t %s %s %s %s %s %s  %s \t %s \t %s %s \n'

                      %(item.date,item.order,item.red1,item.red2,item.red3,item.red4,item.red5,item.red6,item.blue,item.money,item.firstPrize,item.secondPrize))

                self.log.info(u'将日期为:%s 的数据存入"%s"...' %(item.date, fileName.decode('GBK')))

                    



if __name__ == '__main__':

    GDCBN = GetDoubleColorBallNumber()

import logging

import getpass

import sys





#### 定义MyLog类

class MyLog(object):

#### 类MyLog的构造函数

	def __init__(self):

		self.user = getpass.getuser()

		self.logger = logging.getLogger(self.user)

		self.logger.setLevel(logging.DEBUG)



####  日志文件名

		self.logFile = sys.argv[0][0:-3] + '.log'

		self.formatter = logging.Formatter('%(asctime)-12s %(levelname)-8s %(name)-10s %(message)-12s\r\n')



####  日志显示到屏幕上并输出到日志文件内

		self.logHand = logging.FileHandler(self.logFile, encoding='utf8')

		self.logHand.setFormatter(self.formatter)

		self.logHand.setLevel(logging.DEBUG)



		self.logHandSt = logging.StreamHandler()

		self.logHandSt.setFormatter(self.formatter)

		self.logHandSt.setLevel(logging.DEBUG)



		self.logger.addHandler(self.logHand)

		self.logger.addHandler(self.logHandSt)



####  日志的5个级别对应以下的5个函数

	def debug(self,msg):

		self.logger.debug(msg)



	def info(self,msg):

		self.logger.info(msg)



	def warn(self,msg):

		self.logger.warn(msg)



	def error(self,msg):

		self.logger.error(msg)



	def critical(self,msg):

		self.logger.critical(msg)



if __name__ == '__main__':

	mylog = MyLog()

	mylog.debug(u"I'm debug 测试中文")

	mylog.info("I'm info")

	mylog.warn("I'm warn")

	mylog.error(u"I'm error 测试中文")

	mylog.critical("I'm critical")

import xlwt

class SavaBallDate(object):

    def __init__(self, items):

        self.items = items

        self.run(self.items)

        

    def run(self,items):

        fileName = u'双色球.xls'.encode('GBK')

        book = xlwt.Workbook(encoding='utf8')

        sheet=book.add_sheet('ball', cell_overwrite_ok=True)

        sheet.write(0, 0, u'开奖日期'.encode('utf8'))

        sheet.write(0, 1, u'期号'.encode('utf8'))

        sheet.write(0, 2, u'红1'.encode('utf8'))

        sheet.write(0, 3, u'红2'.encode('utf8'))

        sheet.write(0, 4, u'红3'.encode('utf8'))

        sheet.write(0, 5, u'红4'.encode('utf8'))

        sheet.write(0, 6, u'红5'.encode('utf8'))

        sheet.write(0, 7, u'红6'.encode('utf8'))

        sheet.write(0, 8, u'蓝'.encode('utf8'))

        sheet.write(0, 9, u'销售金额'.encode('utf8'))

        sheet.write(0, 10, u'一等奖'.encode('utf8'))

        sheet.write(0, 11, u'二等奖'.encode('utf8'))

        i = 1

        while i <= len(items):

            item = items[i-1]

            sheet.write(i, 0, item.date)

            sheet.write(i, 1, item.order)

            sheet.write(i, 2, item.red1)

            sheet.write(i, 3, item.red2)

            sheet.write(i, 4, item.red3)

            sheet.write(i, 5, item.red4)

            sheet.write(i, 6, item.red5)

            sheet.write(i, 7, item.red6)

            sheet.write(i, 8, item.blue)

            sheet.write(i, 9, item.money)

            sheet.write(i, 10, item.firstPrize)

            sheet.write(i, 11, item.secondPrize)

            i += 1

        book.save(fileName)

        





if __name__ == '__main__':

    pass

<div id="content">

    <div class="login-wrap">

		<div class="w">

            <div class="login-form">

                <div class="login-tab login-tab-l">

                    <a href="javascript:void(0)" clstag="pageclick|keycount|201607144|1"> 扫码登录</a>

                </div>

                <div class="login-tab login-tab-r">

                    <a href="javascript:void(0)" clstag="pageclick|keycount|201607144|2">账户登录</a>

                </div>

                <div class="login-box">

                    <div class="mt tab-h">

                    </div>

                    <div class="msg-wrap">

						                        <div class="msg-error hide"><b></b></div>

                    </div>

#-*-coding=utf-8-*-

import requests,datetime

from lxml import etree

import pandas as pd

from sqlalchemy import create_engine

import random,time



user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/37.0.2062.120 Chrome/37.0.2062.120 Safari/537.36'

headers={'User-Agent':user_agent}



def getWebContent(url):

	# url=url.format(kind)

	r=requests.get(url,headers=headers)

	

	if r.status_code==200:

		return r.text

	else:

		return None



def getCoinList():

	content=getWebContent(url='https://coinmarketcap.com/tokens/views/all/'

)

	tree=etree.HTML(content)

	# table=tree.xpath()

	# if not table:

	# 	reutrn None

	df=pd.DataFrame()



	# for coin in table:

	df['currency-symbol']=tree.xpath('//tbody/tr//span[@class="currency-symbol"]/a/text()')	

	df['currency-name']=tree.xpath('//tbody/tr//a[@class="currency-name-container"]/text()')

	df['platformsymbol']=tree.xpath('//tbody/tr/@data-platformsymbol')

	df['platform-name']=tree.xpath('//tbody/tr//td[@class="no-wrap platform-name"]/a/text()')



	df['price(->bct)']=tree.xpath('//tbody/tr//td[@class="no-wrap text-right"]/a/@data-btc')

	df['datetime']=datetime.datetime.now()

	return df



def getBtcPrice():

	btc_p_url='https://www.coinegg.com/index/pricebtc'

	try:

		r=requests.get(url=btc_p_url,headers=headers)

	except Exception,e:

		print e

		return None



	return r.json().get('data').get('cny')





def coinegg_coins():

	url='https://www.coinegg.com/coin/btc/allcoin?t={}'.format(random.random())

	print url

	retry=3

	for _ in range(retry):

		text=getWebContent(url)

		if not text:

			continue

		time.sleep(random.random())



	if not text:

		print 'failed to get web content'

		return None



	return map(lambda x:x.upper(),eval(text).keys())



def find_fake():

	df=getCoinList()

	coinegglist=set(coinegg_coins())

	# print coinegglist

	fakecoin=set(df['currency-symbol'].values)

	print coinegglist & fakecoin

def main():

	# datastore()

	# print	getBtcPrice()

	# coinegg_coins()

	find_fake()



if __name__ == '__main__':

	main()

# -*-coding=utf-8-*-



import requests

session = requests.Session()

user = ''

password = ''



def getCode():

    headers = {

        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36'}

    url = 'http://vexx.pro/verify/code.html'

    s = session.get(url=url, headers=headers)



    with open('code.png', 'wb') as f:

        f.write(s.content)



    code = raw_input('input the code: ')

    print 'code is ', code



    login_url = 'http://vexx.pro/login/up_login.html'

    post_data = {

        'moble': user,

        'mobles': '+86',

        'password': password,

        'verify': code,

        'login_token': ''}



    login_s = session.post(url=login_url, headers=header, data=post_data)

    print login_s.status_code



    zzc_url = 'http://vexx.pro/ajax/check_zzc/'

    zzc_s = session.get(url=zzc_url, headers=headers)

    print zzc_s.text



def main():

    getCode()



if __name__ == '__main__':

    main()

import tushare as ts

#获取2017年第3季度的盈利能力数据

ts.get_profit_data(2017,3)

	import scipy.misc

	lena=scipy.misc.lena()

	plt.gray()

	plt.imshow(lena)

	plt.colorbar()

	plt.show()

def profit(code,start,end):

    try:

        df=ts.get_k_data(code,start=start,end=end)

    except Exception,e:

        print e

        return None

    try:

        p=(df['close'].iloc[-1]-df['close'].iloc[0])/df['close'].iloc[0]*100.00

    except Exception,e:

        print e

        return None 

    return round(p,2)

profit('300678','2016-12-31','2017-12-31')

def price_change():

    basic=ts.get_stock_basics()

    pro=



    for code in basic.index.values:

        print code

        p=profit(code,'2016-12-31','2017-12-31')

        pro.append(p)

    basic['price_change']=pro

    basic.to_csv('2017_all_price_change.xls',encoding='gbk')

    df=pd.read_csv('2017_all_price_change.xls',encoding='gbk')

    df.to_excel('2017_all_price_change.xls',encoding='gbk')

def analysis():

    df=pd.read_excel('2017_all_price_change.xls',encoding='gbk')

    df=df.sort_values(by='price_change',ascending=False)

    df.to_excel('2017-year.xls',encoding='gbk')

engine=get_engine('stock')

df.to_sql('2017years',engine)

def get_engine(db):

    engine = create_engine('mysql+pymysql://{}:{}@{}:{}/{}?charset=utf8'.format(MYSQL_USER, MYSQL_PASSWORD, MYSQL_HOST, MYSQL_PORT, db))

    return engine

• 2017-year.zip

    df = pd.DataFrame({'A': [1, 2, 3, 4, 5], 'B': [10, 20, 30, 40, 50]})

    df2 = pd.DataFrame({'A': [6], 'B': [60]})

    print('df', df)

    print('df2', df2)



    df_x = [df, df2]

    result = pd.concat(df_x)

    print('first result ', result)

result2= result.reset_index()

result2 = result.reset_index(drop=True)

result3 = result.reindex(columns=['A','C'])

    df = pd.DataFrame({'A': [1, 2, 3, 4, 5], 'B': [10, 20, 30, 40, 50]})

    df2 = pd.DataFrame({'A': [6], 'B': [60]})

    print 'df\n', df

    print 'df2\n', df2

    df_x = [df, df2]

    result = pd.concat(df_x)

    print 'first result\n', result

    basic=ts.get_stock_basics()

    

    print basic.info()

    print basic.head(10)



    basic.to_excel('basic2017.xls',encoding='gbk')

    basic.to_csv('2017_all_price_change.xls',encoding='gbk')

    df=pd.read_csv('2017_all_price_change.xls',encoding='gbk')

    df.to_excel('2017_all_price_change.xls',encoding='gbk')

	df=pd.read_excel('600050.xls',dtype={'datetime':np.datetime64})

	df=df.set_index('datetime')

	df['close'].plot(use_index=True,grid=True)

	plt.show()

Server has startup warnings: 

2017-12-17T22:56:19.702+0800 I STORAGE  [initandlisten] 

2017-12-17T22:56:19.702+0800 I STORAGE  [initandlisten] ** WARNING: Using the XFS filesystem is strongly recommended with the WiredTiger storage engine

2017-12-17T22:56:19.702+0800 I STORAGE  [initandlisten] **          See http://dochub.mongodb.org/core ... ystem

2017-12-17T22:56:20.600+0800 I CONTROL  [initandlisten] 

2017-12-17T22:56:20.600+0800 I CONTROL  [initandlisten] ** WARNING: Access control is not enabled for the database.

2017-12-17T22:56:20.600+0800 I CONTROL  [initandlisten] **          Read and write access to data and configuration is unrestricted.

2017-12-17T22:56:20.600+0800 I CONTROL  [initandlisten] 

2017-12-17T22:56:20.600+0800 I CONTROL  [initandlisten] ** WARNING: This server is bound to localhost.

2017-12-17T22:56:20.600+0800 I CONTROL  [initandlisten] **          Remote systems will be unable to connect to this server. 

2017-12-17T22:56:20.600+0800 I CONTROL  [initandlisten] **          Start the server with --bind_ip <address> to specify which IP 

2017-12-17T22:56:20.600+0800 I CONTROL  [initandlisten] **          addresses it should serve responses from, or with --bind_ip_all to

2017-12-17T22:56:20.600+0800 I CONTROL  [initandlisten] **          bind to all interfaces. If this behavior is desired, start the

2017-12-17T22:56:20.600+0800 I CONTROL  [initandlisten] **          server with --bind_ip 127.0.0.1 to disable this warning.

2017-12-17T22:56:20.600+0800 I CONTROL  [initandlisten] 

2017-12-17T22:56:20.601+0800 I CONTROL  [initandlisten] 

2017-12-17T22:56:20.601+0800 I CONTROL  [initandlisten] ** WARNING: /sys/kernel/mm/transparent_hugepage/defrag is 'always'.

2017-12-17T22:56:20.601+0800 I CONTROL  [initandlisten] **        We suggest setting it to 'never'

2017-12-17T22:56:20.601+0800 I CONTROL  [initandlisten] 

scrapy startproject headerchange

scrapy genspider headervalidation helloacm.com

class HeadervalidationSpider(scrapy.Spider):

    name = 'headervalidation'

    allowed_domains = ['helloacm.com']

    start_urls = ['http://helloacm.com/api/user-agent/']



    def parse(self, response):

        print '*'*20

        print response.body

        print '*'*20

scrapy crawl headervalidation

# Crawl responsibly by identifying yourself (and your website) on the user-agent

USER_AGENT = 'Hello World'

scrapy crawl headervalidation

2017-12-14 16:17:35 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023

2017-12-14 16:17:35 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://helloacm.com/api/us

er-agent/> from <GET http://helloacm.com/api/user-agent/>

2017-12-14 16:17:36 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://helloacm.com/api/user-agent/> (referer: None)



[b]********************

"Hello World"

********************

[/b]2017-12-14 16:17:37 [scrapy.core.engine] INFO: Closing spider (finished)

2017-12-14 16:17:37 [scrapy.statscollectors] INFO: Dumping Scrapy stats:

{'downloader/request_bytes': 406,

 'downloader/request_count': 2,

 'downloader/request_method_count/GET': 2,

 'downloader/response_bytes': 796,

 'downloader/response_count': 2,

 'downloader/response_status_count/200': 1,

 'downloader/response_status_count/301': 1,

 'finish_reason': 'finished',

 'finish_time': datetime.datetime(2017, 12, 14, 8, 17, 37, 29000),

 'log_count/DEBUG': 3,

 'log_count/INFO': 7,

 'response_received_count': 1,

 'scheduler/dequeued': 2,

 'scheduler/dequeued/memory': 2,

 'scheduler/enqueued': 2,

 'scheduler/enqueued/memory': 2,

 'start_time': datetime.datetime(2017, 12, 14, 8, 17, 35, 137000)}

2017-12-14 16:17:37 [scrapy.core.engine] INFO: Spider closed (finished)

# Override the default request headers:

 DEFAULT_REQUEST_HEADERS = {

    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',

    'Accept-Language': 'en',

    'User-Agent':'Hello World'

 }

class HeadervalidationSpider(scrapy.Spider):

    name = 'headervalidation'

    allowed_domains = ['helloacm.com']





    def start_requests(self):

        header={'User-Agent':'Hello World'}

        yield scrapy.Request(url='http://helloacm.com/api/user-agent/',headers=header)



    def parse(self, response):

        print '*'*20

        print response.body

        print '*'*20

2017-12-14 16:17:35 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023

2017-12-14 16:17:35 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://helloacm.com/api/us

er-agent/> from <GET http://helloacm.com/api/user-agent/>

2017-12-14 16:17:36 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://helloacm.com/api/user-agent/> (referer: None)



********************

"Hello World"

********************

2017-12-14 16:17:37 [scrapy.core.engine] INFO: Closing spider (finished)

2017-12-14 16:17:37 [scrapy.statscollectors] INFO: Dumping Scrapy stats:

#-*-coding=utf-8-*-

from scrapy.contrib.downloadermiddleware.useragent import UserAgentMiddleware



class CustomerUserAgent(UserAgentMiddleware):

    def process_request(self, request, spider):

        ua='HELLO World?????????'

        request.headers.setdefault('User-Agent',ua)

DOWNLOADER_MIDDLEWARES = {

    'headerchange.customerMiddleware.customMiddleware.CustomerUserAgent':10

#    'headerchange.middlewares.MyCustomDownloaderMiddleware': 543,

}