*/10 * * * * date >> /home/user/test.log

*/40 * * * * date >> /home/user/test.log

 File "/usr/local/lib/python2.7/dist-packages/matplotlib/pyplot.py", line 115, in <module>

    _backend_mod, new_figure_manager, draw_if_interactive, _show = pylab_setup()

  File "/usr/local/lib/python2.7/dist-packages/matplotlib/backends/__init__.py", line 32, in pylab_setup

    globals(),locals(),[backend_name],0)

  File "/usr/local/lib/python2.7/dist-packages/matplotlib/backends/backend_gtk3agg.py", line 11, in <module>

    from . import backend_gtk3

  File "/usr/local/lib/python2.7/dist-packages/matplotlib/backends/backend_gtk3.py", line 54, in <module>

    cursors.MOVE          : Gdk.Cursor.new(Gdk.CursorType.FLEUR),

TypeError: constructor returned NULL

# -*- coding=utf-8 -*-

import datetime



__author__ = 'Rocky'

'''

http://30daydo.com

Contact: weigesysu@qq.com

'''

# 每天的涨跌停

import urllib2, re, time, xlrd, xlwt, sys, os

import setting

import pandas as pd

import tushare as ts

from setting import LLogger

reload(sys)

sys.setdefaultencoding('gbk')



logger = LLogger('zdt.log')

class GetZDT:

    def __init__(self,current):

        self.user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/64.0.3282.167 Chrome/64.0.3282.167 Safari/537.36"

        # self.today = time.strftime("%Y%m%d")

        self.today=current

        self.path = os.path.join(os.path.dirname(__file__), 'data')

        self.zdt_url = 'http://home.flashdata2.jrj.com.cn/limitStatistic/ztForce/' + self.today + ".js"

        self.zrzt_url = 'http://hqdata.jrj.com.cn/zrztjrbx/limitup.js'



        self.host = "home.flashdata2.jrj.com.cn"

        self.reference = "http://stock.jrj.com.cn/tzzs/z ... ot%3B



        self.header_zdt = {"User-Agent": self.user_agent,

                           "Host": self.host,

                           "Referer": self.reference}



        self.zdt_indexx = [u'代码', u'名称', u'最新价格', u'涨跌幅', u'封成比', u'封流比', u'封单金额', u'最后一次涨停时间', u'第一次涨停时间', u'打开次数',

                           u'振幅',

                           u'涨停强度']



        self.zrzt_indexx = [u'序号', u'代码', u'名称', u'昨日涨停时间', u'最新价格', u'今日涨幅', u'最大涨幅', u'最大跌幅', u'是否连板', u'连续涨停次数',

                            u'昨日涨停强度', u'今日涨停强度', u'是否停牌', u'昨天的日期', u'昨日涨停价', u'今日开盘价格', u'今日开盘涨幅']

        self.header_zrzt = {"User-Agent": self.user_agent,

                            "Host": "hqdata.jrj.com.cn",

                            "Referer": "http://stock.jrj.com.cn/tzzs/zrztjrbx.shtml"

                            }



    def getdata(self, url, headers, retry=5):

        req = urllib2.Request(url=url, headers=headers)

        for i in range(retry):

            try:

                resp = urllib2.urlopen(req,timeout=20)

                content = resp.read()

                md_check = re.findall('summary|lasttradedate',content)

                if content and len(md_check)>0:

                    return content

                else:

                    time.sleep(60)

                    logger.log('failed to get content, retry: {}'.format(i))

                    continue

            except Exception, e:

                logger.log(e)

                time.sleep(60)

                continue

        return None



    def convert_json(self, content):

        p = re.compile(r'"Data":(.*)};', re.S)

        if len(content)<=0:

            logger.log('Content\'s length is 0')

            exit(0)

        result = p.findall(content)

        if result:

            try:

            # print result

                t1 = result[0]

                t2 = list(eval(t1))

                return t2

            except Exception,e:

                logger.log(e)

                return None

        else:

            return None





    def save_to_dataframe(self, data, indexx, choice, post_fix):

        engine = setting.get_engine('db_zdt')

        if not data:

            exit()

        data_len = len(data)

        if choice == 1:

            for i in range(data_len):

                data[choice] = data[choice].decode('gbk')



        df = pd.DataFrame(data, columns=indexx)



        filename = os.path.join(self.path, self.today + "_" + post_fix + ".xls")

        if choice == 1:

            df[u'今天的日期']=self.today

            df.to_excel(filename, encoding='gbk')

            try:

                df.to_sql(self.today + post_fix, engine, if_exists='fail')

            except Exception,e:

                logger.log(e)





    def storedata(self):

        zdt_content = self.getdata(self.zdt_url, headers=self.header_zdt)

        logger.log('zdt Content'+zdt_content)

        zdt_js = self.convert_json(zdt_content)

        self.save_to_dataframe(zdt_js, self.zdt_indexx, 1, 'zdt')

        time.sleep(5)



if __name__ == '__main__':

    date_list = [datetime.datetime.strftime(i,'%Y%m%d') for i in list(pd.date_range('20170401','20171231'))]

    for today in date_list:



        if not ts.is_holiday(datetime.datetime.strptime(today,'%Y%m%d').strftime('%Y-%m-%d')):

            print today

            obj = GetZDT(today)

            obj.storedata()

        else:

            logger.log('Holiday')

# -*- coding=utf-8 -*-

__author__ = 'Rocky'

'''

http://30daydo.com

Contact: weigesysu@qq.com

'''

# 每天的涨跌停

import re

import time

import xlrd

import xlwt

import sys

import os

import setting

from setting import is_holiday, DATA_PATH

import pandas as pd

import tushare as ts

from setting import llogger

import requests

from send_mail import sender_139

import datetime

# reload(sys)

# sys.setdefaultencoding('gbk')



logger = llogger(__file__)





class GetZDT:

    def __init__(self):

        self.user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/64.0.3282.167 Chrome/64.0.3282.167 Safari/537.36"

        self.today = time.strftime("%Y%m%d")

        self.path = DATA_PATH

        self.zdt_url = 'http://home.flashdata2.jrj.com.cn/limitStatistic/ztForce/' + \

            self.today + ".js"

        self.zrzt_url = 'http://hqdata.jrj.com.cn/zrztjrbx/limitup.js'



        self.host = "home.flashdata2.jrj.com.cn"

        self.reference = "http://stock.jrj.com.cn/tzzs/z ... ot%3B



        self.header_zdt = {"User-Agent": self.user_agent,

                           "Host": self.host,

                           "Referer": self.reference}



        self.zdt_indexx = [u'代码', u'名称', u'最新价格', u'涨跌幅', u'封成比', u'封流比', u'封单金额', u'最后一次涨停时间', u'第一次涨停时间', u'打开次数',

                           u'振幅',

                           u'涨停强度']



        self.zrzt_indexx = [u'序号', u'代码', u'名称', u'昨日涨停时间', u'最新价格', u'今日涨幅', u'最大涨幅', u'最大跌幅', u'是否连板', u'连续涨停次数',

                            u'昨日涨停强度', u'今日涨停强度', u'是否停牌', u'昨天的日期', u'昨日涨停价', u'今日开盘价格', u'今日开盘涨幅']

        self.header_zrzt = {"User-Agent": self.user_agent,

                            "Host": "hqdata.jrj.com.cn",

                            "Referer": "http://stock.jrj.com.cn/tzzs/zrztjrbx.shtml"

                            }



    def getdata(self, url, headers, retry=5):

        for i in range(retry):

            try:

                resp = requests.get(url=url, headers=headers)

                content = resp.text

                md_check = re.findall('summary|lasttradedate', content)

                if content and len(md_check) > 0:

                    return content

                else:

                    time.sleep(60)

                    logger.info('failed to get content, retry: {}'.format(i))

                    continue

            except Exception as e:

                logger.info(e)

                time.sleep(60)

                continue

        return None



    def convert_json(self, content):

        p = re.compile(r'"Data":(.*)};', re.S)

        if len(content) <= 0:

            logger.info('Content\'s length is 0')

            exit(0)

        result = p.findall(content)

        if result:

            try:

                # print(result)

                t1 = result[0]

                t2 = list(eval(t1))

                return t2

            except Exception as e:

                logger.info(e)

                return None

        else:

            return None



    # 2016-12-27 to do this

    def save_excel(self, date, data):

        # data is list type

        w = xlwt.Workbook(encoding='gbk')

        ws = w.add_sheet(date)

        excel_filename = date + ".xls"

        # sheet=open_workbook(excel_filenme)

        # table=wb.sheets()[0]

        xf = 0

        ctype = 1

        rows = len(data)

        point_x = 1

        point_y = 0

        ws.write(0, 0, u'代码')

        ws.write(0, 1, u'名称')

        ws.write(0, 2, u'最新价格')

        ws.write(0, 3, u'涨跌幅')

        ws.write(0, 4, u'封成比')

        ws.write(0, 5, u'封流比')

        ws.write(0, 6, u'封单金额')

        ws.write(0, 7, u'第一次涨停时间')

        ws.write(0, 8, u'最后一次涨停时间')

        ws.write(0, 9, u'打开次数')

        ws.write(0, 10, u'振幅')

        ws.write(0, 11, u'涨停强度')

        print("Rows:%d" % rows)

        for row in data:

            rows = len(data)

            cols = len(row)

            point_y = 0

            for col in row:

                # print(col)

                # table.put_cell(row,col,)

                # print(col)

                ws.write(point_x, point_y, col)

                # print("[%d,%d]" % (point_x, point_y))

                point_y = point_y + 1



            point_x = point_x + 1



        w.save(excel_filename)



    def save_to_dataframe(self, data, indexx, choice, post_fix):

        engine = setting.get_engine('db_zdt')

        if not data:

            exit()

        data_len = len(data)

        if choice == 1:

            for i in range(data_len):

                data[i][choice] = data[i][choice]



        df = pd.DataFrame(data, columns=indexx)



        filename = os.path.join(

            self.path, self.today + "_" + post_fix + ".xls")



        # 今日涨停

        if choice == 1:

            df['今天的日期'] = self.today

            df.to_excel(filename, encoding='gbk')

            try:

                df.to_sql(self.today + post_fix, engine, if_exists='fail')

            except Exception as e:

                logger.info(e)

        # 昨日涨停

        if choice == 2:

            df = df.set_index(u'序号')

            df[u'最大涨幅'] = df[u'最大涨幅'].map(lambda x: round(x * 100, 3))

            df[u'最大跌幅'] = df[u'最大跌幅'].map(lambda x: round(x * 100, 3))

            df[u'今日开盘涨幅'] = df[u'今日开盘涨幅'].map(lambda x: round(x * 100, 3))

            df[u'昨日涨停强度'] = df[u'昨日涨停强度'].map(lambda x: round(x, 0))

            df[u'今日涨停强度'] = df[u'今日涨停强度'].map(lambda x: round(x, 0))

            try:

                df.to_sql(self.today + post_fix, engine, if_exists='fail')

            except Exception as e:

                logger.info(e)



            avg = round(df['今日涨幅'].mean(), 2)

            current = datetime.datetime.now().strftime('%Y-%m-%d')

            title = '昨天涨停个股今天{}\n的平均涨幅{}\n'.format(current, avg)

            try:

                sender_139(title, title)

            except Exception as e:

                print(e)



    # 昨日涨停今日的状态，今日涨停



    def storedata(self):

        zdt_content = self.getdata(self.zdt_url, headers=self.header_zdt)

        logger.info('zdt Content' + zdt_content)

        zdt_js = self.convert_json(zdt_content)

        self.save_to_dataframe(zdt_js, self.zdt_indexx, 1, 'zdt')

        time.sleep(0.5)

        zrzt_content = self.getdata(self.zrzt_url, headers=self.header_zrzt)

        logger.info('zrzt Content' + zdt_content)



        zrzt_js = self.convert_json(zrzt_content)

        self.save_to_dataframe(zrzt_js, self.zrzt_indexx, 2, 'zrzt')





if __name__ == '__main__':

    # today='2018-04-16'

    # 填补以前的数据

    # x=pd.date_range('20170101','20180312')

    # date_list = [datetime.datetime.strftime(i,'%Y%m%d') for i in list(pd.date_range('20170401','20171231'))



    if is_holiday():

        logger.info('Holiday')

        exit()

    logger.info("start")

    obj = GetZDT()

    obj.storedata()

[6.2, 20.4, 18.6, 20.0, 1366.2, 117.2, 609.4, 237.2, 407.0, 798.8, 1324.6, 46327.6, 81275.4, 1604854.8, 11903.0, 24018.8, 783.8, 27538.8, 867.4, 2291.2, 3753.0, 10562.0, 6540.6, 3519988.4, 15201.8, 405.2, 135648.2, 101014.8, 15703.4, 5506.0, 5369.0, 7870.0, 3815.0, 8325712.4, 23636.6, 1556.8, 95365.8, 1262.4, 625404.2, 6300430.8, 126025.0, 31276.0, 28717.4, 23184.0, 29007.4, 20586.0, 4581.0, 10091574.4, 111960.6, 3848.0, 1643635.4, 135605.6, 80114.2, 12795.2, 14133.4, 724014.4, 5365.4, 12682.8, 7472.2, 10351.6, 35163.8, 542070.8, 22053.0, 6292.4, 83359.4, 15030.8, 45987.8, 114825.2, 3121.4, 609555.6, 1307542.6, 8402.4, 62410.2, 231369.6, 285103.8, 660060.4, 719555.8, 69072.0, 41764.6, 181634.8, 40792.2, 57721.2, 59925.8, 134405.6, 41730.6, 62014.0, 1200179.8, 32587184.0, 5717.4, 13583.2, 24479.4, 339389.2, 12449571.4, 29882201.2, 2729118.6, 942646.8, 37097.8, 4795026.0, 359242.2]

#-*-coding=utf-8

'''

可转债监控

'''

import tushare as ts

from setting import get_engine

engine = get_engine('db_bond')

import pandas as pd

import datetime

class ConvertBond():



    def __init__(self):

        self.conn=ts.get_apis()

        self.allBonds=ts.new_cbonds(pause=2)

        self.onSellBond=self.allBonds.dropna(subset=['marketprice'])

        self.today=datetime.datetime.now().strftime('%Y-%m-%d %H:%M')



    def stockPrice(self,code):

        stock_df = ts.get_realtime_quotes(code)

        price = float(stock_df['price'].values[0])

        return price



    def dataframe(self):

        price_list=

        for code in self.onSellBond['scode']:

            price_list.append(self.stockPrice(code))

        self.onSellBond['stock_price']=price_list

        self.onSellBond['ratio'] = (

self.onSellBond['marketprice']

/(self.onSellBond['stock_price'] / self.onSellBond['convprice'])-1)*100

        self.onSellBond['Updated']=self.today

        self.onSellBond.to_sql('tb_bond',engine,if_exists='replace')



    def closed(self):

        ts.close_apis(self.conn)



def main():

    bond=ConvertBond()

    bond.dataframe()

    bond.closed()

if __name__=='__main__':

    main()

import os

import MySQLdb

MYSQL_USER = *********

MYSQL_PASSWORD = ********

MYSQL_HOST = *********

MYSQL_PORT = *****



def get_engine(db):

    engine = create_engine('mysql+pymysql://{}:{}@{}:{}/{}?charset=utf8'.format(MYSQL_USER, MYSQL_PASSWORD, MYSQL_HOST, MYSQL_PORT, db))

    return engine

urlparse.urldefrag(url)¶

If url contains a fragment identifier, returns a modified version of url with no fragment identifier, and the fragment identifier as a separate string. If there is no fragment identifier in url, returns url unmodified and an empty string.

code='300527'

df = ts.get_today_ticks(code)

df.sort_values(by='volume',ascending=False).head(100)

import re

from bs4 import BeautifulSoup

import urllib2

from mylog import MyLog as mylog

from save2excel import SavaBallDate



class DoubleColorBallItem(object):

    date = None

    order = None

    red1 = None

    red2 = None

    red3 = None

    red4 = None

    red5 = None

    red6 = None

    blue = None

    money = None

    firstPrize = None

    secondPrize = None



class GetDoubleColorBallNumber(object):

    '''这个类用于获取双色球中奖号码， 返回一个txt文件

    '''

    def __init__(self):

        self.urls = 

        self.log = mylog()

        self.getUrls()

        self.items = self.spider(self.urls)

        self.pipelines(self.items)

        self.log.info('beging save data to excel \r\n')

        SavaBallDate(self.items)

        self.log.info('save data to excel end ...\r\n')

      

        

    def getUrls(self):

        '''获取数据来源网页

        '''

        URL = r'http://kaijiang.zhcw.com/zhcw/html/ssq/list_1.html'

        htmlContent = self.getResponseContent(URL)

        soup = BeautifulSoup(htmlContent, 'lxml')

        tag = soup.find_all(re.compile('p'))[-1]

        pages = tag.strong.get_text()        

        for i in xrange(1, int(pages)+1):

            url = r'http://kaijiang.zhcw.com/zhcw/html/ssq/list_' + str(i) + '.html'

            self.urls.append(url)

            self.log.info(u'添加URL:%s 到URLS \r\n' %url)

            

    def getResponseContent(self, url):

        '''这里单独使用一个函数返回页面返回值，是为了后期方便的加入proxy和headers等

        '''

        try:

            response = urllib2.urlopen(url.encode('utf8'))

        except:

            self.log.error(u'Python 返回URL:%s  数据失败  \r\n' %url)

        else:

            self.log.info(u'Python 返回URUL:%s  数据成功 \r\n' %url)

            return response.read()

        

            

    def spider(self,urls):

        '''这个函数的作用是从获取的数据中过滤得到中奖信息

        '''

        items = 

        for url in urls:

            htmlContent = self.getResponseContent(url)

            soup = BeautifulSoup(htmlContent, 'lxml')

            tags = soup.find_all('tr', attrs={})

            for tag in tags:

                if tag.find('em'):

                    item = DoubleColorBallItem()

                    tagTd = tag.find_all('td')

                    item.date = tagTd[0].get_text()

                    item.order = tagTd[1].get_text()

                    tagEm = tagTd[2].find_all('em')

                    item.red1 = tagEm[0].get_text()

                    item.red2 = tagEm[1].get_text()

                    item.red3 = tagEm[2].get_text()

                    item.red4 = tagEm[3].get_text()

                    item.red5 = tagEm[4].get_text()

                    item.red6 = tagEm[5].get_text()

                    item.blue = tagEm[6].get_text()

                    item.money = tagTd[3].find('strong').get_text()

                    item.firstPrize = tagTd[4].find('strong').get_text()

                    item.secondPrize = tagTd[5].find('strong').get_text()

                    items.append(item)

                    self.log.info(u'获取日期为:%s 的数据成功' %(item.date))

        return items

    

    def pipelines(self,items):

        fileName = u'双色球.txt'.encode('GBK')

        with open(fileName, 'w') as fp:

            for item in items:

                fp.write('%s %s \t %s %s %s %s %s %s  %s \t %s \t %s %s \n'

                      %(item.date,item.order,item.red1,item.red2,item.red3,item.red4,item.red5,item.red6,item.blue,item.money,item.firstPrize,item.secondPrize))

                self.log.info(u'将日期为:%s 的数据存入"%s"...' %(item.date, fileName.decode('GBK')))

                    



if __name__ == '__main__':

    GDCBN = GetDoubleColorBallNumber()

import logging

import getpass

import sys





#### 定义MyLog类

class MyLog(object):

#### 类MyLog的构造函数

	def __init__(self):

		self.user = getpass.getuser()

		self.logger = logging.getLogger(self.user)

		self.logger.setLevel(logging.DEBUG)



####  日志文件名

		self.logFile = sys.argv[0][0:-3] + '.log'

		self.formatter = logging.Formatter('%(asctime)-12s %(levelname)-8s %(name)-10s %(message)-12s\r\n')



####  日志显示到屏幕上并输出到日志文件内

		self.logHand = logging.FileHandler(self.logFile, encoding='utf8')

		self.logHand.setFormatter(self.formatter)

		self.logHand.setLevel(logging.DEBUG)



		self.logHandSt = logging.StreamHandler()

		self.logHandSt.setFormatter(self.formatter)

		self.logHandSt.setLevel(logging.DEBUG)



		self.logger.addHandler(self.logHand)

		self.logger.addHandler(self.logHandSt)



####  日志的5个级别对应以下的5个函数

	def debug(self,msg):

		self.logger.debug(msg)



	def info(self,msg):

		self.logger.info(msg)



	def warn(self,msg):

		self.logger.warn(msg)



	def error(self,msg):

		self.logger.error(msg)



	def critical(self,msg):

		self.logger.critical(msg)



if __name__ == '__main__':

	mylog = MyLog()

	mylog.debug(u"I'm debug 测试中文")

	mylog.info("I'm info")

	mylog.warn("I'm warn")

	mylog.error(u"I'm error 测试中文")

	mylog.critical("I'm critical")

import xlwt

class SavaBallDate(object):

    def __init__(self, items):

        self.items = items

        self.run(self.items)

        

    def run(self,items):

        fileName = u'双色球.xls'.encode('GBK')

        book = xlwt.Workbook(encoding='utf8')

        sheet=book.add_sheet('ball', cell_overwrite_ok=True)

        sheet.write(0, 0, u'开奖日期'.encode('utf8'))

        sheet.write(0, 1, u'期号'.encode('utf8'))

        sheet.write(0, 2, u'红1'.encode('utf8'))

        sheet.write(0, 3, u'红2'.encode('utf8'))

        sheet.write(0, 4, u'红3'.encode('utf8'))

        sheet.write(0, 5, u'红4'.encode('utf8'))

        sheet.write(0, 6, u'红5'.encode('utf8'))

        sheet.write(0, 7, u'红6'.encode('utf8'))

        sheet.write(0, 8, u'蓝'.encode('utf8'))

        sheet.write(0, 9, u'销售金额'.encode('utf8'))

        sheet.write(0, 10, u'一等奖'.encode('utf8'))

        sheet.write(0, 11, u'二等奖'.encode('utf8'))

        i = 1

        while i <= len(items):

            item = items[i-1]

            sheet.write(i, 0, item.date)

            sheet.write(i, 1, item.order)

            sheet.write(i, 2, item.red1)

            sheet.write(i, 3, item.red2)

            sheet.write(i, 4, item.red3)

            sheet.write(i, 5, item.red4)

            sheet.write(i, 6, item.red5)

            sheet.write(i, 7, item.red6)

            sheet.write(i, 8, item.blue)

            sheet.write(i, 9, item.money)

            sheet.write(i, 10, item.firstPrize)

            sheet.write(i, 11, item.secondPrize)

            i += 1

        book.save(fileName)

        





if __name__ == '__main__':

    pass