获取 排除满足强赎天数剩余N天的可转债，排除强赎可转债。



比如：DAY=4 ，那么假设强赎满足天数为15天，那么返回市场上强赎强赎倒数天数大于4天的可转债，即返回小于或者等于11天以下的转债。把满足条件还差4天的转债排除掉了，当然，已经公告强赎的也会被排除。







import requests



URL = 'http://11.11.11.11/api/redeem/'

# DAY 强赎倒数剩余天数小于 DAY （4）天， 已经公告强赎的也是被排除的

DAY = 4



SIGN = '私信星主获取' # sign和以前一致



data = {'day': DAY,

        'sign': SIGN}

r = requests.post(URL, data=data)



print(r.json())

#获取当日的股票池

g.stock_list = get_Ashares(g.current_date)

get_history(70, '1d', ['close','volume'], g.stock_list, fq='dypre', include=False)

class Command(ScrapyCommand):



    requires_project = True



    def syntax(self):

        return '[options]'



    def short_desc(self):

        return 'Runs all of the spiders - My Defined'



    def run(self,args,opts):

        print('==================')

        print(type(self.crawler_process))

        spider_list = self.crawler_process.spiders.list() # 找到爬虫类



        for name in spider_list:

            print('=================')

            print(name)

            self.crawler_process.crawl(name,**opts.__dict__)



        self.crawler_process.start()

      def __init__(self, settings=None):

        if isinstance(settings, dict) or settings is None:

            settings = Settings(settings)

        self.settings = settings

        self.spider_loader = _get_spider_loader(settings) # 构造爬虫

        self._crawlers = set()

        self._active = set()

        self.bootstrap_failed = False

def _get_spider_loader(settings):



    cls_path = settings.get('SPIDER_LOADER_CLASS')

    

    # settings文件没有定义SPIDER_LOADER_CLASS，所以这里获取到的是系统的默认配置文件，

    # 默认配置文件在接下来的代码块A

    # SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'    

    

    loader_cls = load_object(cls_path) 

    # 这个函数就是根据路径转为类对象，也就是上面crapy.spiderloader.SpiderLoader 这个

    # 字符串变成一个类对象

    # 具体的load_object 对象代码见下面代码块B



    return loader_cls.from_settings(settings.frozencopy())

# 代码块A

#......省略若干

SCHEDULER = 'scrapy.core.scheduler.Scheduler'

SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'

SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'

SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.ScrapyPriorityQueue'



SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader' 就是这个值

SPIDER_LOADER_WARN_ONLY = False



SPIDER_MIDDLEWARES = {}

# 代码块B 为了方便，我把异常处理的去除

from importlib import import_module #导入第三方库



def load_object(path):

    dot = path.rindex('.') 

    module, name = path[:dot], path[dot+1:]

    # 上面把路径分为基本路径+模块名

    

    mod = import_module(module)

    obj = getattr(mod, name)

    # 获取模块里面那个值

    

    return obj

In [33]: mod = import_module(module)                                                                                                                                             



In [34]: mod                                                                                                                                                                     

Out[34]: <module 'scrapy.spiderloader' from '/home/xda/anaconda3/lib/python3.7/site-packages/scrapy/spiderloader.py'>



In [35]: getattr(mod,name)                                                                                                                                                       

Out[35]: scrapy.spiderloader.SpiderLoader



In [36]: obj = getattr(mod,name)                                                                                                                                                 



In [37]: obj                                                                                                                                                                     

Out[37]: scrapy.spiderloader.SpiderLoader



In [38]: type(obj)                                                                                                                                                               

Out[38]: type

    def from_settings(cls, settings):

        return cls(settings)

class SpiderLoader(object):

    """

    SpiderLoader is a class which locates and loads spiders

    in a Scrapy project.

    """

    def __init__(self, settings):

        self.spider_modules = settings.getlist('SPIDER_MODULES') 

        # 获得settting中的模块名字，创建scrapy的时候就默认帮你生成了

        # 你可以看看你的settings文件里面的内容就可以找到这个值，是一个list

        

        self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY')

        self._spiders = {}

        self._found = defaultdict(list)

        self._load_all_spiders() # 加载所有爬虫

def _load_all_spiders(self):

        for name in self.spider_modules:            



                for module in walk_modules(name): # 这个遍历文件夹里面的文件，然后再转化为类对象，

                    # 保存到字典：self._spiders = {}       

                    self._load_spiders(module) # 模块变成spider



        self._check_name_duplicates() # 去重，如果名字一样就异常

def iter_spider_classes(module):

    from scrapy.spiders import Spider



    for obj in six.itervalues(vars(module)): # 找到模块里面的变量，然后迭代出来

        if inspect.isclass(obj) and \

           issubclass(obj, Spider) and \

           obj.__module__ == module.__name__ and \

           getattr(obj, 'name', None): # 有name属性，继承于Spider

           yield obj

def function_2(x):

    return x[0]**2+x[1]**2

def numerical_gradient(f,x):

    grad = np.zeros_like(x)

    h=1e-4

    for idx in range(x.size):

        temp_v = x[idx]

        x[idx]=temp_v+h

        f1=f(x)

        print(x,f1)

        x[idx]=temp_v-h

        f2=f(x)

        print(x,f2)

        ret = (f1-f2)/(2*h)

        print(ret)

        x[idx]=temp_v

        grad[idx]=ret

        

    return grad

numerical_gradient(function_2,np.array([3,4]))

a=np.array([[[1,2],[3,4]],[[11,12],[13,14]]])

array([[[ 1,  2],

        [ 3,  4]],



       [[11, 12],

        [13, 14]]])

a.sum(axis=0)

array([[12, 14],

       [16, 18]])

a.sum(axis=1)

array([[ 4,  6],

       [24, 26]])

a.sum(axis=-3)

array([[12, 14],

       [16, 18]])

# -*- coding: utf-8 -*-

# website: http://30daydo.com

# @Time : 2019/2/23 19:34

# @File : youdao.py

# 解密有道词典的JS





import hashlib

import random

import requests

import time





def md5_(word):

    s = bytes(word, encoding='utf8')

    m = hashlib.md5()

    m.update(s)

    ret = m.hexdigest()

    return ret



def get_sign(word, salt):

    ret = md5_('fanyideskweb' + word + salt + 'p09@Bn{h02_BIEe]$P^nG')

    return ret





def youdao(word):

    url = 'http://fanyi.youdao.com/translate_o?smartresult=dict&smartresult=rule'

    headers = {

        'Host': 'fanyi.youdao.com',

        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0',

        'Accept': 'application/json, text/javascript, */*; q=0.01',

        'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',

        'Accept-Encoding': 'gzip, deflate',

        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',

        'X-Requested-With': 'XMLHttpRequest',

        'Referer': 'http://fanyi.youdao.com/',

        'Content-Length': '252',

        'Cookie': 'YOUDAO_MOBILE_ACCESS_TYPE=1; OUTFOX_SEARCH_USER_ID=1672542763@10.169.0.83; JSESSIONID=aaaWzxpjeDu1gbhopLzKw; ___rl__test__cookies=1550913722828; OUTFOX_SEARCH_USER_ID_NCOO=372126049.6326876',

        'Connection': 'keep-alive',

        'Pragma': 'no-cache',

        'Cache-Control': 'no-cache',

    }



    ts = str(int(time.time()*1000))

    salt=ts+str(random.randint(0,10))

    bv = md5_("5.0 (Windows)")

    sign= get_sign(word,salt)



    post_data = {

        'i': word,

        'from': 'AUTO', 'to': 'AUTO', 'smartresult': 'dict', 'client': 'fanyideskweb', 'salt': salt,

        'sign': sign, 'ts': ts, 'bv': bv, 'doctype': 'json', 'version': '2.1',

        'keyfrom': 'fanyi.web', 'action': 'FY_BY_REALTIME', 'typoResult': 'false'

    }



    r = requests.post(

        url=url,

        headers=headers,

        data=post_data

    )



    for item in r.json().get('smartResult',{}).get('entries'):

        print(item)



word='student'

youdao(word)

from PIL import Image

im = Image.open('chinese.jpg')

plt.figure(figsize=(20,20))

plt.imshow(im)



pytesseract.image_to_string(im,lang='chi_sim')

'可L又使用以下的语句i上图片显示大 此'

""" 读取图片 """

def get_file_content(filePath):

    with open(filePath, 'rb') as fp:

        return fp.read()



image = get_file_content('example.jpg')



""" 调用通用文字识别, 图片参数为本地图片 """

client.basicGeneral(image);



""" 如果有可选参数 """

options = {}

options["language_type"] = "CHN_ENG"

options["detect_direction"] = "true"

options["detect_language"] = "true"

options["probability"] = "true"



from aip import AipOcr



""" 你的 APPID AK SK """

APP_ID = '你的 App ID'

API_KEY = '你的 Api Key'

SECRET_KEY = '你的 Secret Key'



client = AipOcr(APP_ID, API_KEY, SECRET_KEY)





""" 带参数调用通用文字识别, 图片参数为本地图片 """

client.basicGeneral(image, options)



url = "https//www.x.com/sample.jpg"



""" 调用通用文字识别, 图片参数为远程url图片 """

client.basicGeneralUrl(url);



""" 如果有可选参数 """

options = {}

options["language_type"] = "CHN_ENG"

options["detect_direction"] = "true"

options["detect_language"] = "true"

options["probability"] = "true"



""" 带参数调用通用文字识别, 图片参数为远程url图片 """

client.basicGeneralUrl(url, options)

可以使用以下的语句让图片显示大些

from  setting import get_mysql_conn,get_engine

import pandas as pd

import pymongo

from pyecharts import Geo,Style,Map

engine = get_engine('db_stock',local='local')

# 堆叠图

from pyecharts import Bar

df = pd.read_sql('tb_bond_jisilu',con=engine)



result ={}

for name,grades in df.groupby('评级'):

#     print(name,grades[['可转债名称','可转债价格']])

    for each in grades['可转债价格']:

        result.setdefault(name,)

        result[name].append(each)

        

        

# 确定价格的范围



value = [str(i) for i in range(85,140)]

ret = [0]*len(value)

ret1 = dict(zip(value,ret))



ret_A_add = ret1.copy()

for item in result['A+']:

    for k in ret1:

        if float(k)+0.5>item and float(k)-0.5<=item:

            ret_A_add[k]+=1

            

retAA_ = ret1.copy()

for item in result['']:

    for k in ret1:

        if float(k)+0.5>item and float(k)-0.5<=item:

            retAA_[k]+=1



retAA = ret1.copy()

for item in result['AA']:

    for k in ret1:

        if float(k)+0.5>item and float(k)-0.5<=item:

            retAA[k]+=1

            

retAA_add = ret1.copy()

for item in result['AA+']:

    for k in ret1:

        if float(k)+0.5>item and float(k)-0.5<=item:

            retAA_add[k]+=1

            

retAAA = ret1.copy()

for item in result['AAA']:

    for k in ret1:

        if float(k)+0.5>item and float(k)-0.5<=item:

            retAAA[k]+=1



bar = Bar('可转债价格分布')

bar.add('A+',value,list(ret_A_add.values()),is_stack=True,yaxis_max=11)

bar.add('',value,list(retAA_.values()),is_stack=True,yaxis_max=11)

bar.add('AA',value,list(retAA.values()),is_stack=True,yaxis_max=11)

bar.add('AA+',value,list(retAA_add.values()),is_stack=True,yaxis_max=11)

bar.add('AAA',value,list(retAAA.values()),is_stack=True,yaxis_max=11)

#-*-coding=utf-8

'''

可转债监控

'''

import tushare as ts

from setting import get_engine

engine = get_engine('db_bond')

import pandas as pd

import datetime

class ConvertBond():



    def __init__(self):

        self.conn=ts.get_apis()

        self.allBonds=ts.new_cbonds(pause=2)

        self.onSellBond=self.allBonds.dropna(subset=['marketprice'])

        self.today=datetime.datetime.now().strftime('%Y-%m-%d %H:%M')



    def stockPrice(self,code):

        stock_df = ts.get_realtime_quotes(code)

        price = float(stock_df['price'].values[0])

        return price



    def dataframe(self):

        price_list=

        for code in self.onSellBond['scode']:

            price_list.append(self.stockPrice(code))

        self.onSellBond['stock_price']=price_list

        self.onSellBond['ratio'] = (

self.onSellBond['marketprice']

/(self.onSellBond['stock_price'] / self.onSellBond['convprice'])-1)*100

        self.onSellBond['Updated']=self.today

        self.onSellBond.to_sql('tb_bond',engine,if_exists='replace')



    def closed(self):

        ts.close_apis(self.conn)



def main():

    bond=ConvertBond()

    bond.dataframe()

    bond.closed()

if __name__=='__main__':

    main()

import os

import MySQLdb

MYSQL_USER = *********

MYSQL_PASSWORD = ********

MYSQL_HOST = *********

MYSQL_PORT = *****



def get_engine(db):

    engine = create_engine('mysql+pymysql://{}:{}@{}:{}/{}?charset=utf8'.format(MYSQL_USER, MYSQL_PASSWORD, MYSQL_HOST, MYSQL_PORT, db))

    return engine

    df = pd.DataFrame({'A': [1, 2, 3, 4, 5], 'B': [10, 20, 30, 40, 50]})

    df2 = pd.DataFrame({'A': [6], 'B': [60]})

    print('df', df)

    print('df2', df2)



    df_x = [df, df2]

    result = pd.concat(df_x)

    print('first result ', result)

result2= result.reset_index()

result2 = result.reset_index(drop=True)

result3 = result.reindex(columns=['A','C'])

    if df is None:

        print "None len==0"

        return False

__author__ = 'rocchen'

# working v1.0

from bs4 import BeautifulSoup

import urllib2, datetime, time, codecs, cookielib, random, threading

import os,sys





def getInfo(max_index_user=5):

    stock_news_site =

"http://ggjd.cnstock.com/gglist/search/ggkx/"

 

    my_userAgent = [

        'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',

        'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',

        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0',

        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)',

        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)',

        'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',

        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',

        'Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',

        'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11',

        'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11',

        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)',

        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',

        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)',

        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)']

    index = 0

    max_index = max_index_user

    num = 1

    temp_time = time.strftime("[%Y-%m-%d]-[%H-%M]", time.localtime())



    store_filename = "StockNews-%s.log" % temp_time

    fOpen = codecs.open(store_filename, 'w', 'utf-8')



    while index < max_index:

        user_agent = random.choice(my_userAgent)

        # print user_agent

        company_news_site = stock_news_site + str(index)

        # content = urllib2.urlopen(company_news_site)

        headers = {'User-Agent': user_agent, 'Host': "ggjd.cnstock.com", 'DNT': '1',

                   'Accept': 'text/html, application/xhtml+xml, */*', }

        req = urllib2.Request(url=company_news_site, headers=headers)

        resp = None

        raw_content = ""

        try:

            resp = urllib2.urlopen(req, timeout=30)



        except urllib2.HTTPError as e:

            e.fp.read()

        except urllib2.URLError as e:

            if hasattr(e, 'code'):

                print "error code %d" % e.code

            elif hasattr(e, 'reason'):

                print "error reason %s " % e.reason



        finally:

            if resp:

                raw_content = resp.read()

                time.sleep(2)

                resp.close()



        soup = BeautifulSoup(raw_content, "html.parser")

        all_content = soup.find_all("span", "time")



        for i in all_content:

            news_time = i.string

            node = i.next_sibling

            str_temp = "No.%s \n%s\t%s\n---> %s \n\n" % (str(num), news_time, node['title'], node['href'])

            #print "inside %d" %num

            #print str_temp

            fOpen.write(str_temp)

            num = num + 1



        #print "index %d" %index

        index = index + 1



    fOpen.close()





def execute_task(n=60):

    period = int(n)

    while True:

        print datetime.datetime.now()

        getInfo(3)

        

        time.sleep(60 * period)

        





if __name__ == "__main__":



    sub_folder = os.path.join(os.getcwd(), "stock")

    if not os.path.exists(sub_folder):

        os.mkdir(sub_folder)

    os.chdir(sub_folder)

    start_time = time.time()  # user can change the max index number getInfo(10), by default is getInfo(5)

    if len(sys.argv) <2:

        n = raw_input("Input Period : ? mins to download every cycle")

    else:

        n=int(sys.argv[1])

    execute_task(n)

    end_time = time.time()

    print "Total time: %s s." % str(round((end_time - start_time), 4))

#-*-coding=utf-8-*-

__author__ = 'rocky chen'

from bs4 import BeautifulSoup

import urllib2,sys,StringIO,gzip,time,random,re,urllib,os

reload(sys)

sys.setdefaultencoding('utf-8')

class Xitek():

    def __init__(self):

        self.url="http://photo.xitek.com/"

        user_agent="Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)"

        self.headers={"User-Agent":user_agent}

        self.last_page=self.__get_last_page()





    def __get_last_page(self):

        html=self.__getContentAuto(self.url)

        bs=BeautifulSoup(html,"html.parser")

        page=bs.find_all('a',class_="blast")

        last_page=page[0]['href'].split('/')[-1]

        return int(last_page)





    def __getContentAuto(self,url):

        req=urllib2.Request(url,headers=self.headers)

        resp=urllib2.urlopen(req)

        #time.sleep(2*random.random())

        content=resp.read()

        info=resp.info().get("Content-Encoding")

        if info==None:

            return content

        else:

            t=StringIO.StringIO(content)

            gziper=gzip.GzipFile(fileobj=t)

            html = gziper.read()

            return html



    #def __getFileName(self,stream):





    def __download(self,url):

        p=re.compile(r'href="(/photoid/\d+)"')

        #html=self.__getContentNoZip(url)



        html=self.__getContentAuto(url)



        content = p.findall(html)

        for i in content:

            print i



            photoid=self.__getContentAuto(self.url+i)

            bs=BeautifulSoup(photoid,"html.parser")

            final_link=bs.find('img',class_="mimg")['src']

            print final_link

            #pic_stream=self.__getContentAuto(final_link)

            title=bs.title.string.strip()

            filename = re.sub('[\/:*?"<>|]', '-', title)

            filename=filename+'.jpg'

            urllib.urlretrieve(final_link,filename)

            #f=open(filename,'w')

            #f.write(pic_stream)

            #f.close()

        #print html

        #bs=BeautifulSoup(html,"html.parser")

        #content=bs.find_all(p)

        #for i in content:

        #    print i

        '''

        print bs.title

        element_link=bs.find_all('div',class_="element")

        print len(element_link)

        k=1

        for href in element_link:



            #print type(href)

            #print href.tag

        '''

        '''

            if href.children[0]:

                print href.children[0]

        '''

        '''

            t=0



            for i in href.children:

                #if i.a:

                if t==0:

                    #print k

                    if i['href']

                    print link



                        if p.findall(link):

                            full_path=self.url[0:len(self.url)-1]+link

                            sub_html=self.__getContent(full_path)

                            bs=BeautifulSoup(sub_html,"html.parser")

                            final_link=bs.find('img',class_="mimg")['src']

                            #time.sleep(2*random.random())

                            print final_link

                    #k=k+1

                #print type(i)

                #print i.tag

                #if hasattr(i,"href"):

                    #print i['href']

                #print i.tag

                t=t+1

                #print "*"



        '''



        '''

            if href:

                if href.children:

                    print href.children[0]

        '''

            #print "one element link"







    def getPhoto(self):



        start=0

        #use style/0

        photo_url="http://photo.xitek.com/style/0/p/"

        for i in range(start,self.last_page+1):

            url=photo_url+str(i)

            print url

            #time.sleep(1)

            self.__download(url)



        '''

        url="http://photo.xitek.com/style/0/p/10"

        self.__download(url)

        '''

        #url="http://photo.xitek.com/style/0/p/0"

        #html=self.__getContent(url)

        #url="http://photo.xitek.com/"

        #html=self.__getContentNoZip(url)

        #print html

        #'''

def main():

    sub_folder = os.path.join(os.getcwd(), "content")

    if not os.path.exists(sub_folder):

        os.mkdir(sub_folder)

    os.chdir(sub_folder)

    obj=Xitek()

    obj.getPhoto()





if __name__=="__main__":

    main()

# Copyright (c) 2009 rupa deadwyler under the WTFPL license



# maintains a jump-list of the directories you actually use

#

# INSTALL:

#     * put something like this in your .bashrc/.zshrc:

#         . /path/to/z.sh

#     * cd around for a while to build up the db

#     * PROFIT!!

#     * optionally:

#         set $_Z_CMD in .bashrc/.zshrc to change the command (default z).

#         set $_Z_DATA in .bashrc/.zshrc to change the datafile (default ~/.z).

#         set $_Z_NO_RESOLVE_SYMLINKS to prevent symlink resolution.

#         set $_Z_NO_PROMPT_COMMAND if you're handling PROMPT_COMMAND yourself.

#         set $_Z_EXCLUDE_DIRS to an array of directories to exclude.

#         set $_Z_OWNER to your username if you want use z while sudo with $HOME kept

#

# USE:

#     * z foo     # cd to most frecent dir matching foo

#     * z foo bar # cd to most frecent dir matching foo and bar

#     * z -r foo  # cd to highest ranked dir matching foo

#     * z -t foo  # cd to most recently accessed dir matching foo

#     * z -l foo  # list matches instead of cd

#     * z -c foo  # restrict matches to subdirs of $PWD



[ -d "${_Z_DATA:-$HOME/.z}" ] && {

    echo "ERROR: z.sh's datafile (${_Z_DATA:-$HOME/.z}) is a directory."

}



_z() {



    local datafile="${_Z_DATA:-$HOME/.z}"



    # bail if we don't own ~/.z and $_Z_OWNER not set

    [ -z "$_Z_OWNER" -a -f "$datafile" -a ! -O "$datafile" ] && return



    # add entries

    if [ "$1" = "--add" ]; then

        shift



        # $HOME isn't worth matching

        [ "$*" = "$HOME" ] && return



        # don't track excluded directory trees

        local exclude

        for exclude in "${_Z_EXCLUDE_DIRS[@]}"; do

            case "$*" in "$exclude*") return;; esac

        done



        # maintain the data file

        local tempfile="$datafile.$RANDOM"

        while read line; do

            # only count directories

            [ -d "${line%%\|*}" ] && echo $line

        done < "$datafile" | awk -v path="$*" -v now="$(date +%s)" -F"|" '

            BEGIN {

                rank[path] = 1

                time[path] = now

            }

            $2 >= 1 {

                # drop ranks below 1

                if( $1 == path ) {

                    rank[$1] = $2 + 1

                    time[$1] = now

                } else {

                    rank[$1] = $2

                    time[$1] = $3

                }

                count += $2

            }

            END {

                if( count > 9000 ) {

                    # aging

                    for( x in rank ) print x "|" 0.99*rank[x] "|" time[x]

                } else for( x in rank ) print x "|" rank[x] "|" time[x]

            }

        ' 2>/dev/null >| "$tempfile"

        # do our best to avoid clobbering the datafile in a race condition

        if [ $? -ne 0 -a -f "$datafile" ]; then

            env rm -f "$tempfile"

        else

            [ "$_Z_OWNER" ] && chown $_Z_OWNER:$(id -ng $_Z_OWNER) "$tempfile"

            env mv -f "$tempfile" "$datafile" || env rm -f "$tempfile"

        fi



    # tab completion

    elif [ "$1" = "--complete" -a -s "$datafile" ]; then

        while read line; do

            [ -d "${line%%\|*}" ] && echo $line

        done < "$datafile" | awk -v q="$2" -F"|" '

            BEGIN {

                if( q == tolower(q) ) imatch = 1

                q = substr(q, 3)

                gsub(" ", ".*", q)

            }

            {

                if( imatch ) {

                    if( tolower($1) ~ tolower(q) ) print $1

                } else if( $1 ~ q ) print $1

            }

        ' 2>/dev/null



    else

        # list/go

        while [ "$1" ]; do case "$1" in

            --) while [ "$1" ]; do shift; local fnd="$fnd${fnd:+ }$1";done;;

            -*) local opt=${1:1}; while [ "$opt" ]; do case ${opt:0:1} in

                    c) local fnd="^$PWD $fnd";;

                    h) echo "${_Z_CMD:-z} [-chlrtx] args" >&2; return;;

                    x) sed -i -e "\:^${PWD}|.*:d" "$datafile";;

                    l) local list=1;;

                    r) local typ="rank";;

                    t) local typ="recent";;

                esac; opt=${opt:1}; done;;

             *) local fnd="$fnd${fnd:+ }$1";;

        esac; local last=$1; [ "$#" -gt 0 ] && shift; done

        [ "$fnd" -a "$fnd" != "^$PWD " ] || local list=1



        # if we hit enter on a completion just go there

        case "$last" in

            # completions will always start with /

            /*) [ -z "$list" -a -d "$last" ] && cd "$last" && return;;

        esac



        # no file yet

        [ -f "$datafile" ] || return



        local cd

        cd="$(while read line; do

            [ -d "${line%%\|*}" ] && echo $line

        done < "$datafile" | awk -v t="$(date +%s)" -v list="$list" -v typ="$typ" -v q="$fnd" -F"|" '

            function frecent(rank, time) {

                # relate frequency and time

                dx = t - time

                if( dx < 3600 ) return rank * 4

                if( dx < 86400 ) return rank * 2

                if( dx < 604800 ) return rank / 2

                return rank / 4

            }

            function output(files, out, common) {

                # list or return the desired directory

                if( list ) {

                    cmd = "sort -n >&2"

                    for( x in files ) {

                        if( files[x] ) printf "%-10s %s\n", files[x], x | cmd

                    }

                    if( common ) {

                        printf "%-10s %s\n", "common:", common > "/dev/stderr"

                    }

                } else {

                    if( common ) out = common

                    print out

                }

            }

            function common(matches) {

                # find the common root of a list of matches, if it exists

                for( x in matches ) {

                    if( matches[x] && (!short || length(x) < length(short)) ) {

                        short = x

                    }

                }

                if( short == "/" ) return

                # use a copy to escape special characters, as we want to return

                # the original. yeah, this escaping is awful.

                clean_short = short

                gsub(/\[\(\)\[\]\|\]/, "\\\\&", clean_short)

                for( x in matches ) if( matches[x] && x !~ clean_short ) return

                return short

            }

            BEGIN {

                gsub(" ", ".*", q)

                hi_rank = ihi_rank = -9999999999

            }

            {

                if( typ == "rank" ) {

                    rank = $2

                } else if( typ == "recent" ) {

                    rank = $3 - t

                } else rank = frecent($2, $3)

                if( $1 ~ q ) {

                    matches[$1] = rank

                } else if( tolower($1) ~ tolower(q) ) imatches[$1] = rank

                if( matches[$1] && matches[$1] > hi_rank ) {

                    best_match = $1

                    hi_rank = matches[$1]

                } else if( imatches[$1] && imatches[$1] > ihi_rank ) {

                    ibest_match = $1

                    ihi_rank = imatches[$1]

                }

            }

            END {

                # prefer case sensitive

                if( best_match ) {

                    output(matches, best_match, common(matches))

                } else if( ibest_match ) {

                    output(imatches, ibest_match, common(imatches))

                }

            }

        ')"

        [ $? -gt 0 ] && return

        [ "$cd" ] && cd "$cd"

    fi

}



alias ${_Z_CMD:-z}='_z 2>&1'



[ "$_Z_NO_RESOLVE_SYMLINKS" ] || _Z_RESOLVE_SYMLINKS="-P"



if type compctl >/dev/null 2>&1; then

    # zsh

    [ "$_Z_NO_PROMPT_COMMAND" ] || {

        # populate directory list, avoid clobbering any other precmds.

        if [ "$_Z_NO_RESOLVE_SYMLINKS" ]; then

            _z_precmd() {

                _z --add "${PWD:a}"

            }

        else

            _z_precmd() {

                _z --add "${PWD:A}"

            }

        fi

        [[ -n "${precmd_functions[(r)_z_precmd]}" ]] || {

            precmd_functions[$(($#precmd_functions+1))]=_z_precmd

        }

    }

    _z_zsh_tab_completion() {

        # tab completion

        local compl

        read -l compl

        reply=(${(f)"$(_z --complete "$compl")"})

    }

    compctl -U -K _z_zsh_tab_completion _z

elif type complete >/dev/null 2>&1; then

    # bash

    # tab completion

    complete -o filenames -C '_z --complete "$COMP_LINE"' ${_Z_CMD:-z}

    [ "$_Z_NO_PROMPT_COMMAND" ] || {

        # populate directory list. avoid clobbering other PROMPT_COMMANDs.

        grep "_z --add" <<< "$PROMPT_COMMAND" >/dev/null || {

            PROMPT_COMMAND="$PROMPT_COMMAND"$'\n''_z --add "$(command pwd '$_Z_RESOLVE_SYMLINKS' 2>/dev/null)" 2>/dev/null;'

        }

    }

fi

# -*- coding=utf-8 -*-

__author__ = 'rocky @ www.30daydo.com'

import urllib2, re, os, codecs,sys,datetime

from bs4 import BeautifulSoup

# example https://zhhrb.sinaapp.com/index.php?date=20160610

from mail_template import MailAtt

reload(sys)

sys.setdefaultencoding('utf-8')



def save2file(filename, content):

    filename = filename + ".txt"

    f = codecs.open(filename, 'a', encoding='utf-8')

    f.write(content)

    f.close()





def getPost(date_time, filter_p):

    url = 'https://zhhrb.sinaapp.com/index.php?date=' + date_time

    user_agent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)"

    header = {"User-Agent": user_agent}

    req = urllib2.Request(url, headers=header)

    resp = urllib2.urlopen(req)

    content = resp.read()

    p = re.compile('<h2 class="question-title">(.*)</h2></br></a>')

    result = re.findall(p, content)

    count = -1

    row = -1

    for i in result:

        #print i

        return_content = re.findall(filter_p, i)



        if return_content:

            row = count

            break

            #print return_content[0]

        count = count + 1

    #print row

    if row == -1:

        return 0

    link_p = re.compile('<a href="(.*)" target="_blank" rel="nofollow">')

    link_result = re.findall(link_p, content)[row + 1]

    print link_result

    result_req = urllib2.Request(link_result, headers=header)

    result_resp = urllib2.urlopen(result_req)

    #result_content= result_resp.read()

    #print result_content



    bs = BeautifulSoup(result_resp, "html.parser")

    title = bs.title.string.strip()

    #print title

    filename = re.sub('[\/:*?"<>|]', '-', title)

    print filename

    print date_time

    save2file(filename, title)

    save2file(filename, "\n\n\n\n--------------------%s Detail----------------------\n\n" %date_time)



    detail_content = bs.find_all('div', class_='content')



    for i in detail_content:

        #print i

        save2file(filename,"\n\n-------------------------answer  -------------------------\n\n")

        for j in i.strings:



            save2file(filename, j)



    smtp_server = 'smtp.126.com'

    from_mail = sys.argv[1]

    password = sys.argv[2]

    to_mail = 'xxxxx@kindle.cn'

    send_kindle = MailAtt(smtp_server, from_mail, password, to_mail)

    send_kindle.send_txt(filename)





def main():

    sub_folder = os.path.join(os.getcwd(), "content")

    if not os.path.exists(sub_folder):

        os.mkdir(sub_folder)

    os.chdir(sub_folder)





    date_time = '20160611'

    filter_p = re.compile('大误.*')

    ori_day=datetime.date(datetime.date.today().year,01,01)

    t=datetime.date(datetime.date.today().year,datetime.date.today().month,datetime.date.today().day)

    delta=(t-ori_day).days

    print delta

    for i in range(delta):

        day=datetime.date(datetime.date.today().year,01,01)+datetime.timedelta(i)

        getPost(day.strftime("%Y%m%d"),filter_p)

    #getPost(date_time, filter_p)



if __name__ == "__main__":

    main()

# -*-coding=utf-8-*-

#抓取雪球的收藏文章

__author__ = 'Rocky'

import requests,cookielib,re,json,time

from toolkit import Toolkit

from lxml import etree

url='https://xueqiu.com/snowman/login'

session = requests.session()



session.cookies = cookielib.LWPCookieJar(filename="cookies")

try:

    session.cookies.load(ignore_discard=True)

except:

    print "Cookie can't load"



agent = 'Mozilla/5.0 (Windows NT 5.1; rv:33.0) Gecko/20100101 Firefox/33.0'

headers = {'Host': 'xueqiu.com',

           'Referer': 'https://xueqiu.com/',

           'Origin':'https://xueqiu.com',

           'User-Agent': agent}

account=Toolkit.getUserData('data.cfg')

print account['snowball_user']

print account['snowball_password']



data={'username':account['snowball_user'],'password':account['snowball_password']}

s=session.post(url,data=data,headers=headers)

print s.status_code

#print s.text

session.cookies.save()

fav_temp='https://xueqiu.com/favs?page=1'

collection=session.get(fav_temp,headers=headers)

fav_content= collection.text

p=re.compile('"maxPage":(\d+)')

maxPage=p.findall(fav_content)[0]

print maxPage

print type(maxPage)

maxPage=int(maxPage)

print type(maxPage)

for i in range(1,maxPage+1):

    fav='https://xueqiu.com/favs?page=%d' %i

    collection=session.get(fav,headers=headers)

    fav_content= collection.text

    #print fav_content

    p=re.compile('var favs = {(.*?)};',re.S|re.M)

    result=p.findall(fav_content)[0].strip()



    new_result='{'+result+'}'

    #print type(new_result)

    #print new_result

    data=json.loads(new_result)

    use_data= data['list']

    host='https://xueqiu.com'

    for i in use_data:

        url=host+ i['target']

        print url

        txt_content=session.get(url,headers=headers).text

        #print txt_content.text



        tree=etree.HTML(txt_content)

        title=tree.xpath('//title/text()')[0]



        filename = re.sub('[\/:*?"<>|]', '-', title)

        print filename



        content=tree.xpath('//div[@class="detail"]')

        for i in content:

            Toolkit.save2filecn(filename, i.xpath('string(.)'))

        #print content

        #Toolkit.save2file(filename,)

        time.sleep(10)

import socket,sys

import time

from thread_test import MyThread



socket.setdefaulttimeout(1)

#设置每个线程socket的timeou时间，超过1秒没有反应就认为端口不开放

thread_num=4

#线程数目 

ip_end=256

ip_start=0

scope=ip_end/thread_num



def scan(ip_head,ip_low, port):

    try:

        # Alert !!! below statement should be inside scan function. Else each it is one s

        ip=ip_head+str(ip_low)

	print ip

	s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

	s.connect((ip, port))

	#通过这一句判断 是否连通

        s.close()

	print "ip %s port %d open\n" %(ip,port)

        return True

    except:

        return False





def scan_range(ip_head,ip_range,port):

	start,end=ip_range

	for i in range(start,end):

		scan(ip_head,i,port)



if len(sys.argv)<3:

	print "input ip and port"

	exit()



ip_head=sys.argv[1]

port=int(sys.argv[2])





ip_range=

for i in range(thread_num):

	x_range=[i*scope,(i+1)*scope-1]

	ip_range.append(x_range)



threads=

for i in range(thread_num):

	t=MyThread(scan_range,(ip_head,ip_range,port))

	threads.append(t)

for i in range(thread_num):

	threads.start()

for i in range(thread_num):

	threads.join()

	#设置进程阻塞，防止主线程退出了，其他的多线程还在运行



print "*****end*****"

import thread,threading,time,datetime

from time import sleep,ctime

def loop1():

	print "start %s " %ctime()

	print "start in loop1"

	sleep(3)

	print "end %s " %ctime()



def loop2():

	print "sart %s " %ctime()

	print "start in loop2"

	sleep(6)

	print "end %s " %ctime()





class MyThread(threading.Thread):

	def __init__(self,fun,arg,name=""):

		threading.Thread.__init__(self)

		self.fun=fun

		self.arg=arg

		self.name=name

		#self.result



	def run(self):

		self.result=apply(self.fun,self.arg)

	

	def getResult(self):

		return self.result



def fib(n):

	if n<2:

		return 1

	else:

		return fib(n-1)+fib(n-2)





def sum(n):

	if n<2:

		return 1

	else:

		return n+sum(n-1)	



def fab(n):

	if n<2:

		return 1

	else:

		return n*fab(n-1)



def single_thread():		

	print fib(12)		

	print sum(12)

	print fab(12)





def multi_thread():

	print "in multithread"

	fun_list=[fib,sum,fab]

	n=len(fun_list)

	threads=

	count=12

	for i in range(n):

		t=MyThread(fun_list,(count,),fun_list.__name__)

		threads.append(t)

	for i in range(n):

		threads.start()



	for i in range(n):

		threads.join()

		result= threads.getResult()

		print result

def main():

	'''

	print "start at main"

	thread.start_new_thread(loop1,())

	thread.start_new_thread(loop2,())

	sleep(10)

	print "end at main"

	'''

	start=ctime()

	#print "Used %f" %(end-start).seconds

	print start	

	single_thread()

	end=ctime()

	print end

	multi_thread()

	#print "used %s" %(end-start).seconds 

if __name__=="__main__":

	main()

# -*-coding=utf-8-*-

__author__ = 'Rocky'

# -*-coding=utf-8-*-

from email.mime.text import MIMEText

from email.mime.multipart import MIMEMultipart

import smtplib

from email import Encoders, Utils

import urllib2

import time

import re

import sys

import os



from bs4 import BeautifulSoup



from email.Header import Header



reload(sys)

sys.setdefaultencoding('utf-8')





class GetContent():

    def __init__(self, id):



        # 给出的第一个参数 就是你要下载的问题的id

        # 比如 想要下载的问题链接是 https://www.zhihu.com/question/29372574

        # 那么 就输入 python zhihu.py 29372574



        id_link = "/question/" + id

        self.getAnswer(id_link)



    def save2file(self, filename, content):

        # 保存为电子书文件

        filename = filename + ".txt"

        f = open(filename, 'a')

        f.write(content)

        f.close()



    def getAnswer(self, answerID):

        host = "http://www.zhihu.com"

        url = host + answerID

        print url

        user_agent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)"

        # 构造header 伪装一下

        header = {"User-Agent": user_agent}

        req = urllib2.Request(url, headers=header)



        try:

            resp = urllib2.urlopen(req)

        except:

            print "Time out. Retry"

            time.sleep(30)

            # try to switch with proxy ip

            resp = urllib2.urlopen(req)

        # 这里已经获取了 网页的代码，接下来就是提取你想要的内容。 使用beautifulSoup 来处理，很方便

        try:

            bs = BeautifulSoup(resp)



        except:

            print "Beautifulsoup error"

            return None



        title = bs.title

        # 获取的标题



        filename_old = title.string.strip()

        print filename_old

        filename = re.sub('[\/:*?"<>|]', '-', filename_old)

        # 用来保存内容的文件名，因为文件名不能有一些特殊符号，所以使用正则表达式过滤掉



        self.save2file(filename, title.string)





        detail = bs.find("div", class_="zm-editable-content")



        self.save2file(filename, "\n\n\n\n--------------------Detail----------------------\n\n")

        # 获取问题的补充内容



        if detail is not None:



            for i in detail.strings:

                self.save2file(filename, unicode(i))



        answer = bs.find_all("div", class_="zm-editable-content clearfix")

        k = 0

        index = 0

        for each_answer in answer:



            self.save2file(filename, "\n\n-------------------------answer %s via  -------------------------\n\n" % k)



            for a in each_answer.strings:

                # 循环获取每一个答案的内容，然后保存到文件中

                self.save2file(filename, unicode(a))

            k += 1

            index = index + 1



        smtp_server = 'smtp.126.com'

        from_mail = 'your@126.com'

        password = 'yourpassword'

        to_mail = 'yourname@kindle.cn'



        # send_kindle=MailAtt(smtp_server,from_mail,password,to_mail)

        # send_kindle.send_txt(filename)



        # 调用发送邮件函数，把电子书发送到你的kindle用户的邮箱账号，这样你的kindle就可以收到电子书啦

        print filename





class MailAtt():

    def __init__(self, smtp_server, from_mail, password, to_mail):

        self.server = smtp_server

        self.username = from_mail.split("@")[0]

        self.from_mail = from_mail

        self.password = password

        self.to_mail = to_mail



        # 初始化邮箱设置



    def send_txt(self, filename):

        # 这里发送附件尤其要注意字符编码，当时调试了挺久的，因为收到的文件总是乱码

        self.smtp = smtplib.SMTP()

        self.smtp.connect(self.server)

        self.smtp.login(self.username, self.password)

        self.msg = MIMEMultipart()

        self.msg['to'] = self.to_mail

        self.msg['from'] = self.from_mail

        self.msg['Subject'] = "Convert"

        self.filename = filename + ".txt"

        self.msg['Date'] = Utils.formatdate(localtime=1)

        content = open(self.filename.decode('utf-8'), 'rb').read()

        # print content

        self.att = MIMEText(content, 'base64', 'utf-8')

        self.att['Content-Type'] = 'application/octet-stream'

        # self.att["Content-Disposition"] = "attachment;filename=\"%s\"" %(self.filename.encode('gb2312'))

        self.att["Content-Disposition"] = "attachment;filename=\"%s\"" % Header(self.filename, 'gb2312')

        # print self.att["Content-Disposition"]

        self.msg.attach(self.att)



        self.smtp.sendmail(self.msg['from'], self.msg['to'], self.msg.as_string())

        self.smtp.quit()





if __name__ == "__main__":



    sub_folder = os.path.join(os.getcwd(), "content")

    # 专门用于存放下载的电子书的目录



    if not os.path.exists(sub_folder):

        os.mkdir(sub_folder)



    os.chdir(sub_folder)



    id = sys.argv[1]

    # 给出的第一个参数 就是你要下载的问题的id

    # 比如 想要下载的问题链接是 https://www.zhihu.com/question/29372574

    # 那么 就输入 python zhihu.py 29372574





    # id_link="/question/"+id

    obj = GetContent(id)

    # obj.getAnswer(id_link)



    # 调用获取函数



    print "Done"

# -*-coding=utf-8-*-

__author__ = 'Rocky'

import requests

import cookielib

import re

import json

import time

import os

from getContent import GetContent

agent='Mozilla/5.0 (Windows NT 5.1; rv:33.0) Gecko/20100101 Firefox/33.0'

headers={'Host':'www.zhihu.com',

         'Referer':'https://www.zhihu.com',

         'User-Agent':agent}



#全局变量

session=requests.session()



session.cookies=cookielib.LWPCookieJar(filename="cookies")



try:

    session.cookies.load(ignore_discard=True)

except:

    print "Cookie can't load"



def isLogin():

    url='https://www.zhihu.com/settings/profile'

    login_code=session.get(url,headers=headers,allow_redirects=False).status_code

    print login_code

    if login_code == 200:

        return True

    else:

        return False



def get_xsrf():

    url='http://www.zhihu.com'

    r=session.get(url,headers=headers,allow_redirects=False)

    txt=r.text

    result=re.findall(r'<input type=\"hidden\" name=\"_xsrf\" value=\"(\w+)\"/>',txt)[0]

    return result



def getCaptcha():

    #r=1471341285051

    r=(time.time()*1000)

    url='http://www.zhihu.com/captcha.gif?r='+str(r)+'&type=login'



    image=session.get(url,headers=headers)

    f=open("photo.jpg",'wb')

    f.write(image.content)

    f.close()





def Login():

    xsrf=get_xsrf()

    print xsrf

    print len(xsrf)

    login_url='http://www.zhihu.com/login/email'

    data={

    '_xsrf':xsrf,

    'password':'*',

    'remember_me':'true',

    'email':'*'

    }

    try:

        content=session.post(login_url,data=data,headers=headers)

        login_code=content.text

        print content.status_code

        #this line important ! if no status, if will fail and execute the except part

        #print content.status



        if content.status_code != requests.codes.ok:

            print "Need to verification code !"

            getCaptcha()

            #print "Please input the code of the captcha"

            code=raw_input("Please input the code of the captcha")

            data['captcha']=code

            content=session.post(login_url,data=data,headers=headers)

            print content.status_code



            if content.status_code==requests.codes.ok:

                print "Login successful"

                session.cookies.save()

            #print login_code

        else:

            session.cookies.save()

    except:

        print "Error in login"

        return False



def focus_question():

    focus_id=

    url='https://www.zhihu.com/question/following'

    content=session.get(url,headers=headers)

    print content

    p=re.compile(r'<a class="question_link" href="/question/(\d+)" target="_blank" data-id')

    id_list=p.findall(content.text)

    pattern=re.compile(r'<input type=\"hidden\" name=\"_xsrf\" value=\"(\w+)\"/>')

    result=re.findall(pattern,content.text)[0]

    print result

    for i in id_list:

        print i

        focus_id.append(i)



    url_next='https://www.zhihu.com/node/ProfileFollowedQuestionsV2'

    page=20

    offset=20

    end_page=500

    xsrf=re.findall(r'<input type=\"hidden\" name=\"_xsrf\" value=\"(\w+)\"',content.text)[0]

    while offset < end_page:

        #para='{"offset":20}'

        #print para

        print "page: %d" %offset

        params={"offset":offset}

        params_json=json.dumps(params)



        data={

        'method':'next',

        'params':params_json,

        '_xsrf':xsrf

        }

        #注意上面那里 post的data需要一个xsrf的字段，不然会返回403 的错误，这个在抓包的过程中一直都没有看到提交到xsrf，所以自己摸索出来的

        offset=offset+page

        headers_l={

        'Host':'www.zhihu.com',

        'Referer':'https://www.zhihu.com/question/following',

        'User-Agent':agent,

        'Origin':'https://www.zhihu.com',

        'X-Requested-With':'XMLHttpRequest'

        }

        try:

            s=session.post(url_next,data=data,headers=headers_l)

            #print s.status_code

            #print s.text

            msgs=json.loads(s.text)

            msg=msgs['msg']

            for i in msg:

                id_sub=re.findall(p,i)



                for j in id_sub:

                    print j

                    id_list.append(j)



        except:

            print "Getting Error "





    return id_list



def main():



    if isLogin():

        print "Has login"

    else:

        print "Need to login"

        Login()

    list_id=focus_question()

    for i in list_id:

        print i

        obj=GetContent(i)



    #getCaptcha()

if __name__=='__main__':

    sub_folder=os.path.join(os.getcwd(),"content")

    #专门用于存放下载的电子书的目录



    if not os.path.exists(sub_folder):

        os.mkdir(sub_folder)



    os.chdir(sub_folder)



    main()