file_path=r'9927_15562445086485238.png'

    file=open(file_path, 'rb').read()

    r=requests.post(url=code_url,data=file)

    print(r.text)

js=re.sub('\r\n','',js)

import aiohttp

import asyncio



page = 30



post_data = {

    'page': 1,

    'pageSize': 10,

    'keyWord': '',

    'dpIds': '',

}



headers = {

    "Accept": "application/json, text/javascript, */*; q=0.01",

    "Accept-Encoding": "gzip, deflate",

    "Accept-Language": "en-US,en;q=0.9",

    "Connection": "keep-alive",

    "Content-Length": "34",

    "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",

    "Cookie": "secure; JSESSIONID=8NGWetn7NWF7Hb-SSkrbbzGDbYQzmNM_gjKj8wql4PXn2uc7ruv0!-96282387; __jsluid=72f938f1aa890b0ab98d726eb9d7d36f; Hm_lvt_606ad402d71f074871f1daa788ba943d=1557302782; Hm_lpvt_606ad402d71f074871f1daa788ba943d=1557302788",

    "Host": "credit.chaozhou.gov.cn",

    "Origin": "http://credit.chaozhou.gov.cn",

    "Referer": "http://credit.chaozhou.gov.cn/ ... ot%3B,

    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36",

    "X-Requested-With": "XMLHttpRequest",

}



result=[]





async def fetch(session,url, data):

    async with session.post(url=url, data=data, headers=headers) as response:

        return await response.json()



async def parse(html):

    xzcf_list = html.get('newtxzcfList')

    if xzcf_list is None:

        return

    for i in xzcf_list:

        result.append(i)



async def downlod(page):

    data=post_data.copy()

    data['page']=page

    url = 'http://credit.chaozhou.gov.cn/tfieldTypeActionJson!initXzcfListnew.do'

    async with aiohttp.ClientSession() as session:

            html=await fetch(session,url,data)

            await parse(html)



loop = asyncio.get_event_loop()

tasks=[asyncio.ensure_future(downlod(i)) for i in range(1,page)]

tasks=asyncio.gather(*tasks)

# print(tasks)

loop.run_until_complete(tasks)

# loop.close()

# print(result)

count=0

for i in result:

    print(i.get('cfXdrMc'))

    count+=1

print(f'total {count}')

import requests

from lxml import etree

import asyncio

import aiohttp

import pandas

import re

import math

import time



loction_info = '''    1→杭州

    2→武汉

    3→北京

    按ENTER确认：'''

loction_select = input(loction_info)

loction_dic = {'1': 'hz',

               '2': 'wh',

               '3': 'bj'}

city_url = 'https://{}.lianjia.com/ershoufang/'.format(loction_dic[loction_select])

down = input('请输入价格下限（万）:')

up = input('请输入价格上限（万）:')



inter_list = [(int(down), int(up))]





def half_inter(inter):

    lower = inter[0]

    upper = inter[1]

    delta = int((upper - lower) / 2)

    inter_list.remove(inter)

    print('已经缩小价格区间', inter)

    inter_list.append((lower, lower + delta))

    inter_list.append((lower + delta, upper))





pagenum = {}





def get_num(inter):

    url = city_url + 'bp{}ep{}/'.format(inter[0], inter[1])

    r = requests.get(url).text

    print(r)

    num = int(etree.HTML(r).xpath("//h2[@class='total fl']/span/text()")[0].strip())

    pagenum[(inter[0], inter[1])] = num

    return num





totalnum = get_num(inter_list[0])



judge = True

while judge:

    a = [get_num(x) > 3000 for x in inter_list]

    if True in a:

        judge = True

    else:

        judge = False

    for i in inter_list:

        if get_num(i) > 3000:

            half_inter(i)

print('价格区间缩小完毕！')



url_lst = []

url_lst_failed = []

url_lst_successed = []

url_lst_duplicated = []



for i in inter_list:

    totalpage = math.ceil(pagenum[i] / 30)

    for j in range(1, totalpage + 1):

        url = city_url + 'pg{}bp{}ep{}/'.format(j, i[0], i[1])

        url_lst.append(url)

print('url列表获取完毕！')



info_lst = []





async def get_info(url):

    async with aiohttp.ClientSession() as session:

        async with session.get(url, timeout=5) as resp:

            if resp.status != 200:

                url_lst_failed.append(url)

            else:

                url_lst_successed.append(url)

            r = await resp.text()

            nodelist = etree.HTML(r).xpath("//ul[@class='sellListContent']/li")

            # print('-------------------------------------------------------------')

            # print('开始抓取第{}个页面的数据,共计{}个页面'.format(url_lst.index(url),len(url_lst)))

            # print('开始抓取第{}个页面的数据,共计{}个页面'.format(url_lst.index(url), len(url_lst)))

            # print('开始抓取第{}个页面的数据,共计{}个页面'.format(url_lst.index(url), len(url_lst)))

            # print('-------------------------------------------------------------')

            info_dic = {}

            index = 1

            print('开始抓取{}'.format(resp.url))

            print('开始抓取{}'.format(resp.url))

            print('开始抓取{}'.format(resp.url))

            for node in nodelist:

                try:

                    info_dic['title'] = node.xpath(".//div[@class='title']/a/text()")[0]

                except:

                    info_dic['title'] = '/'

                try:

                    info_dic['href'] = node.xpath(".//div[@class='title']/a/@href")[0]

                except:

                    info_dic['href'] = '/'

                try:

                    info_dic['xiaoqu'] = \

                    node.xpath(".//div[@class='houseInfo']")[0].xpath('string(.)').replace(' ', '').split('|')[0]

                except:

                    info_dic['xiaoqu'] = '/'

                try:

                    info_dic['huxing'] = \

                    node.xpath(".//div[@class='houseInfo']")[0].xpath('string(.)').replace(' ', '').split('|')[1]

                except:

                    info_dic['huxing'] = '/'

                try:

                    info_dic['area'] = \

                    node.xpath(".//div[@class='houseInfo']")[0].xpath('string(.)').replace(' ', '').split('|')[2]

                except:

                    info_dic['area'] = '/'

                try:

                    info_dic['chaoxiang'] = \

                    node.xpath(".//div[@class='houseInfo']")[0].xpath('string(.)').replace(' ', '').split('|')[3]

                except:

                    info_dic['chaoxiang'] = '/'

                try:

                    info_dic['zhuangxiu'] = \

                    node.xpath(".//div[@class='houseInfo']")[0].xpath('string(.)').replace(' ', '').split('|')[4]

                except:

                    info_dic['zhuangxiu'] = '/'

                try:

                    info_dic['dianti'] = \

                    node.xpath(".//div[@class='houseInfo']")[0].xpath('string(.)').replace(' ', '').split('|')[5]

                except:

                    info_dic['dianti'] = '/'

                try:

                    info_dic['louceng'] = re.findall('\((.*)\)', node.xpath(".//div[@class='positionInfo']/text()")[0])

                except:

                    info_dic['louceng'] = '/'

                try:

                    info_dic['nianxian'] = re.findall('\)(.*?)年', node.xpath(".//div[@class='positionInfo']/text()")[0])

                except:

                    info_dic['nianxian'] = '/'

                try:

                    info_dic['guanzhu'] = ''.join(re.findall('[0-9]', node.xpath(".//div[@class='followInfo']/text()")[

                        0].replace(' ', '').split('/')[0]))

                except:

                    info_dic['guanzhu'] = '/'

                try:

                    info_dic['daikan'] = ''.join(re.findall('[0-9]',

                                                            node.xpath(".//div[@class='followInfo']/text()")[0].replace(

                                                                ' ', '').split('/')[1]))

                except:

                    info_dic['daikan'] = '/'

                try:

                    info_dic['fabu'] = node.xpath(".//div[@class='followInfo']/text()")[0].replace(' ', '').split('/')[

                        2]

                except:

                    info_dic['fabu'] = '/'

                try:

                    info_dic['totalprice'] = node.xpath(".//div[@class='totalPrice']/span/text()")[0]

                except:

                    info_dic['totalprice'] = '/'

                try:

                    info_dic['unitprice'] = node.xpath(".//div[@class='unitPrice']/span/text()")[0].replace('单价', '')

                except:

                    info_dic['unitprice'] = '/'

                if True in [info_dic['href'] in dic.values() for dic in info_lst]:

                    url_lst_duplicated.append(info_dic)

                else:

                    info_lst.append(info_dic)

                print('第{}条:    {}→房屋信息抓取完毕！'.format(index, info_dic['title']))

                index += 1

                info_dic = {}





start = time.time()



# 首次抓取url_lst中的信息，部分url没有对其发起请求，不知道为什么

tasks = [asyncio.ensure_future(get_info(url)) for url in url_lst]

loop = asyncio.get_event_loop()

loop.run_until_complete(asyncio.wait(tasks))



# 将没有发起请求的url放入一个列表，对其进行循环抓取，直到所有url都被发起请求

url_lst_unrequested = []

for url in url_lst:

    if url not in url_lst_successed or url_lst_failed:

        url_lst_unrequested.append(url)

while len(url_lst_unrequested) > 0:

    tasks_unrequested = [asyncio.ensure_future(get_info(url)) for url in url_lst_unrequested]

    loop.run_until_complete(asyncio.wait(tasks_unrequested))

    url_lst_unrequested = []

    for url in url_lst:

        if url not in url_lst_successed:

            url_lst_unrequested.append(url)

end = time.time()

print('当前价格区间段内共有{}套二手房源\(包含{}条重复房源\),实际获得{}条房源信息。'.format(totalnum, len(url_lst_duplicated), len(info_lst)))

print('总共耗时{}秒'.format(end - start))



df = pandas.DataFrame(info_lst)

df.to_csv("ljwh.csv", encoding='gbk')

./kafka-server-start.sh -daemon ../config/server.properties

systemctl stop firewalld.service #关闭防火墙

systemctl disable firewalld.service #禁止启动防火墙

db.collection.findOneAndUpdate(

   <filter>,

   <update>,

   {

     projection: <document>,

     sort: <document>,

     maxTimeMS: <number>,

     upsert: <boolean>,

     returnNewDocument: <boolean>,

     collation: <document>,

     arrayFilters: [ <filterdocument1>, ... ]

   }

)

>>> db.example.find_one_and_update(

...     {'_id': 'userid'},

...     {'$inc': {'seq': 1}},

...     projection={'seq': True, '_id': False},

...     return_document=ReturnDocument.AFTER)

# Redis集群地址

REDIS_MASTER_NODES = [

    {"host": "192.168.10.233", "port": "30001"},

    {"host": "192.168.10.234", "port": "30002"},

    {"host": "192.168.10.235", "port": "30003"},

]



# 使用的哈希函数数，默认为6  

BLOOMFILTER_HASH_NUMBER = 6



# Bloomfilter使用的Redis内存位，30表示2 ^ 30 = 128MB，默认为22 (1MB 可去重130W URL)  

BLOOMFILTER_BIT = 22



# 不清空redis队列  

SCHEDULER_PERSIST = True  

# 调度队列  

SCHEDULER = "scrapy_redis_cluster.scheduler.Scheduler"  

# 去重 

DUPEFILTER_CLASS = "scrapy_redis_cluster.dupefilter.RFPDupeFilter"  

# queue  

SCHEDULER_QUEUE_CLASS = 'scrapy_redis_cluster.queue.PriorityQueue'

    def __init__(self,page=None,*args, **kwargs):

        super(Gaode,self).__init__(*args, **kwargs)

        self.page=page

        

        

    def start_requests(self):

        XXXXXX 调用self.page 即可

        yield Request(XXXX)

# _*_ coding: utf-8 _*_



from selenium import webdriver

import time



__author__ = 'Silent_Coder'

__date__ = '2019/3/12 22:41'



HOME_PAGE = 'https://www.xuexi.cn/'

VIDEO_LINK = 'https://www.xuexi.cn/a191dbc3067d516c3e2e17e2e08953d6/b87d700beee2c44826a9202c75d18c85.html?pageNumber=39'

LONG_VIDEO_LINK = 'https://www.xuexi.cn/f65dae4a57fe21fcc36f3506d660891c/b2e5aa79be613aed1f01d261c4a2ae17.html'

LONG_VIDEO_LINK2 = 'https://www.xuexi.cn/0040db2a403b0b9303a68b9ae5a4cca0/b2e5aa79be613aed1f01d261c4a2ae17.html'

TEST_VIDEO_LINK = 'https://www.xuexi.cn/8e35a343fca20ee32c79d67e35dfca90/7f9f27c65e84e71e1b7189b7132b4710.html'

SCORES_LINK = 'https://pc.xuexi.cn/points/my-points.html'

LOGIN_LINK = 'https://pc.xuexi.cn/points/login.html'

ARTICLES_LINK = 'https://www.xuexi.cn/d05cad69216e688d304bb91ef3aac4c6/9a3668c13f6e303932b5e0e100fc248b.html'



options = webdriver.ChromeOptions()

options.add_experimental_option('excludeSwitches', ['enable-automation'])

browser = webdriver.Chrome(executable_path=r'D:\OneDrive\Python\selenium\chromedriver.exe',options=options)





def login_simulation():

    """模拟登录"""

    # 方式一：使用cookies方式

    # 先自己登录，然后复制token值覆盖

    # cookies = {'name': 'token', 'value': ''}

    # browser.add_cookie(cookies)



    # 方式二：自己扫码登录

    browser.get(LOGIN_LINK)

    browser.maximize_window()

    browser.execute_script("var q=document.documentElement.scrollTop=1000")

    time.sleep(10)

    browser.get(HOME_PAGE)

    print("模拟登录完毕\n")





def watch_videos():

    """观看视频"""

    browser.get(VIDEO_LINK)

    videos = browser.find_elements_by_xpath("//div[@id='Ck3ln2wlyg3k00']")

    spend_time = 0



    for i, video in enumerate(videos):

        if i > 6:

            break

        video.click()

        all_handles = browser.window_handles

        browser.switch_to_window(all_handles[-1])

        browser.get(browser.current_url)



        # 点击播放

        browser.find_element_by_xpath("//div[@class='outter']").click()

        # 获取视频时长

        video_duration_str = browser.find_element_by_xpath("//span[@class='duration']").get_attribute('innerText')

        video_duration = int(video_duration_str.split(':')[0]) * 60 + int(video_duration_str.split(':')[1])

        # 保持学习，直到视频结束

        time.sleep(video_duration + 3)

        spend_time += video_duration + 3

        browser.close()

        browser.switch_to_window(all_handles[0])



    # if spend_time < 3010:

    #     browser.get(LONG_VIDEO_LINK)

    #     browser.execute_script("var q=document.documentElement.scrollTop=850")

    #     try:

    #         browser.find_element_by_xpath("//div[@class='outter']").click()

    #     except:

    #         pass

    #

    #     # 观看剩下的时间

    #     time.sleep(3010 - spend_time)

    browser.get(TEST_VIDEO_LINK)

    time.sleep(3010 - spend_time)

    print("播放视频完毕\n")





def read_articles():

    """阅读文章"""

    browser.get(ARTICLES_LINK)

    articles = browser.find_elements_by_xpath("//div[@id='Ca4gvo4bwg7400']")

    for index, article in enumerate(articles):

        if index > 7:

            break

        article.click()

        all_handles = browser.window_handles

        browser.switch_to_window(all_handles[-1])

        browser.get(browser.current_url)

        for i in range(0, 2000, 100):



            js_code = "var q=document.documentElement.scrollTop=" + str(i)

            browser.execute_script(js_code)

            time.sleep(5)

        for i in range(2000, 0, -100):

            js_code = "var q=document.documentElement.scrollTop=" + str(i)

            browser.execute_script(js_code)

            time.sleep(5)

        time.sleep(80)

        browser.close()

        browser.switch_to_window(all_handles[0])

    print("阅读文章完毕\n")





def get_scores():

    """获取当前积分"""

    browser.get(SCORES_LINK)

    time.sleep(2)

    gross_score = browser.find_element_by_xpath("//*[@id='app']/div/div[2]/div/div[2]/div[2]/span[1]")\

        .get_attribute('innerText')

    today_score = browser.find_element_by_xpath("//span[@class='my-points-points']").get_attribute('innerText')

    print("当前总积分：" + str(gross_score))

    print("今日积分：" + str(today_score))

    print("获取积分完毕，即将退出\n")





if __name__ == '__main__':

    login_simulation()  # 模拟登录

    read_articles()     # 阅读文章

    watch_videos()      # 观看视频

    get_scores()        # 获得今日积分

    browser.quit()

# -*- coding: utf-8 -*-

# website: http://30daydo.com

# @Time : 2019/2/23 19:34

# @File : youdao.py

# 解密有道词典的JS





import hashlib

import random

import requests

import time





def md5_(word):

    s = bytes(word, encoding='utf8')

    m = hashlib.md5()

    m.update(s)

    ret = m.hexdigest()

    return ret



def get_sign(word, salt):

    ret = md5_('fanyideskweb' + word + salt + 'p09@Bn{h02_BIEe]$P^nG')

    return ret





def youdao(word):

    url = 'http://fanyi.youdao.com/translate_o?smartresult=dict&smartresult=rule'

    headers = {

        'Host': 'fanyi.youdao.com',

        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0',

        'Accept': 'application/json, text/javascript, */*; q=0.01',

        'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',

        'Accept-Encoding': 'gzip, deflate',

        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',

        'X-Requested-With': 'XMLHttpRequest',

        'Referer': 'http://fanyi.youdao.com/',

        'Content-Length': '252',

        'Cookie': 'YOUDAO_MOBILE_ACCESS_TYPE=1; OUTFOX_SEARCH_USER_ID=1672542763@10.169.0.83; JSESSIONID=aaaWzxpjeDu1gbhopLzKw; ___rl__test__cookies=1550913722828; OUTFOX_SEARCH_USER_ID_NCOO=372126049.6326876',

        'Connection': 'keep-alive',

        'Pragma': 'no-cache',

        'Cache-Control': 'no-cache',

    }



    ts = str(int(time.time()*1000))

    salt=ts+str(random.randint(0,10))

    bv = md5_("5.0 (Windows)")

    sign= get_sign(word,salt)



    post_data = {

        'i': word,

        'from': 'AUTO', 'to': 'AUTO', 'smartresult': 'dict', 'client': 'fanyideskweb', 'salt': salt,

        'sign': sign, 'ts': ts, 'bv': bv, 'doctype': 'json', 'version': '2.1',

        'keyfrom': 'fanyi.web', 'action': 'FY_BY_REALTIME', 'typoResult': 'false'

    }



    r = requests.post(

        url=url,

        headers=headers,

        data=post_data

    )



    for item in r.json().get('smartResult',{}).get('entries'):

        print(item)



word='student'

youdao(word)

   headers = {'Accept': 'application/json,text/javascript,*/*;q=0.01', 'Accept-Encoding':

        'gzip,deflate,br',

               'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', 'Cache-Control': 'no-cache',

               # 'Connection': 'keep-alive',

               'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8',

               'Cookie': 'JSESSIONID=ABAAABAABEEAAJAACF8F22F99AFA35F9EEC28F2D0E46A41;_ga=GA1.2.331323650.1548204973;_gat=1;Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1548204973;user_trace_token=20190123085612-adf35b62-1ea9-11e9-b744-5254005c3644;LGSID=20190123085612-adf35c69-1ea9-11e9-b744-5254005c3644;PRE_UTM=;PRE_HOST=;PRE_SITE=;PRE_LAND=https%3A%2F%2Fwww.lagou.com%2F;LGUID=20190123085612-adf35ed5-1ea9-11e9-b744-5254005c3644;_gid=GA1.2.1809874038.1548204973;index_location_city=%E6%B7%B1%E5%9C%B3;TG-TRACK-CODE=index_search;SEARCH_ID=169bf76c08b548f8830967a1968d10ca;Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1548204985;LGRID=20190123085624-b52a0555-1ea9-11e9-b744-5254005c3644',

               'Host': 'www.lagou.com', 'Origin': 'https://www.lagou.com', 'Pragma': 'no-cache',

               'Referer': 'https://www.lagou.com/jobs/list_%E7%88%AC%E8%99%AB?labelWords=&fromSearch=true&suginput=',

               'User-Agent': 'Mozilla/5.0(WindowsNT6.3;WOW64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/71.0.3578.98Safari/537.36',

               'X-Anit-Forge-Code': '0',

               'X-Anit-Forge-Token': 'None',

               'X-Requested-With': 'XMLHttpRequest'

               }

    # data='中国人'

    data='hello'

  

    ret = CryptoJS.AES.encrypt(data,'secret key 123')

    

    content = ret.toString()

    result = CryptoJS.AES.decrypt(content,'secret key 123')

    print(result.toString(CryptoJS.enc.Utf8))

s='耐克球鞋'

s =s.encode('gb2312')

 data = {'__VIEWSTATE': view_state,

        '__EVENTVALIDATION': event_validation,

        'txtTKeyword': s,

        'btQuery.x': 41,

        'btQuery.y': 24,

        }



r = session.post(url=self.base_url, headers=headers,

                     data=data,proxies=self.get_proxy()

import grequests

import requests

import bs4



def simple_request(url):

    page = requests.get(url)

    return page



urls = [

    'http://www.heroku.com',

    'http://python-tablib.org',

    'http://httpbin.org',

    'http://python-requests.org',

    'http://kennethreitz.com'

]



rs = [grequests.get(simple_request(u)) for u in urls]





grequests.map(rs)

rs = (grequests.get(u) for u in urls)

requests = grequests.map(rs)

for response in requests:

    market_watch(response.content)

# -*-coding=utf-8-*-



# @Time : 2018/10/23 9:26

# @File : money_reward.py

import requests

from collections import OrderedDict

import time

import datetime

import pymongo

import config



session = requests.Session()

def get_proxy(retry=10):

    proxyurl = 'http://{}:8081/dynamicIp/common/getDynamicIp.do'.format(config.PROXY)

    count = 0

    for i in range(retry):

        try:

            r = requests.get(proxyurl, timeout=10)

        except Exception as e:

            print(e)

            count += 1

            print('代理获取失败,重试' + str(count))

            time.sleep(1)



        else:

            js = r.json()

            proxyServer = 'http://{0}:{1}'.format(js.get('ip'), js.get('port'))

            proxies_random = {

                'http': proxyServer

            }

            return proxies_random





def get_content(url):

    headers = {

        # 此处添加个人的header信息

    }

    try:

        proxy = get_proxy()

    except Exception as e:

        print(e)

        proxy = get_proxy()



    try:

        r = session.get(url=url, headers=headers,proxies=proxy,timeout=10)

    except Exception as e:

        print(e)

        proxy = get_proxy()

        r = session.get(url=url, headers=headers,proxies=proxy,timeout=10)



    return r





def parse_content(post_id):

    url = 'https://xueqiu.com/statuses/reward/list_by_user.json?status_id={}&page=1&size=99999999'.format(post_id)

    r = get_content(url)

    print(r.text)

    if r.status_code != 200:

        print('status code != 200')

        failed_doc.insert({'post_id':post_id,'status':0})

        return None



    try:



        js_data = r.json()

    except Exception as e:

        print(e)

        print('can not parse to json')

        print(post_id)

        failed_doc.insert({'post_id': post_id, 'status': 0})

        return



    ret = 

    been_reward_user = '元卫南'

    for item in js_data.get('items'):

        name = item.get('name')

        amount = item.get('amount')

        description = item.get('description')

        user_id = item.get('user_id')

        created_at = item.get('created_at')

        if created_at:

            created_at = datetime.datetime.fromtimestamp(int(created_at) / 1000).strftime('%Y-%m-%d %H:%M:%S')



        d = OrderedDict()

        d['name'] = name

        d['user_id'] = user_id

        d['amount'] = amount / 100

        d['description'] = description

        d['created_at'] = created_at

        d['been_reward'] = been_reward_user

        ret.append(d)



    print(ret)

    if ret:

        doc.insert_many(ret)

        failed_doc.insert({'post_id':post_id,'status':1})







def get_all_page_id(user_id):

    doc = db['db_parker']['xueqiu_zhuanglan']



    get_page_url = 'https://xueqiu.com/statuses/original/timeline.json?user_id={}&page=1'.format(user_id)

    r = get_content(get_page_url)

    max_page = int(r.json().get('maxPage'))



    for i in range(1, max_page + 1):

        url = 'https://xueqiu.com/statuses/original/timeline.json?user_id=2227798650&page={}'.format(i)

        r = get_content(url)

        js_data = r.json()

        ret = 



        for item in js_data.get('list'):

            d = OrderedDict()



            d['article_id'] = item.get('id')

            d['title'] = item.get('title')

            d['description'] = item.get('description')

            d['view_count'] = item.get('view_count')

            d['target'] = 'https://xueqiu.com/' + item.get('target')

            d['user_id']= item.get('user_id')

            d['created_at'] = datetime.datetime.fromtimestamp(int(item.get('created_at')) / 1000).strftime(

                '%Y-%m-%d %H:%M:%S')



            ret.append(d)

        print(d)

        doc.insert_many(ret)



def loop_page_id():

    doc = db['db_parker']['xueqiu_zhuanglan']

    ret = doc.find({},{'article_id':1})

    failed_doc = db['db_parker']['xueqiu_reward_status']

    failed_ret = failed_doc.find({'status':1})

    article_id_list =

    for i in failed_ret:

        article_id_list.append(i.get('article_id'))



    for item in ret:

        article_id = item.get('article_id')

        print(article_id)

        if article_id in article_id_list:

            continue

        else:

            parse_content(article_id)



loop_page_id()

from PIL import Image

im = Image.open('test_0.jpg')

pytesseract.image_to_string(im)