描述: 集思录-实时数据-REITs-A股 REITs

限量: 单次返回所有 REITs 的基本信息数据
 

    browser = await pyppeteer.launch(

        {'headless': False,

         'userDataDir': UserDataDir,

         'defaultViewport': {'width': 1800, 'height': 1000},

         # 'enable-automation':False,

         # 'ignoreDefaultArgs':['--enable-automation'],

         'ignoreDefaultArgs':True,

         }

ret = self.doc.update_one({'announcementId':announcementId},{'$setOnInsert':item},upsert=True)

ret.matched_count, ret.modified_count

    browser = await pyppeteer.launch(

        {'headless': False,

         'userDataDir': UserDataDir,

         'defaultViewport': {'width': 1800, 'height': 1000},

         # 'enable-automation':False,

         'ignoreDefaultArgs':['--enable-automation'],

         }

    )

    browser = await pyppeteer.launch(

        {'headless': False,

         'userDataDir': UserDataDir,

         'defaultViewport': {'width': 1800, 'height': 1000}

         }

    )

        self.handleSIGINT = options.get('handleSIGINT', True)

        self.handleSIGTERM = options.get('handleSIGTERM', True)

        self.handleSIGHUP = options.get('handleSIGHUP', True)

        self.ignoreHTTPSErrors = options.get('ignoreHTTPSErrors', False)

        self.defaultViewport = options.get('defaultViewport', {'width': 800, 'height': 600})  # noqa: E501

        self.slowMo = options.get('slowMo', 0)

        self.timeout = options.get('timeout', 30000)

        self.autoClose = options.get('autoClose', True)

urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='storage.googleapis.com', port=443): Max retries exceeded with url: /chromium-browser-snapshots/Win_

x64/588429/chrome-win32.zip (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x00000000037D3880>: Failed to establish a new connect

ion: [Errno 11004] getaddrinfo failed'))

res = requests.get(url, stream=True, headers=headers, verify=False)

        with open(dst, "ab") as f:

            dp = Down_progress(file_size, first_size, dst)

            dp.start()

            chunk_size = 1024

            for chunk in res.iter_content(chunk_size = chunk_size):

                if chunk:

                    f.write(chunk)

                    dp.update(chunk_size)

http:\u002F\u002F1256122120.vod2.myqcloud.com\u002F53ad1740vodtranscq1256122120\u002F17ebe5145285890810945319765\u002Fv.f20.mp4

#!/usr/bin/env python

# http://30daydo.com

import os

import re

import js2py

import requests

from lxml import etree

from clint.textui import progress

import fire

from loguru import logger



file='crawler'

logger.add(

    "logs/%s.log" % file,

    format="{time:MM-DD HH:mm:ss} {level} {message}",

)



headers = {

    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36",

}

proxies = {}



# 如果代理不稳定，不推荐使用

# local proxy service

# proxies example:

# proxies = {

#     "http": "socks5://127.0.0.1:1080",

#     "https": "socks5://127.0.0.1:1080",

# }



def list_page(url):

    logger.info("crawling : %s" % url)

    resp = requests.get(url, headers=headers, proxies=proxies, verify=False)

    html = etree.HTML(resp.text)



    buff = '//*[@class="phimage"]/a/'

    names = html.xpath(f"{buff}@href")

    urls = html.xpath(f"{buff}img/@data-mediabook")

    for i in range(len(urls)):

        try:

            url = urls

[i]            name = re.findall("=ph(\w+)", names[i])[-1]

            logger.info(f"{url} {name}")

            download(url, name, "webm")

        except Exception as err:

            logger.error(err)





def detail_page(url):

    s = requests.Session()

    resp = s.get(url, headers=headers, proxies=proxies, verify=False)

    html = etree.HTML(resp.content)



    title = "".join(html.xpath("//h1//text()")).strip()

    logger.info(title)



    js_temp = html.xpath("//script/text()")

    for j in js_temp:

        if "flashvars" in j:

            videoUrl = exeJs(j)



            download(videoUrl, title, "mp4")

            continue





def exeJs(js):

    flashvars = re.findall("flashvars_\d+", js)[0]

    js = "\n\t".join(js.split("\n\t")[:-5]).strip()



    js = js.replace("// var nextVideoObject = flashvars_['nextVideo'];",'')

    js+=flashvars

    res = js2py.eval_js(js)



    if res.quality_720p:

        return res.quality_720p

    elif res.quality_480p:

        return res.quality_480p

    elif res.quality_240p:

        return res.quality_240p

    else:

        logger.error("parse url error")





def download(url, name, filetype):

    logger.info(f"{url} {name} {filetype}")

    filepath = "%s/%s.%s" % (filetype, name, filetype)

    if os.path.exists(filepath):

        logger.info("this file had been downloaded :: %s" % filepath)

        return

    else:

        response = requests.get(url, headers=headers, proxies=proxies, stream=True)

        with open(filepath, "wb") as file:

            total_length = int(response.headers.get("content-length"))

            for ch in progress.bar(

                response.iter_content(chunk_size=2391975),

                expected_size=(total_length / 1024) + 1,

            ):

                if ch:

                    file.write(ch)



        logger.info("download success :: %s" % filepath)





def run(_arg=None):

    paths = ["webm", "mp4"]

    for path in paths:

        if not os.path.exists(path):

            os.mkdir(path)

    if _arg == "webm":

        # https://www.pornhub.com/categories

        urls = [

            # "https://www.pornhub.com/video?o=tr",

            # "https://www.pornhub.com/video?o=ht",

            # "https://www.pornhub.com/video?o=mv",

            "https://www.pornhub.com/video",

        ]

        for url in urls:

            list_page(url)

    elif _arg == "mp4":

        with open("download.txt", "r") as file:

            keys = list(set(file.readlines()))

        logger.info(keys)

        keys += [d.strip(".webm") for d in os.listdir("webm/")]

        for key in keys:

            if not key.strip():

                continue

            url = "https://www.pornhub.com/view_v ... ot%3B % key.strip()

            logger.info("url: {}", url)

            detail_page(url)

    else:

        _str = """

tips:

    python crawler.py webm

        - 下载热门页面的缩略图，路径为webm文件夹下



    python crawler.py mp4

        - 该命令会下载webm文件下对应的mp4文件

        - 也可以将目标地址写入download.txt中

        """

        logger.info(_str)

        return

    logger.info("finish !")





if __name__ == "__main__":

    fire.Fire(run)

[/i][/i]

[i]

var A397151C04723421F = '397151C04723421F';

function doLogin(){

    var data = $('#login_form').serializeObjectToJson();

    data['_post_type'] = 'ajax';

    data['aes'] = 1;

    data['user_name'] = jslencode(data['user_name'], A397151C04723421F);

    data['password'] = jslencode(data['password'], A397151C04723421F);

    $.post('/account/ajax/login_process/', data, function(rst){

        on_login_error_processer(rst);

    }, 'json');

# -*- coding: utf-8 -*-

# @Time : 2020/11/27 12:15

# @File : js_executor.py

# @Author : Rocky C@www.30daydo.com

import execjs



def main():

    encode_user = ctx.call('jslencode', user, key)

    encode_password = ctx.call('jslencode', password, key)

    print(encode_user)

    print(encode_password)





if __name__ == '__main__':

    main()

(function(root, factory) {

    if (typeof exports === "object") {

        module.exports = exports = factory()

    } else {

        if (typeof define === "function" && define.amd) {

            define(, factory)

        } else {

            root.CryptoJS = factory()

        }

    }

}(this, function() {

    var CryptoJS = CryptoJS || (function(Math, undefined) {

        var create = Object.create || (function() {

var CryptoJS = CryptoJS || (function(Math, undefined) {

        var create = Object.create || (function() {

            function F() {}

            return function(obj) {

                var subtype;

                F.prototype = obj;

                subtype = new F();

                F.prototype = null;

                return subtype

            }

        }());

        var C = {};

        var C_lib = C.lib = {};

        var Base = C_lib.Base = (function() {

https://www.jisilu.cn/static/js/crypto-js-3.3.0-min.js

import sys

sys.path.append('..')

import asyncio

import datetime

import aiohttp

import re

import time

from parsel import Selector

from configure.settings import DBSelector

from common.BaseService import BaseService



SLEEP = 2



headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0',

           'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2'}



URL_MAP = {'home_page': 'https://holdle.com/stocks/industry', 'base': 'https://holdle.com'}





class AsyncMongo():

    def __init__(self):

        self.DB = DBSelector()

        self.client = self.DB.mongo(location_type='qq', async_type=True)

        self.db = self.client['db_stock']



    async def update(self, table,data):

        self.doc= self.db[table]

        await self.doc.insert_many(data)





class Holdle(BaseService):



    def __init__(self):

        super(Holdle, self).__init__()

        self.data_processor = AsyncMongo()

        self.tables_list =['ROE','Cash_Ratio','Gross_Margin','Operation_Margin','Net_Profit_Ratio','Dividend_ratio']



    async def home_page(self):

        start = time.time()

        async with aiohttp.ClientSession() as session:

            async with session.get(url=URL_MAP['home_page'], headers=headers) as response:

                html = await response.text()  # 这个阻塞

                resp = Selector(text=html)

                industries = resp.xpath('//ul[@class="list-unstyled"]/a')

                task_list = []

                for industry in industries:

                    json_data = {}

                    industry_url = industry.xpath('.//@href').extract_first()

                    industry_name = industry.xpath('.//li/text()').extract_first()

                    industry_name = industry_name.replace('-', '').strip()

                    json_data['industry_url'] = industry_url

                    json_data['industry_name'] = industry_name



                    task = asyncio.ensure_future(self.detail_list(session, industry_url, json_data))

                    task_list.append(task)



                await asyncio.gather(*task_list)

                end = time.time()



                print(f'time used {end - start}')



    async def detail_list(self, session, url, json_data):



        async with session.get(URL_MAP['base'] + url, headers=headers) as response:

            response = await response.text()

            await self.parse_detail(response, json_data)



    async def parse_detail(self, html, json_data=None):

            resp = Selector(text=html)

            industry=json_data['industry_name']

            tables = resp.xpath('//table[@class="table table-bordered"]')

            if len(tables)!=6:

                raise ValueError



            for index,table in enumerate(self.tables_list):

                rows = tables[index].xpath('.//tr')

                result = []

                for row in rows[1:]:

                    stock_name = row.xpath('.//td[1]/text()').extract_first()

                    value = row.xpath('.//td[2]/text()').extract_first()

                    value = float(value)

                    d={'industry':industry,'name':stock_name,'value':value,'crawltime':datetime.datetime.now()}

                    result.append(d)

                await self.data_processor.update(table,result)





app = Holdle()

loop = asyncio.get_event_loop()

loop.run_until_complete(app.home_page())

# -*- coding: utf-8 -*-

# @Time : 2020/11/24 21:42

# @File : sync_spider.py

# @Author : Rocky C@www.30daydo.com

import requests

import sys

sys.path.append('..')

import asyncio

import datetime

import aiohttp

import re

import time

from parsel import Selector

from configure.settings import DBSelector

from common.BaseService import BaseService



SLEEP = 2



headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0',

           'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2'}



URL_MAP = {'home_page': 'https://holdle.com/stocks/industry', 'base': 'https://holdle.com'}





class Holdle(BaseService):



    def __init__(self):

        super(Holdle, self).__init__()



        self.DB = DBSelector()

        self.client = self.DB.mongo(location_type='qq', async_type=True)

        self.session = requests.Session()



    def run(self):

        start = time.time()



        response = self.session.get(url=URL_MAP['home_page'], headers=headers)

        html =  response.text  # 这个阻塞

        resp = Selector(text=html)

        industries = resp.xpath('//ul[@class="list-unstyled"]/a')

        for industry in industries:

            json_data = {}

            industry_url = industry.xpath('.//@href').extract_first()

            industry_name = industry.xpath('.//li/text()').extract_first()

            json_data['industry_url'] = industry_url

            json_data['industry_name'] = industry_name

            self.detail_list(industry_url, json_data)



        end = time.time()

        print(f'time used {end-start}')



    def detail_list(self, url, json_data):



        response = self.session.get(URL_MAP['base']+url, headers=headers)

        response =response.text

        self.parse_detail(response, json_data)



    def parse_detail(self, html, json_data=None):

        resp = Selector(text=html)

        title =resp.xpath('//title/text()').extract_first()

        print(title)





app = Holdle()

app.run()

# -*- coding: utf-8 -*-

# website: http://30daydo.com

# @Time : 2020/9/22 10:07

# @File : bilibili_hot_anchor.py



# 异步爬取首页与列表



import asyncio

import datetime

import aiohttp

import re

import time

from motor.motor_asyncio import AsyncIOMotorClient

from parsel import Selector

from settings import _json_data



SLEEP = 60 * 10

INFO = _json_data['mongo']['arm']

host = INFO['host']

port = INFO['port']

user = INFO['user']

password = INFO['password']

connect_uri = f'mongodb://{user}:{password}@{host}:{port}'



client = AsyncIOMotorClient(connect_uri)



db = client['db_parker']



home_url = 'https://www.bilibili.com/ranking'

headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0',

           'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2'}





def convertor(number_str):

    '''

    将小数点的万变为整数

    :param number_str:

    :return:

    '''

    number = re.search('(\d+\.+\d+)', number_str)

    if number:

        number = float(number.group(1))

        if re.search('万', number_str):

            number = int(number * 10000)

    else:

        number = 0



    return number





async def home_page():

    async with aiohttp.ClientSession() as session:

        while True:

            start = time.time()

            async with session.get(url=home_url, headers=headers) as response:

                html = await response.text()

                resp = Selector(text=html)

                items = resp.xpath('//ul[@class="rank-list"]/li')

                for item in items:

                    json_data = {}

                    number = item.xpath('.//div[@class="num"]/text()').extract_first()

                    info = item.xpath('.//div[@class="info"][1]')

                    title = info.xpath('.//a/text()').extract_first()



                    detail_url = info.xpath('.//a/@href').extract_first()

                    play_number = info.xpath('.//div[@class="detail"]/span[1]/text()').extract_first()

                    viewing_number = info.xpath('.//div[@class="detail"]/span[2]/text()').extract_first()



                    json_data['number'] = int(number)

                    json_data['title'] = title

                    json_data['play_number'] = convertor(play_number)

                    json_data['viewing_number'] = convertor(viewing_number)



                    json_data['url'] = detail_url



                    task = asyncio.create_task(detail_list(session, detail_url, json_data))

                    # await detail_url()

                end = time.time()

                print(f'time used {end-start}')

                await asyncio.sleep(SLEEP)  # 暂停10分钟

                print(f'sleep for {SLEEP}')





async def detail_list(session, url, json_data):

    async with session.get(url, headers=headers) as response:

        response = await response.text()

        await parse_detail(response, json_data)





async def parse_detail(html, json_data=None):

    resp = Selector(text=html)

    info = resp.xpath('//div[@id="v_desc"]/div[@class="info open"]/text()').extract_first()



    if not info:

        info = '这个家伙很懒'



    json_data['info'] = info.strip()

    current = datetime.datetime.now()

    json_data['crawltime'] = current

    await db['bilibili'].update_one({'url': json_data['url']}, {'$set': json_data}, True, True)





loop = asyncio.get_event_loop()

loop.run_until_complete(home_page())