from __future__ import print_function



import re



import requests



from frontera.contrib.requests.manager import RequestsFrontierManager

# from frontera.contrib.requests.manager import RequestsFrontierManager

from frontera import Settings



from six.moves.urllib.parse import urljoin





SETTINGS = Settings()

SETTINGS.BACKEND = 'frontera.contrib.backends.memory.FIFO'

# SETTINGS.BACKEND = 'frontera.contrib.backends.memory.MemoryDistributedBackend'



SETTINGS.LOGGING_MANAGER_ENABLED = True

SETTINGS.LOGGING_BACKEND_ENABLED = True

SETTINGS.MAX_REQUESTS = 100

SETTINGS.MAX_NEXT_REQUESTS = 10



SEEDS = [

    'http://www.imdb.com',

]



LINK_RE = re.compile(r'<a.+?href="(.*?)".?>', re.I)





def extract_page_links(response):

    return [urljoin(response.url, link) for link in LINK_RE.findall(response.text)]



if __name__ == '__main__':



    frontier = RequestsFrontierManager(SETTINGS)

    frontier.add_seeds([requests.Request(url=url) for url in SEEDS])

    while True:

        next_requests = frontier.get_next_requests()

        if not next_requests:

            break

        for request in next_requests:

                try:

                    response = requests.get(request.url)

                    links = [

                        requests.Request(url=url)

                        for url in extract_page_links(response)

                    ]

                    frontier.page_crawled(response)

                    print('Crawled', response.url, '(found', len(links), 'urls)')



                    if links:

                        frontier.links_extracted(request, links)

                except requests.RequestException as e:

                    error_code = type(e).__name__

                    frontier.request_error(request, error_code)

                    print('Failed to process request', request.url, 'Error:', e)

raise NameError("Module '%s' doesn't define any object named '%s'" % (module, name))

NameError: Module 'frontera.contrib.backends.memory' doesn't define any object named 'FIFO'

import re

import requests

import scrapy

from scrapy import Request

from rabbit_spider import settings

from scrapy.log import logger

import json

from rabbit_spider.items import RabbitSpiderItem

import datetime

from scrapy.selector import Selector

import pika



# from scrapy_rabbitmq.spiders import RabbitMQMixin

# from scrapy.contrib.spiders import CrawlSpider



class Website(scrapy.Spider):

    name = "rabbit"



    def start_requests(self):

        headers = {'Accept': '*/*',

                   'Accept-Encoding': 'gzip, deflate, br',

                   'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',

                   'Host': '36kr.com',

                   'Referer': 'https://36kr.com/information/web_news',

                   'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36'

                   }



        url = 'https://36kr.com/information/web_news'

        



        yield Request(url=url,

                      headers=headers)



    def parse(self, response):

       



        credentials = pika.PlainCredentials('admin', 'admin')

        connection = pika.BlockingConnection(pika.ConnectionParameters('192.168.1.101', 5672, '/', credentials))



        channel = connection.channel()

        channel.exchange_declare(exchange='direct_log', exchange_type='direct')



        result = channel.queue_declare(exclusive=True, queue='')



        queue_name = result.method.queue



        # print(queue_name)

        # infos = sys.argv[1:] if len(sys.argv)>1 else ['info']

        info = 'info'



        # 绑定多个值



        channel.queue_bind(

            exchange='direct_log',

            routing_key=info,

            queue=queue_name

        )

        print('start to receive [{}]'.format(info))



        channel.basic_consume(

            on_message_callback=self.callback_func,

            queue=queue_name,

            auto_ack=True,

        )



        channel.start_consuming()





    def callback_func(self, ch, method, properties, body):

        print(body)

from scrapy import cmdline

cmdline.execute('scrapy crawl rabbit'.split())

import pika

import settings



credentials = pika.PlainCredentials('admin','admin')

connection = pika.BlockingConnection(pika.ConnectionParameters('192.168.1.101',5672,'/',credentials))



channel = connection.channel()

channel.exchange_declare(exchange='direct_log',exchange_type='direct') # fanout 就是组播



routing_key = 'info'

message='https://36kr.com/pp/api/aggregation-entity?type=web_latest_article&b_id=59499&per_page=30'

channel.basic_publish(

	exchange='direct_log',

	routing_key=routing_key,

	body=message

	)



print('sending message {}'.format(message))

connection.close()

Twisted-16.7.0 is coming soon, and it deprecates twisted.web.client.getPage (and client.HTTPClientFactory). We use these in some of the unit tests, to fetch one of the HTTP WAPI/WUI pages and make sure the contents look right.

We need to change these tests to use twisted.web.client.Agent instead, or a package named "treq", which is a Twisted flavor of the excellent (but blocking) requests library.

# -*- coding: utf-8 -*-

# website: http://30daydo.com

# @Time : 2019/6/30 12:03

# @File : main.py



import requests

import re

url = 'https://www.ximalaya.com/revision/play/album?albumId=23057324&pageNum=1&sort=1&pageSize=60'

headers={'User-Agent':'Xiaomi'}



r = requests.get(url=url,headers=headers)

js_data = r.json()

data_list = js_data.get('data',{}).get('tracksAudioPlay',)

for item in data_list:

    trackName=item.get('trackName')

    trackName=re.sub(':','',trackName)

    src_url = item.get('src')

    try:

        r0=requests.get(src_url,headers=headers)

    except Exception as e:

        print(e)

        print(trackName)

    else:

        with open('{}.m4a'.format(trackName),'wb') as f:

            f.write(r0.content)

        print('{} downloaded'.format(trackName))

    file_path=r'9927_15562445086485238.png'

    file=open(file_path, 'rb').read()

    r=requests.post(url=code_url,data=file)

    print(r.text)

js=re.sub('\r\n','',js)

import aiohttp

import asyncio



page = 30



post_data = {

    'page': 1,

    'pageSize': 10,

    'keyWord': '',

    'dpIds': '',

}



headers = {

    "Accept": "application/json, text/javascript, */*; q=0.01",

    "Accept-Encoding": "gzip, deflate",

    "Accept-Language": "en-US,en;q=0.9",

    "Connection": "keep-alive",

    "Content-Length": "34",

    "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",

    "Cookie": "secure; JSESSIONID=8NGWetn7NWF7Hb-SSkrbbzGDbYQzmNM_gjKj8wql4PXn2uc7ruv0!-96282387; __jsluid=72f938f1aa890b0ab98d726eb9d7d36f; Hm_lvt_606ad402d71f074871f1daa788ba943d=1557302782; Hm_lpvt_606ad402d71f074871f1daa788ba943d=1557302788",

    "Host": "credit.chaozhou.gov.cn",

    "Origin": "http://credit.chaozhou.gov.cn",

    "Referer": "http://credit.chaozhou.gov.cn/ ... ot%3B,

    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36",

    "X-Requested-With": "XMLHttpRequest",

}



result=[]





async def fetch(session,url, data):

    async with session.post(url=url, data=data, headers=headers) as response:

        return await response.json()



async def parse(html):

    xzcf_list = html.get('newtxzcfList')

    if xzcf_list is None:

        return

    for i in xzcf_list:

        result.append(i)



async def downlod(page):

    data=post_data.copy()

    data['page']=page

    url = 'http://credit.chaozhou.gov.cn/tfieldTypeActionJson!initXzcfListnew.do'

    async with aiohttp.ClientSession() as session:

            html=await fetch(session,url,data)

            await parse(html)



loop = asyncio.get_event_loop()

tasks=[asyncio.ensure_future(downlod(i)) for i in range(1,page)]

tasks=asyncio.gather(*tasks)

# print(tasks)

loop.run_until_complete(tasks)

# loop.close()

# print(result)

count=0

for i in result:

    print(i.get('cfXdrMc'))

    count+=1

print(f'total {count}')