url = 'http://xyhz.huizhou.gov.cn/static/js/common/jigsaw/images/{}.jpg'

headers={'User-Agent':'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'}

async def getPage(num):



    async with aiohttp.ClientSession() as session:

        async with session.get(url.format(num),headers=headers) as resp:

            if resp.status==200:

                f= await aiofiles.open('{}.jpg'.format(num),mode='wb')

                await f.write(await resp.read())

                await f.close()



loop = asyncio.get_event_loop()

tasks = [getPage(i) for i in range(5)]

loop.run_until_complete(asyncio.wait(tasks))

# -*-coding=utf-8-*-



# @Time : 2019/9/9 9:25

# @File : mail_backup.py

# @Author : 

import codecs

import re

import config

import os

from exchangelib import DELEGATE, Account, Credentials, Configuration, NTLM, Message, Mailbox, HTMLBody,FileAttachment,ItemAttachment

from exchangelib.protocol import BaseProtocol, NoVerifyHTTPAdapter





#此句用来消除ssl证书错误，exchange使用自签证书需加上

BaseProtocol.HTTP_ADAPTER_CLS = NoVerifyHTTPAdapter





# 输入你的域账号如example\xxx

cred = Credentials(r'example\xxx', 你的邮箱密码)



configx = Configuration(server='mail.credlink.com', credentials=cred, auth_type=NTLM)

a = Account(

    primary_smtp_address='你的邮箱地址', config=configx, autodiscover=False, access_type=DELEGATE

)





for item in a.inbox.all().order_by('-datetime_received')[:100]:

    print(item.subject, item.sender, item.unique_body,item.datetime_received)

   

    name = item.subject

    name = re.sub('[\/:*?"<>|]', '-', name)

    local_path = os.path.join('inbox', name+'.html')

    with codecs.open(local_path, 'w','utf-8') as f:

        f.write(item.unique_body)



    for attachment in item.attachments:

        if isinstance(attachment, FileAttachment):

            name = attachment.name

            name = re.sub('[\/:*?"<>|]','-',name)

            local_path = os.path.join('inbox', attachment.name)

            with codecs.open(local_path, 'wb') as f:

                f.write(attachment.content)

            print('Saved attachment to', local_path)



        elif isinstance(attachment, ItemAttachment):

            if isinstance(attachment.item, Message):

                name=attachment.item.subject

                name = re.sub('[\/:*?"<>|]', '-', name)

                local_path = os.path.join('inbox', 'attachment')

                with codecs.open(local_path, 'w') as f:

                    f.write(attachment.item.body)

import time



LOOP = 2*10**8



def add(x,y):

    return x+y



def cpu_pressure(loop):

    

    for i in range(loop):

        result = add(i,i+1)





if __name__ == '__main__':

    start = time.time()

    cpu_pressure(LOOP)

    print(f'time used {time.time()-start}s')

class Command(ScrapyCommand):



    requires_project = True



    def syntax(self):

        return '[options]'



    def short_desc(self):

        return 'Runs all of the spiders - My Defined'



    def run(self,args,opts):

        print('==================')

        print(type(self.crawler_process))

        spider_list = self.crawler_process.spiders.list() # 找到爬虫类



        for name in spider_list:

            print('=================')

            print(name)

            self.crawler_process.crawl(name,**opts.__dict__)



        self.crawler_process.start()

      def __init__(self, settings=None):

        if isinstance(settings, dict) or settings is None:

            settings = Settings(settings)

        self.settings = settings

        self.spider_loader = _get_spider_loader(settings) # 构造爬虫

        self._crawlers = set()

        self._active = set()

        self.bootstrap_failed = False

def _get_spider_loader(settings):



    cls_path = settings.get('SPIDER_LOADER_CLASS')

    

    # settings文件没有定义SPIDER_LOADER_CLASS，所以这里获取到的是系统的默认配置文件，

    # 默认配置文件在接下来的代码块A

    # SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'    

    

    loader_cls = load_object(cls_path) 

    # 这个函数就是根据路径转为类对象，也就是上面crapy.spiderloader.SpiderLoader 这个

    # 字符串变成一个类对象

    # 具体的load_object 对象代码见下面代码块B



    return loader_cls.from_settings(settings.frozencopy())

# 代码块A

#......省略若干

SCHEDULER = 'scrapy.core.scheduler.Scheduler'

SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'

SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'

SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.ScrapyPriorityQueue'



SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader' 就是这个值

SPIDER_LOADER_WARN_ONLY = False



SPIDER_MIDDLEWARES = {}

# 代码块B 为了方便，我把异常处理的去除

from importlib import import_module #导入第三方库



def load_object(path):

    dot = path.rindex('.') 

    module, name = path[:dot], path[dot+1:]

    # 上面把路径分为基本路径+模块名

    

    mod = import_module(module)

    obj = getattr(mod, name)

    # 获取模块里面那个值

    

    return obj

In [33]: mod = import_module(module)                                                                                                                                             



In [34]: mod                                                                                                                                                                     

Out[34]: <module 'scrapy.spiderloader' from '/home/xda/anaconda3/lib/python3.7/site-packages/scrapy/spiderloader.py'>



In [35]: getattr(mod,name)                                                                                                                                                       

Out[35]: scrapy.spiderloader.SpiderLoader



In [36]: obj = getattr(mod,name)                                                                                                                                                 



In [37]: obj                                                                                                                                                                     

Out[37]: scrapy.spiderloader.SpiderLoader



In [38]: type(obj)                                                                                                                                                               

Out[38]: type

    def from_settings(cls, settings):

        return cls(settings)

class SpiderLoader(object):

    """

    SpiderLoader is a class which locates and loads spiders

    in a Scrapy project.

    """

    def __init__(self, settings):

        self.spider_modules = settings.getlist('SPIDER_MODULES') 

        # 获得settting中的模块名字，创建scrapy的时候就默认帮你生成了

        # 你可以看看你的settings文件里面的内容就可以找到这个值，是一个list

        

        self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY')

        self._spiders = {}

        self._found = defaultdict(list)

        self._load_all_spiders() # 加载所有爬虫

def _load_all_spiders(self):

        for name in self.spider_modules:            



                for module in walk_modules(name): # 这个遍历文件夹里面的文件，然后再转化为类对象，

                    # 保存到字典：self._spiders = {}       

                    self._load_spiders(module) # 模块变成spider



        self._check_name_duplicates() # 去重，如果名字一样就异常

def iter_spider_classes(module):

    from scrapy.spiders import Spider



    for obj in six.itervalues(vars(module)): # 找到模块里面的变量，然后迭代出来

        if inspect.isclass(obj) and \

           issubclass(obj, Spider) and \

           obj.__module__ == module.__name__ and \

           getattr(obj, 'name', None): # 有name属性，继承于Spider

           yield obj

* * * * * export DISPLAY=:0; gedit

import pyautogui as pag

import datetime



def neck_rest():

    f = open('neck_record.txt', 'a')

    ret = pag.prompt("Rest! Protect your neck !")

    if ret == 'rest':

        f.write(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))

        f.write('\t')

        f.write('Rest')

        f.write('\n')

    else:

        f.write(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))

        f.write('\t')

        f.write('Failed to rest')

        f.write('\n')

    f.close()



neck_rest()

redis://[:password]@localhost:6379/0

rediss://[:password]@localhost:6379/0

unix://[:password]@/path/to/socket.sock?db=0

db.test_tab.insert({array:[]})

db.test_tab.insert({array:[]})

db.test_tab.insert({array:[]})

db.test_tab.insert({array:[1,2,3,4,5]})

db.test_tab.insert({array:[1,2,3,4,5,6]})

db.getCollection("example").find({array:{$exists:true,$ne:[]}}); # 字段不为0

    from . import (constants, error, message, context,

ImportError: DLL load failed: 找不到指定的模块。

 # helper

    class RedisHelp(object):



        def __init__(self,channel):

            # self.pool = redis.ConnectionPool('10.18.6.46',port=6379)



            # self.conn = redis.Redis(connection_pool=self.pool)

            # 上面的方式无法使用订阅者 发布者模式



            self.conn = redis.Redis(host='10.18.6.46')

            self.publish_channel = channel

            self.subscribe_channel = channel





        def publish(self,msg):

            self.conn.publish(self.publish_channel,msg) # 1. 渠道名 ,2 信息



        def subscribe(self):

            self.pub = self.conn.pubsub()

            self.pub.subscribe(self.subscribe_channel)

            self.pub.parse_response()

            print('initial')

            return self.pub





    helper = RedisHelp('cuiqingcai')



    # 订阅者

    if sys.argv[1]=='s':

        print('in subscribe mode')

        pub = helper.subscribe()

        while 1:

            print('waiting for publish')

            pubsub.check_health()

            msg = pub.parse_response()



            s=str(msg[2],encoding='utf-8')

            print(s)

            if s=='exit':

                break





    # 发布者

    elif sys.argv[1]=='p':

        print('in publish mode')

        msg = sys.argv[2]

        print(f'msg -> {msg}')

        helper.publish(msg)

db.test.update({},{$rename:{'旧字段':'新字段'}},true,true)

db.getCollection('example').update({},{$rename:{'corp':'企业'}})

db.getCollection('example').update({},{$rename:{'corp.address':'corp.地址'}})

from random import randint



randint(0,1)

Out[25]: 1



randint(0,1)

Out[26]: 1



randint(0,1)

Out[27]: 1



randint(0,1)

Out[28]: 1



randint(0,1)

Out[29]: 0



randint(0,1)

Out[30]: 1

     file='test.txt'

     cmd = f'rsync -av  {file} root@10.18.6.46:/home/cjw/'



      p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,executable="/bin/bash")

      output, error = p.communicate()

      if p.returncode != 0:

          print("Error while running - %s" % cmd)

          print(error)

      print(output)

 p0 = multiprocessing.Process(name=str(0), target=foo, args=(0,))

    p0.start()

    p1 = multiprocessing.Process(name=str(1), target=foo, args=(1,))

    p1.start()

    p2 = multiprocessing.Process(name=str(2), target=foo, args=(2,))

    p2.start()

    p3 = multiprocessing.Process(name=str(3), target=foo, args=(3,))

    p3.start()

    p4 = multiprocessing.Process(name=str(4), target=foo, args=(4,))

    p4.start()



    p5 = multiprocessing.Process(name=str(5), target=foo, args=(5,))

    p5.start()

    

    p0.join()

    p1.join()

    p2.join()

    p3.join()

    p4.join()

    p5.join()

$ wget https://www.python.org/ftp/pyt ... 1.tgz 

$ tar zxvf Python-3.6.1.tgz $ cd Python-3.6.1

$ sudo ./configure && sudo make && sudo make install

Python 3.6.1 (default, Jul 21 2019, 14:26:28) 
[GCC 4.9.2] on linux
Type "help", "copyright", "credits" or "license" for more information.
 

from __future__ import print_function



import re



import requests



from frontera.contrib.requests.manager import RequestsFrontierManager

# from frontera.contrib.requests.manager import RequestsFrontierManager

from frontera import Settings



from six.moves.urllib.parse import urljoin





SETTINGS = Settings()

SETTINGS.BACKEND = 'frontera.contrib.backends.memory.FIFO'

# SETTINGS.BACKEND = 'frontera.contrib.backends.memory.MemoryDistributedBackend'



SETTINGS.LOGGING_MANAGER_ENABLED = True

SETTINGS.LOGGING_BACKEND_ENABLED = True

SETTINGS.MAX_REQUESTS = 100

SETTINGS.MAX_NEXT_REQUESTS = 10



SEEDS = [

    'http://www.imdb.com',

]



LINK_RE = re.compile(r'<a.+?href="(.*?)".?>', re.I)





def extract_page_links(response):

    return [urljoin(response.url, link) for link in LINK_RE.findall(response.text)]



if __name__ == '__main__':



    frontier = RequestsFrontierManager(SETTINGS)

    frontier.add_seeds([requests.Request(url=url) for url in SEEDS])

    while True:

        next_requests = frontier.get_next_requests()

        if not next_requests:

            break

        for request in next_requests:

                try:

                    response = requests.get(request.url)

                    links = [

                        requests.Request(url=url)

                        for url in extract_page_links(response)

                    ]

                    frontier.page_crawled(response)

                    print('Crawled', response.url, '(found', len(links), 'urls)')



                    if links:

                        frontier.links_extracted(request, links)

                except requests.RequestException as e:

                    error_code = type(e).__name__

                    frontier.request_error(request, error_code)

                    print('Failed to process request', request.url, 'Error:', e)

raise NameError("Module '%s' doesn't define any object named '%s'" % (module, name))

NameError: Module 'frontera.contrib.backends.memory' doesn't define any object named 'FIFO'

import re

import requests

import scrapy

from scrapy import Request

from rabbit_spider import settings

from scrapy.log import logger

import json

from rabbit_spider.items import RabbitSpiderItem

import datetime

from scrapy.selector import Selector

import pika



# from scrapy_rabbitmq.spiders import RabbitMQMixin

# from scrapy.contrib.spiders import CrawlSpider



class Website(scrapy.Spider):

    name = "rabbit"



    def start_requests(self):

        headers = {'Accept': '*/*',

                   'Accept-Encoding': 'gzip, deflate, br',

                   'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',

                   'Host': '36kr.com',

                   'Referer': 'https://36kr.com/information/web_news',

                   'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36'

                   }



        url = 'https://36kr.com/information/web_news'

        



        yield Request(url=url,

                      headers=headers)



    def parse(self, response):

       



        credentials = pika.PlainCredentials('admin', 'admin')

        connection = pika.BlockingConnection(pika.ConnectionParameters('192.168.1.101', 5672, '/', credentials))



        channel = connection.channel()

        channel.exchange_declare(exchange='direct_log', exchange_type='direct')



        result = channel.queue_declare(exclusive=True, queue='')



        queue_name = result.method.queue



        # print(queue_name)

        # infos = sys.argv[1:] if len(sys.argv)>1 else ['info']

        info = 'info'



        # 绑定多个值



        channel.queue_bind(

            exchange='direct_log',

            routing_key=info,

            queue=queue_name

        )

        print('start to receive [{}]'.format(info))



        channel.basic_consume(

            on_message_callback=self.callback_func,

            queue=queue_name,

            auto_ack=True,

        )



        channel.start_consuming()





    def callback_func(self, ch, method, properties, body):

        print(body)

from scrapy import cmdline

cmdline.execute('scrapy crawl rabbit'.split())

import pika

import settings



credentials = pika.PlainCredentials('admin','admin')

connection = pika.BlockingConnection(pika.ConnectionParameters('192.168.1.101',5672,'/',credentials))



channel = connection.channel()

channel.exchange_declare(exchange='direct_log',exchange_type='direct') # fanout 就是组播



routing_key = 'info'

message='https://36kr.com/pp/api/aggregation-entity?type=web_latest_article&b_id=59499&per_page=30'

channel.basic_publish(

	exchange='direct_log',

	routing_key=routing_key,

	body=message

	)



print('sending message {}'.format(message))

connection.close()

	credentials = pika.PlainCredentials('admin','admin')

	connection = pika.BlockingConnection(pika.ConnectionParameters('192.168.1.101',5672,'/',credentials))



	channel = connection.channel()



	channel.exchange_declare(exchange='logs',exchange_type='fanout')

Twisted-16.7.0 is coming soon, and it deprecates twisted.web.client.getPage (and client.HTTPClientFactory). We use these in some of the unit tests, to fetch one of the HTTP WAPI/WUI pages and make sure the contents look right.

We need to change these tests to use twisted.web.client.Agent instead, or a package named "treq", which is a Twisted flavor of the excellent (but blocking) requests library.

	from scrapy.selector import Selector

	

	def get_response_callback(content):

		txt = str(content,encoding='utf-8')

		resp = Selector(text=txt)

		title = resp.xpath('//title/text()').extract_first()

		print(title)

	

	@defer.inlineCallbacks

	def task():

		url = 'http://www.baidu.com'

		d=getPage(url.encode('utf-8'))

		d.addCallback(get_response_callback)

		yield d



	def done():

		reactor.stop()



	def done1(*args,**kwargs):

		reactor.stop()



	task_list = 

	for i in range(4):

		d=task()

		task_list.append(d)



	dd = defer.DeferredList(task_list)



	dd.addBoth(done)



	reactor.run()

Suppose you have a matrix M whose (i,j)-th element equals



M_ij = 2*i + 3*j

One way to define this matrix would be



i, j = np.indices((2,3))

M = 2*i + 3*j

which yields



array([[0, 3, 6],

       [2, 5, 8]])

In other words, np.indices returns arrays which can be used as indices. The elements in i indicate the row index:



In [12]: i

Out[12]: 

array([[0, 0, 0],

       [1, 1, 1]])

The elements in j indicate the column index:



In [13]: j

Out[13]: 

array([[0, 1, 2],

       [0, 1, 2]])