# -*-coding=utf-8-*-



# @Time : 2019/9/9 9:25

# @File : mail_backup.py

# @Author : 

import codecs

import re

import config

import os

from exchangelib import DELEGATE, Account, Credentials, Configuration, NTLM, Message, Mailbox, HTMLBody,FileAttachment,ItemAttachment

from exchangelib.protocol import BaseProtocol, NoVerifyHTTPAdapter





#此句用来消除ssl证书错误，exchange使用自签证书需加上

BaseProtocol.HTTP_ADAPTER_CLS = NoVerifyHTTPAdapter





# 输入你的域账号如example\xxx

cred = Credentials(r'example\xxx', 你的邮箱密码)



configx = Configuration(server='mail.credlink.com', credentials=cred, auth_type=NTLM)

a = Account(

    primary_smtp_address='你的邮箱地址', config=configx, autodiscover=False, access_type=DELEGATE

)





for item in a.inbox.all().order_by('-datetime_received')[:100]:

    print(item.subject, item.sender, item.unique_body,item.datetime_received)

   

    name = item.subject

    name = re.sub('[\/:*?"<>|]', '-', name)

    local_path = os.path.join('inbox', name+'.html')

    with codecs.open(local_path, 'w','utf-8') as f:

        f.write(item.unique_body)



    for attachment in item.attachments:

        if isinstance(attachment, FileAttachment):

            name = attachment.name

            name = re.sub('[\/:*?"<>|]','-',name)

            local_path = os.path.join('inbox', attachment.name)

            with codecs.open(local_path, 'wb') as f:

                f.write(attachment.content)

            print('Saved attachment to', local_path)



        elif isinstance(attachment, ItemAttachment):

            if isinstance(attachment.item, Message):

                name=attachment.item.subject

                name = re.sub('[\/:*?"<>|]', '-', name)

                local_path = os.path.join('inbox', 'attachment')

                with codecs.open(local_path, 'w') as f:

                    f.write(attachment.item.body)

import time



LOOP = 2*10**8



def add(x,y):

    return x+y



def cpu_pressure(loop):

    

    for i in range(loop):

        result = add(i,i+1)





if __name__ == '__main__':

    start = time.time()

    cpu_pressure(LOOP)

    print(f'time used {time.time()-start}s')

class Command(ScrapyCommand):



    requires_project = True



    def syntax(self):

        return '[options]'



    def short_desc(self):

        return 'Runs all of the spiders - My Defined'



    def run(self,args,opts):

        print('==================')

        print(type(self.crawler_process))

        spider_list = self.crawler_process.spiders.list() # 找到爬虫类



        for name in spider_list:

            print('=================')

            print(name)

            self.crawler_process.crawl(name,**opts.__dict__)



        self.crawler_process.start()

      def __init__(self, settings=None):

        if isinstance(settings, dict) or settings is None:

            settings = Settings(settings)

        self.settings = settings

        self.spider_loader = _get_spider_loader(settings) # 构造爬虫

        self._crawlers = set()

        self._active = set()

        self.bootstrap_failed = False

def _get_spider_loader(settings):



    cls_path = settings.get('SPIDER_LOADER_CLASS')

    

    # settings文件没有定义SPIDER_LOADER_CLASS，所以这里获取到的是系统的默认配置文件，

    # 默认配置文件在接下来的代码块A

    # SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'    

    

    loader_cls = load_object(cls_path) 

    # 这个函数就是根据路径转为类对象，也就是上面crapy.spiderloader.SpiderLoader 这个

    # 字符串变成一个类对象

    # 具体的load_object 对象代码见下面代码块B



    return loader_cls.from_settings(settings.frozencopy())

# 代码块A

#......省略若干

SCHEDULER = 'scrapy.core.scheduler.Scheduler'

SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'

SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'

SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.ScrapyPriorityQueue'



SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader' 就是这个值

SPIDER_LOADER_WARN_ONLY = False



SPIDER_MIDDLEWARES = {}

# 代码块B 为了方便，我把异常处理的去除

from importlib import import_module #导入第三方库



def load_object(path):

    dot = path.rindex('.') 

    module, name = path[:dot], path[dot+1:]

    # 上面把路径分为基本路径+模块名

    

    mod = import_module(module)

    obj = getattr(mod, name)

    # 获取模块里面那个值

    

    return obj

In [33]: mod = import_module(module)                                                                                                                                             



In [34]: mod                                                                                                                                                                     

Out[34]: <module 'scrapy.spiderloader' from '/home/xda/anaconda3/lib/python3.7/site-packages/scrapy/spiderloader.py'>



In [35]: getattr(mod,name)                                                                                                                                                       

Out[35]: scrapy.spiderloader.SpiderLoader



In [36]: obj = getattr(mod,name)                                                                                                                                                 



In [37]: obj                                                                                                                                                                     

Out[37]: scrapy.spiderloader.SpiderLoader



In [38]: type(obj)                                                                                                                                                               

Out[38]: type

    def from_settings(cls, settings):

        return cls(settings)

class SpiderLoader(object):

    """

    SpiderLoader is a class which locates and loads spiders

    in a Scrapy project.

    """

    def __init__(self, settings):

        self.spider_modules = settings.getlist('SPIDER_MODULES') 

        # 获得settting中的模块名字，创建scrapy的时候就默认帮你生成了

        # 你可以看看你的settings文件里面的内容就可以找到这个值，是一个list

        

        self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY')

        self._spiders = {}

        self._found = defaultdict(list)

        self._load_all_spiders() # 加载所有爬虫

def _load_all_spiders(self):

        for name in self.spider_modules:            



                for module in walk_modules(name): # 这个遍历文件夹里面的文件，然后再转化为类对象，

                    # 保存到字典：self._spiders = {}       

                    self._load_spiders(module) # 模块变成spider



        self._check_name_duplicates() # 去重，如果名字一样就异常

def iter_spider_classes(module):

    from scrapy.spiders import Spider



    for obj in six.itervalues(vars(module)): # 找到模块里面的变量，然后迭代出来

        if inspect.isclass(obj) and \

           issubclass(obj, Spider) and \

           obj.__module__ == module.__name__ and \

           getattr(obj, 'name', None): # 有name属性，继承于Spider

           yield obj

* * * * * export DISPLAY=:0; gedit

import pyautogui as pag

import datetime



def neck_rest():

    f = open('neck_record.txt', 'a')

    ret = pag.prompt("Rest! Protect your neck !")

    if ret == 'rest':

        f.write(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))

        f.write('\t')

        f.write('Rest')

        f.write('\n')

    else:

        f.write(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))

        f.write('\t')

        f.write('Failed to rest')

        f.write('\n')

    f.close()



neck_rest()

redis://[:password]@localhost:6379/0

rediss://[:password]@localhost:6379/0

unix://[:password]@/path/to/socket.sock?db=0

db.test_tab.insert({array:[]})

db.test_tab.insert({array:[]})

db.test_tab.insert({array:[]})

db.test_tab.insert({array:[1,2,3,4,5]})

db.test_tab.insert({array:[1,2,3,4,5,6]})

db.getCollection("example").find({array:{$exists:true,$ne:[]}}); # 字段不为0

    from . import (constants, error, message, context,

ImportError: DLL load failed: 找不到指定的模块。

 # helper

    class RedisHelp(object):



        def __init__(self,channel):

            # self.pool = redis.ConnectionPool('10.18.6.46',port=6379)



            # self.conn = redis.Redis(connection_pool=self.pool)

            # 上面的方式无法使用订阅者 发布者模式



            self.conn = redis.Redis(host='10.18.6.46')

            self.publish_channel = channel

            self.subscribe_channel = channel





        def publish(self,msg):

            self.conn.publish(self.publish_channel,msg) # 1. 渠道名 ,2 信息



        def subscribe(self):

            self.pub = self.conn.pubsub()

            self.pub.subscribe(self.subscribe_channel)

            self.pub.parse_response()

            print('initial')

            return self.pub





    helper = RedisHelp('cuiqingcai')



    # 订阅者

    if sys.argv[1]=='s':

        print('in subscribe mode')

        pub = helper.subscribe()

        while 1:

            print('waiting for publish')

            pubsub.check_health()

            msg = pub.parse_response()



            s=str(msg[2],encoding='utf-8')

            print(s)

            if s=='exit':

                break





    # 发布者

    elif sys.argv[1]=='p':

        print('in publish mode')

        msg = sys.argv[2]

        print(f'msg -> {msg}')

        helper.publish(msg)

db.test.update({},{$rename:{'旧字段':'新字段'}},true,true)

db.getCollection('example').update({},{$rename:{'corp':'企业'}})

db.getCollection('example').update({},{$rename:{'corp.address':'corp.地址'}})

from random import randint



randint(0,1)

Out[25]: 1



randint(0,1)

Out[26]: 1



randint(0,1)

Out[27]: 1



randint(0,1)

Out[28]: 1



randint(0,1)

Out[29]: 0



randint(0,1)

Out[30]: 1

     file='test.txt'

     cmd = f'rsync -av  {file} root@10.18.6.46:/home/cjw/'



      p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,executable="/bin/bash")

      output, error = p.communicate()

      if p.returncode != 0:

          print("Error while running - %s" % cmd)

          print(error)

      print(output)

 p0 = multiprocessing.Process(name=str(0), target=foo, args=(0,))

    p0.start()

    p1 = multiprocessing.Process(name=str(1), target=foo, args=(1,))

    p1.start()

    p2 = multiprocessing.Process(name=str(2), target=foo, args=(2,))

    p2.start()

    p3 = multiprocessing.Process(name=str(3), target=foo, args=(3,))

    p3.start()

    p4 = multiprocessing.Process(name=str(4), target=foo, args=(4,))

    p4.start()



    p5 = multiprocessing.Process(name=str(5), target=foo, args=(5,))

    p5.start()

    

    p0.join()

    p1.join()

    p2.join()

    p3.join()

    p4.join()

    p5.join()

$ wget https://www.python.org/ftp/pyt ... 1.tgz 

$ tar zxvf Python-3.6.1.tgz $ cd Python-3.6.1

$ sudo ./configure && sudo make && sudo make install

Python 3.6.1 (default, Jul 21 2019, 14:26:28) 
[GCC 4.9.2] on linux
Type "help", "copyright", "credits" or "license" for more information.
 

from __future__ import print_function



import re



import requests



from frontera.contrib.requests.manager import RequestsFrontierManager

# from frontera.contrib.requests.manager import RequestsFrontierManager

from frontera import Settings



from six.moves.urllib.parse import urljoin





SETTINGS = Settings()

SETTINGS.BACKEND = 'frontera.contrib.backends.memory.FIFO'

# SETTINGS.BACKEND = 'frontera.contrib.backends.memory.MemoryDistributedBackend'



SETTINGS.LOGGING_MANAGER_ENABLED = True

SETTINGS.LOGGING_BACKEND_ENABLED = True

SETTINGS.MAX_REQUESTS = 100

SETTINGS.MAX_NEXT_REQUESTS = 10



SEEDS = [

    'http://www.imdb.com',

]



LINK_RE = re.compile(r'<a.+?href="(.*?)".?>', re.I)





def extract_page_links(response):

    return [urljoin(response.url, link) for link in LINK_RE.findall(response.text)]



if __name__ == '__main__':



    frontier = RequestsFrontierManager(SETTINGS)

    frontier.add_seeds([requests.Request(url=url) for url in SEEDS])

    while True:

        next_requests = frontier.get_next_requests()

        if not next_requests:

            break

        for request in next_requests:

                try:

                    response = requests.get(request.url)

                    links = [

                        requests.Request(url=url)

                        for url in extract_page_links(response)

                    ]

                    frontier.page_crawled(response)

                    print('Crawled', response.url, '(found', len(links), 'urls)')



                    if links:

                        frontier.links_extracted(request, links)

                except requests.RequestException as e:

                    error_code = type(e).__name__

                    frontier.request_error(request, error_code)

                    print('Failed to process request', request.url, 'Error:', e)

raise NameError("Module '%s' doesn't define any object named '%s'" % (module, name))

NameError: Module 'frontera.contrib.backends.memory' doesn't define any object named 'FIFO'

import re

import requests

import scrapy

from scrapy import Request

from rabbit_spider import settings

from scrapy.log import logger

import json

from rabbit_spider.items import RabbitSpiderItem

import datetime

from scrapy.selector import Selector

import pika



# from scrapy_rabbitmq.spiders import RabbitMQMixin

# from scrapy.contrib.spiders import CrawlSpider



class Website(scrapy.Spider):

    name = "rabbit"



    def start_requests(self):

        headers = {'Accept': '*/*',

                   'Accept-Encoding': 'gzip, deflate, br',

                   'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',

                   'Host': '36kr.com',

                   'Referer': 'https://36kr.com/information/web_news',

                   'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36'

                   }



        url = 'https://36kr.com/information/web_news'

        



        yield Request(url=url,

                      headers=headers)



    def parse(self, response):

       



        credentials = pika.PlainCredentials('admin', 'admin')

        connection = pika.BlockingConnection(pika.ConnectionParameters('192.168.1.101', 5672, '/', credentials))



        channel = connection.channel()

        channel.exchange_declare(exchange='direct_log', exchange_type='direct')



        result = channel.queue_declare(exclusive=True, queue='')



        queue_name = result.method.queue



        # print(queue_name)

        # infos = sys.argv[1:] if len(sys.argv)>1 else ['info']

        info = 'info'



        # 绑定多个值



        channel.queue_bind(

            exchange='direct_log',

            routing_key=info,

            queue=queue_name

        )

        print('start to receive [{}]'.format(info))



        channel.basic_consume(

            on_message_callback=self.callback_func,

            queue=queue_name,

            auto_ack=True,

        )



        channel.start_consuming()





    def callback_func(self, ch, method, properties, body):

        print(body)

from scrapy import cmdline

cmdline.execute('scrapy crawl rabbit'.split())

import pika

import settings



credentials = pika.PlainCredentials('admin','admin')

connection = pika.BlockingConnection(pika.ConnectionParameters('192.168.1.101',5672,'/',credentials))



channel = connection.channel()

channel.exchange_declare(exchange='direct_log',exchange_type='direct') # fanout 就是组播



routing_key = 'info'

message='https://36kr.com/pp/api/aggregation-entity?type=web_latest_article&b_id=59499&per_page=30'

channel.basic_publish(

	exchange='direct_log',

	routing_key=routing_key,

	body=message

	)



print('sending message {}'.format(message))

connection.close()

	credentials = pika.PlainCredentials('admin','admin')

	connection = pika.BlockingConnection(pika.ConnectionParameters('192.168.1.101',5672,'/',credentials))



	channel = connection.channel()



	channel.exchange_declare(exchange='logs',exchange_type='fanout')

Twisted-16.7.0 is coming soon, and it deprecates twisted.web.client.getPage (and client.HTTPClientFactory). We use these in some of the unit tests, to fetch one of the HTTP WAPI/WUI pages and make sure the contents look right.

We need to change these tests to use twisted.web.client.Agent instead, or a package named "treq", which is a Twisted flavor of the excellent (but blocking) requests library.

	from scrapy.selector import Selector

	

	def get_response_callback(content):

		txt = str(content,encoding='utf-8')

		resp = Selector(text=txt)

		title = resp.xpath('//title/text()').extract_first()

		print(title)

	

	@defer.inlineCallbacks

	def task():

		url = 'http://www.baidu.com'

		d=getPage(url.encode('utf-8'))

		d.addCallback(get_response_callback)

		yield d



	def done():

		reactor.stop()



	def done1(*args,**kwargs):

		reactor.stop()



	task_list = 

	for i in range(4):

		d=task()

		task_list.append(d)



	dd = defer.DeferredList(task_list)



	dd.addBoth(done)



	reactor.run()

Suppose you have a matrix M whose (i,j)-th element equals



M_ij = 2*i + 3*j

One way to define this matrix would be



i, j = np.indices((2,3))

M = 2*i + 3*j

which yields



array([[0, 3, 6],

       [2, 5, 8]])

In other words, np.indices returns arrays which can be used as indices. The elements in i indicate the row index:



In [12]: i

Out[12]: 

array([[0, 0, 0],

       [1, 1, 1]])

The elements in j indicate the column index:



In [13]: j

Out[13]: 

array([[0, 1, 2],

       [0, 1, 2]])

distanceTransform

Calculates the distance to the closest zero pixel for each pixel of the source image.





Python: cv2.distanceTransform(src, distanceType, maskSize[, dst]) → dst



Python: cv.DistTransform(src, dst, distance_type=CV_DIST_L2, mask_size=3, mask=None, labels=None) → None

Parameters:	

src – 8-bit, single-channel (binary) source image.

dst – Output image with calculated distances. It is a 32-bit floating-point, single-channel image of the same size as src .

distanceType – Type of distance. It can be CV_DIST_L1, CV_DIST_L2 , or CV_DIST_C .

maskSize – Size of the distance transform mask. It can be 3, 5, or CV_DIST_MASK_PRECISE (the latter option is only supported by the first function). In case of the CV_DIST_L1 or CV_DIST_C distance type, the parameter is forced to 3 because a  3\times 3 mask gives the same result as  5\times 5 or any larger aperture.

labels – Optional output 2D array of labels (the discrete Voronoi diagram). It has the type CV_32SC1 and the same size as src . See the details below.

labelType – Type of the label array to build. If labelType==DIST_LABEL_CCOMP then each connected component of zeros in src (as well as all the non-zero pixels closest to the connected component) will be assigned the same label. If labelType==DIST_LABEL_PIXEL then each zero pixel (and all the non-zero pixels closest to it) gets its own label.

The functions distanceTransform calculate the approximate or precise distance from every binary image pixel to the nearest zero pixel. For zero image pixels, the distance will obviously be zero.

When maskSize == CV_DIST_MASK_PRECISE and distanceType == CV_DIST_L2 , the function runs the algorithm described in [Felzenszwalb04]. This algorithm is parallelized with the TBB library.



In other cases, the algorithm [Borgefors86] is used. This means that for a pixel the function finds the shortest path to the nearest zero pixel consisting of basic shifts: horizontal, vertical, diagonal, or knight’s move (the latest is available for a 5\times 5 mask). The overall distance is calculated as a sum of these basic distances. Since the distance function should be symmetric, all of the horizontal and vertical shifts must have the same cost (denoted as a ), all the diagonal shifts must have the same cost (denoted as b ), and all knight’s moves must have the same cost (denoted as c ). For the CV_DIST_C and CV_DIST_L1 types, the distance is calculated precisely, whereas for CV_DIST_L2 (Euclidean distance) the distance can be calculated only with a relative error (a 5\times 5 mask gives more accurate results). For a,``b`` , and c , OpenCV uses the values suggested in the original paper:



CV_DIST_C	(3\times 3)	a = 1, b = 1

CV_DIST_L1	(3\times 3)	a = 1, b = 2

CV_DIST_L2	(3\times 3)	a=0.955, b=1.3693

CV_DIST_L2	(5\times 5)	a=1, b=1.4, c=2.1969

Typically, for a fast, coarse distance estimation CV_DIST_L2, a 3\times 3 mask is used. For a more accurate distance estimation CV_DIST_L2 , a 5\times 5 mask or the precise algorithm is used. Note that both the precise and the approximate algorithms are linear on the number of pixels.



The second variant of the function does not only compute the minimum distance for each pixel (x, y) but also identifies the nearest connected component consisting of zero pixels (labelType==DIST_LABEL_CCOMP) or the nearest zero pixel (labelType==DIST_LABEL_PIXEL). Index of the component/pixel is stored in \texttt{labels}(x, y) . When labelType==DIST_LABEL_CCOMP, the function automatically finds connected components of zero pixels in the input image and marks them with distinct labels. When labelType==DIST_LABEL_CCOMP, the function scans through the input image and marks all the zero pixels with distinct labels.



In this mode, the complexity is still linear. That is, the function provides a very fast way to compute the Voronoi diagram for a binary image. Currently, the second variant can use only the approximate distance transform algorithm, i.e. maskSize=CV_DIST_MASK_PRECISE is not supported yet.



Note

An example on using the distance transform can be found at opencv_source_code/samples/cpp/distrans.cpp

(Python) An example on using the distance transform can be found at opencv_source/samples/python2/distrans.py