# -*- coding: utf-8 -*-

 

"""

PyQt5 tutorial

 

In this example, we create a custom widget.

"""

import sys

from PyQt5.QtWidgets import (QWidget, QSlider, QApplication,

                             QHBoxLayout, QVBoxLayout)

from PyQt5.QtCore import QObject, Qt, pyqtSignal

from PyQt5.QtGui import QPainter, QFont, QColor, QPen

 

 

class Communicate(QObject):

    updateBW = pyqtSignal(int)

 

 

class BurningWidget(QWidget):

    def __init__(self):

        super().__init__()

 

        self.initUI()

 

    def initUI(self):

 

        self.setMinimumSize(1, 30)

        self.value = 75

        self.num = [75, 150, 225, 300, 375, 450, 525, 600, 675]

 

    def setValue(self, value):

 

        self.value = value

 

    def paintEvent(self, e):

 

        qp = QPainter()

        qp.begin(self)

        self.drawWidget(qp)

        qp.end()

 

    def drawWidget(self, qp):

 

        font = QFont('Serif', 7, QFont.Light)

        qp.setFont(font)

 

        size = self.size()

        w = size.width()

        h = size.height()

 

        step = int(round(w / 10.0))

 

        till = int(((w / 750.0) * self.value))

        full = int(((w / 750.0) * 700))

 

        if self.value >= 700:

 

            qp.setPen(QColor(255, 255, 255))

            qp.setBrush(QColor(255, 255, 184))

            qp.drawRect(0, 0, full, h)

            qp.setPen(QColor(255, 175, 175))

            qp.setBrush(QColor(255, 175, 175))

            qp.drawRect(full, 0, till - full, h)

 

        else:

 

            qp.setPen(QColor(255, 255, 255))

            qp.setBrush(QColor(255, 255, 184))

            qp.drawRect(0, 0, till, h)

 

        pen = QPen(QColor(20, 20, 20), 1,

                   Qt.SolidLine)

 

        qp.setPen(pen)

        qp.setBrush(Qt.NoBrush)

        qp.drawRect(0, 0, w - 1, h - 1)

 

        j = 0

 

        for i in range(step, 10 * step, step):

            qp.drawLine(i, 0, i, 5)

            metrics = qp.fontMetrics()

            fw = metrics.width(str(self.num[j]))

            qp.drawText(i - fw / 2, h / 2, str(self.num[j]))

            j = j + 1

 

 

class Example(QWidget):

    def __init__(self):

        super().__init__()

 

        self.initUI()

 

    def initUI(self):

        sld = QSlider(Qt.Horizontal, self)

        sld.setFocusPolicy(Qt.NoFocus)

        sld.setRange(1, 750)

        sld.setValue(75)

        sld.setGeometry(30, 40, 150, 30)

 

        self.c = Communicate()

        self.wid = BurningWidget()

        self.c.updateBW[int].connect(self.wid.setValue)

 

        sld.valueChanged[int].connect(self.changeValue)

        hbox = QHBoxLayout()

        hbox.addWidget(self.wid)

        vbox = QVBoxLayout()

        vbox.addStretch(1)

        vbox.addLayout(hbox)

        self.setLayout(vbox)

 

        self.setGeometry(300, 300, 390, 210)

        self.setWindowTitle('Burning widget')

        self.show()

 

    def changeValue(self, value):

        self.c.updateBW.emit(value)

        self.wid.repaint()

 

 

if __name__ == '__main__':

    app = QApplication(sys.argv)

    ex = Example()

    sys.exit(app.exec_())

class BurningWidget(QWidget):

  

    def __init__(self):      

        super().__init__() 

self.setMinimumSize(1, 30)

font = QFont('Serif', 7, QFont.Light)

qp.setFont(font)

size = self.size()

w = size.width()

h = size.height()

 

step = int(round(w / 10.0))

 

 

till = int(((w / 750.0) * self.value))

full = int(((w / 750.0) * 700))

metrics = qp.fontMetrics()

fw = metrics.width(str(self.num[j]))

qp.drawText(i-fw/2, h/2, str(self.num[j]))

def changeValue(self, value):

          

    self.c.updateBW.emit(value)        

    self.wid.repaint()

from PyQt5.QtCore import Qt,QRect

import time

import socketio



sio = socketio.Client()

start_timer = None





def send_ping():

    global start_timer

    start_timer = time.time()

    sio.emit('ping_from_client')





@sio.event

def connect():

    print('connected to server')

    send_ping()





@sio.event

def pong_from_server(data):

    global start_timer

    latency = time.time() - start_timer

    print('latency is {0:.2f} ms'.format(latency * 1000))

    sio.sleep(1)

    send_ping()





if __name__ == '__main__':

    sio.connect('http://localhost:5000')

    sio.wait()

    print('next')

import jieba.posseg as pseg

seg_list = pseg.cut("我爱北京天安门")

for word,flag in seg_list:

    print(word)

    print(flag)

---------------------------------------------------------------------------

TypeError                                 Traceback (most recent call last)

<ipython-input-5-f105f6980f88> in <module>()

      1 import jieba.posseg as pseg

      2 seg_list = pseg.cut("我爱北京天安门")

----> 3 for word,flag in seg_list:

      4     print(word)

      5     print(flag)



TypeError: cannot unpack non-iterable pair object

原因是新版本中seg_list是一个生成器，所以只能 for win seg_list

然后从word中解包出来

print(w.word)

print(w.flag)

这样问题就解决了。

from lxml.html import etree

from lxml.html import fromstring, HtmlElement



test_html = '''<p><span>hello</span><span>world</span></p>'''

test_element = fromstring(test_html)

etree.strip_tags(test_element,'span') # 清除span标签

etree.tostring(test_element)

url = 'http://xyhz.huizhou.gov.cn/static/js/common/jigsaw/images/{}.jpg'

headers={'User-Agent':'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'}

async def getPage(num):



    async with aiohttp.ClientSession() as session:

        async with session.get(url.format(num),headers=headers) as resp:

            if resp.status==200:

                f= await aiofiles.open('{}.jpg'.format(num),mode='wb')

                await f.write(await resp.read())

                await f.close()



loop = asyncio.get_event_loop()

tasks = [getPage(i) for i in range(5)]

loop.run_until_complete(asyncio.wait(tasks))

# -*-coding=utf-8-*-



# @Time : 2019/9/9 9:25

# @File : mail_backup.py

# @Author : 

import codecs

import re

import config

import os

from exchangelib import DELEGATE, Account, Credentials, Configuration, NTLM, Message, Mailbox, HTMLBody,FileAttachment,ItemAttachment

from exchangelib.protocol import BaseProtocol, NoVerifyHTTPAdapter





#此句用来消除ssl证书错误，exchange使用自签证书需加上

BaseProtocol.HTTP_ADAPTER_CLS = NoVerifyHTTPAdapter





# 输入你的域账号如example\xxx

cred = Credentials(r'example\xxx', 你的邮箱密码)



configx = Configuration(server='mail.credlink.com', credentials=cred, auth_type=NTLM)

a = Account(

    primary_smtp_address='你的邮箱地址', config=configx, autodiscover=False, access_type=DELEGATE

)





for item in a.inbox.all().order_by('-datetime_received')[:100]:

    print(item.subject, item.sender, item.unique_body,item.datetime_received)

   

    name = item.subject

    name = re.sub('[\/:*?"<>|]', '-', name)

    local_path = os.path.join('inbox', name+'.html')

    with codecs.open(local_path, 'w','utf-8') as f:

        f.write(item.unique_body)



    for attachment in item.attachments:

        if isinstance(attachment, FileAttachment):

            name = attachment.name

            name = re.sub('[\/:*?"<>|]','-',name)

            local_path = os.path.join('inbox', attachment.name)

            with codecs.open(local_path, 'wb') as f:

                f.write(attachment.content)

            print('Saved attachment to', local_path)



        elif isinstance(attachment, ItemAttachment):

            if isinstance(attachment.item, Message):

                name=attachment.item.subject

                name = re.sub('[\/:*?"<>|]', '-', name)

                local_path = os.path.join('inbox', 'attachment')

                with codecs.open(local_path, 'w') as f:

                    f.write(attachment.item.body)

import time



LOOP = 2*10**8



def add(x,y):

    return x+y



def cpu_pressure(loop):

    

    for i in range(loop):

        result = add(i,i+1)





if __name__ == '__main__':

    start = time.time()

    cpu_pressure(LOOP)

    print(f'time used {time.time()-start}s')

class Command(ScrapyCommand):



    requires_project = True



    def syntax(self):

        return '[options]'



    def short_desc(self):

        return 'Runs all of the spiders - My Defined'



    def run(self,args,opts):

        print('==================')

        print(type(self.crawler_process))

        spider_list = self.crawler_process.spiders.list() # 找到爬虫类



        for name in spider_list:

            print('=================')

            print(name)

            self.crawler_process.crawl(name,**opts.__dict__)



        self.crawler_process.start()

      def __init__(self, settings=None):

        if isinstance(settings, dict) or settings is None:

            settings = Settings(settings)

        self.settings = settings

        self.spider_loader = _get_spider_loader(settings) # 构造爬虫

        self._crawlers = set()

        self._active = set()

        self.bootstrap_failed = False

def _get_spider_loader(settings):



    cls_path = settings.get('SPIDER_LOADER_CLASS')

    

    # settings文件没有定义SPIDER_LOADER_CLASS，所以这里获取到的是系统的默认配置文件，

    # 默认配置文件在接下来的代码块A

    # SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'    

    

    loader_cls = load_object(cls_path) 

    # 这个函数就是根据路径转为类对象，也就是上面crapy.spiderloader.SpiderLoader 这个

    # 字符串变成一个类对象

    # 具体的load_object 对象代码见下面代码块B



    return loader_cls.from_settings(settings.frozencopy())

# 代码块A

#......省略若干

SCHEDULER = 'scrapy.core.scheduler.Scheduler'

SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'

SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'

SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.ScrapyPriorityQueue'



SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader' 就是这个值

SPIDER_LOADER_WARN_ONLY = False



SPIDER_MIDDLEWARES = {}

# 代码块B 为了方便，我把异常处理的去除

from importlib import import_module #导入第三方库



def load_object(path):

    dot = path.rindex('.') 

    module, name = path[:dot], path[dot+1:]

    # 上面把路径分为基本路径+模块名

    

    mod = import_module(module)

    obj = getattr(mod, name)

    # 获取模块里面那个值

    

    return obj

In [33]: mod = import_module(module)                                                                                                                                             



In [34]: mod                                                                                                                                                                     

Out[34]: <module 'scrapy.spiderloader' from '/home/xda/anaconda3/lib/python3.7/site-packages/scrapy/spiderloader.py'>



In [35]: getattr(mod,name)                                                                                                                                                       

Out[35]: scrapy.spiderloader.SpiderLoader



In [36]: obj = getattr(mod,name)                                                                                                                                                 



In [37]: obj                                                                                                                                                                     

Out[37]: scrapy.spiderloader.SpiderLoader



In [38]: type(obj)                                                                                                                                                               

Out[38]: type

    def from_settings(cls, settings):

        return cls(settings)

class SpiderLoader(object):

    """

    SpiderLoader is a class which locates and loads spiders

    in a Scrapy project.

    """

    def __init__(self, settings):

        self.spider_modules = settings.getlist('SPIDER_MODULES') 

        # 获得settting中的模块名字，创建scrapy的时候就默认帮你生成了

        # 你可以看看你的settings文件里面的内容就可以找到这个值，是一个list

        

        self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY')

        self._spiders = {}

        self._found = defaultdict(list)

        self._load_all_spiders() # 加载所有爬虫

def _load_all_spiders(self):

        for name in self.spider_modules:            



                for module in walk_modules(name): # 这个遍历文件夹里面的文件，然后再转化为类对象，

                    # 保存到字典：self._spiders = {}       

                    self._load_spiders(module) # 模块变成spider



        self._check_name_duplicates() # 去重，如果名字一样就异常

def iter_spider_classes(module):

    from scrapy.spiders import Spider



    for obj in six.itervalues(vars(module)): # 找到模块里面的变量，然后迭代出来

        if inspect.isclass(obj) and \

           issubclass(obj, Spider) and \

           obj.__module__ == module.__name__ and \

           getattr(obj, 'name', None): # 有name属性，继承于Spider

           yield obj

    from . import (constants, error, message, context,

ImportError: DLL load failed: 找不到指定的模块。

from random import randint



randint(0,1)

Out[25]: 1



randint(0,1)

Out[26]: 1



randint(0,1)

Out[27]: 1



randint(0,1)

Out[28]: 1



randint(0,1)

Out[29]: 0



randint(0,1)

Out[30]: 1

from __future__ import print_function



import re



import requests



from frontera.contrib.requests.manager import RequestsFrontierManager

# from frontera.contrib.requests.manager import RequestsFrontierManager

from frontera import Settings



from six.moves.urllib.parse import urljoin





SETTINGS = Settings()

SETTINGS.BACKEND = 'frontera.contrib.backends.memory.FIFO'

# SETTINGS.BACKEND = 'frontera.contrib.backends.memory.MemoryDistributedBackend'



SETTINGS.LOGGING_MANAGER_ENABLED = True

SETTINGS.LOGGING_BACKEND_ENABLED = True

SETTINGS.MAX_REQUESTS = 100

SETTINGS.MAX_NEXT_REQUESTS = 10



SEEDS = [

    'http://www.imdb.com',

]



LINK_RE = re.compile(r'<a.+?href="(.*?)".?>', re.I)





def extract_page_links(response):

    return [urljoin(response.url, link) for link in LINK_RE.findall(response.text)]



if __name__ == '__main__':



    frontier = RequestsFrontierManager(SETTINGS)

    frontier.add_seeds([requests.Request(url=url) for url in SEEDS])

    while True:

        next_requests = frontier.get_next_requests()

        if not next_requests:

            break

        for request in next_requests:

                try:

                    response = requests.get(request.url)

                    links = [

                        requests.Request(url=url)

                        for url in extract_page_links(response)

                    ]

                    frontier.page_crawled(response)

                    print('Crawled', response.url, '(found', len(links), 'urls)')



                    if links:

                        frontier.links_extracted(request, links)

                except requests.RequestException as e:

                    error_code = type(e).__name__

                    frontier.request_error(request, error_code)

                    print('Failed to process request', request.url, 'Error:', e)

raise NameError("Module '%s' doesn't define any object named '%s'" % (module, name))

NameError: Module 'frontera.contrib.backends.memory' doesn't define any object named 'FIFO'

import re

import requests

import scrapy

from scrapy import Request

from rabbit_spider import settings

from scrapy.log import logger

import json

from rabbit_spider.items import RabbitSpiderItem

import datetime

from scrapy.selector import Selector

import pika



# from scrapy_rabbitmq.spiders import RabbitMQMixin

# from scrapy.contrib.spiders import CrawlSpider



class Website(scrapy.Spider):

    name = "rabbit"



    def start_requests(self):

        headers = {'Accept': '*/*',

                   'Accept-Encoding': 'gzip, deflate, br',

                   'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',

                   'Host': '36kr.com',

                   'Referer': 'https://36kr.com/information/web_news',

                   'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36'

                   }



        url = 'https://36kr.com/information/web_news'

        



        yield Request(url=url,

                      headers=headers)



    def parse(self, response):

       



        credentials = pika.PlainCredentials('admin', 'admin')

        connection = pika.BlockingConnection(pika.ConnectionParameters('192.168.1.101', 5672, '/', credentials))



        channel = connection.channel()

        channel.exchange_declare(exchange='direct_log', exchange_type='direct')



        result = channel.queue_declare(exclusive=True, queue='')



        queue_name = result.method.queue



        # print(queue_name)

        # infos = sys.argv[1:] if len(sys.argv)>1 else ['info']

        info = 'info'



        # 绑定多个值



        channel.queue_bind(

            exchange='direct_log',

            routing_key=info,

            queue=queue_name

        )

        print('start to receive [{}]'.format(info))



        channel.basic_consume(

            on_message_callback=self.callback_func,

            queue=queue_name,

            auto_ack=True,

        )



        channel.start_consuming()





    def callback_func(self, ch, method, properties, body):

        print(body)

from scrapy import cmdline

cmdline.execute('scrapy crawl rabbit'.split())

import pika

import settings



credentials = pika.PlainCredentials('admin','admin')

connection = pika.BlockingConnection(pika.ConnectionParameters('192.168.1.101',5672,'/',credentials))



channel = connection.channel()

channel.exchange_declare(exchange='direct_log',exchange_type='direct') # fanout 就是组播



routing_key = 'info'

message='https://36kr.com/pp/api/aggregation-entity?type=web_latest_article&b_id=59499&per_page=30'

channel.basic_publish(

	exchange='direct_log',

	routing_key=routing_key,

	body=message

	)



print('sending message {}'.format(message))

connection.close()

	credentials = pika.PlainCredentials('admin','admin')

	connection = pika.BlockingConnection(pika.ConnectionParameters('192.168.1.101',5672,'/',credentials))



	channel = connection.channel()



	channel.exchange_declare(exchange='logs',exchange_type='fanout')

Twisted-16.7.0 is coming soon, and it deprecates twisted.web.client.getPage (and client.HTTPClientFactory). We use these in some of the unit tests, to fetch one of the HTTP WAPI/WUI pages and make sure the contents look right.

We need to change these tests to use twisted.web.client.Agent instead, or a package named "treq", which is a Twisted flavor of the excellent (but blocking) requests library.

	from scrapy.selector import Selector

	

	def get_response_callback(content):

		txt = str(content,encoding='utf-8')

		resp = Selector(text=txt)

		title = resp.xpath('//title/text()').extract_first()

		print(title)

	

	@defer.inlineCallbacks

	def task():

		url = 'http://www.baidu.com'

		d=getPage(url.encode('utf-8'))

		d.addCallback(get_response_callback)

		yield d



	def done():

		reactor.stop()



	def done1(*args,**kwargs):

		reactor.stop()



	task_list = 

	for i in range(4):

		d=task()

		task_list.append(d)



	dd = defer.DeferredList(task_list)



	dd.addBoth(done)



	reactor.run()