# -*- coding: utf-8 -*-

# website: http://30daydo.com

# @Time : 2020/9/22 10:07

# @File : bilibili_hot_anchor.py



# 异步爬取首页与列表



import asyncio

import datetime

import aiohttp

import re

import time

from motor.motor_asyncio import AsyncIOMotorClient

from parsel import Selector

from settings import _json_data



SLEEP = 60 * 10

INFO = _json_data['mongo']['arm']

host = INFO['host']

port = INFO['port']

user = INFO['user']

password = INFO['password']

connect_uri = f'mongodb://{user}:{password}@{host}:{port}'



client = AsyncIOMotorClient(connect_uri)



db = client['db_parker']



home_url = 'https://www.bilibili.com/ranking'

headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0',

           'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2'}





def convertor(number_str):

    '''

    将小数点的万变为整数

    :param number_str:

    :return:

    '''

    number = re.search('(\d+\.+\d+)', number_str)

    if number:

        number = float(number.group(1))

        if re.search('万', number_str):

            number = int(number * 10000)

    else:

        number = 0



    return number





async def home_page():

    async with aiohttp.ClientSession() as session:

        while True:

            start = time.time()

            async with session.get(url=home_url, headers=headers) as response:

                html = await response.text()

                resp = Selector(text=html)

                items = resp.xpath('//ul[@class="rank-list"]/li')

                for item in items:

                    json_data = {}

                    number = item.xpath('.//div[@class="num"]/text()').extract_first()

                    info = item.xpath('.//div[@class="info"][1]')

                    title = info.xpath('.//a/text()').extract_first()



                    detail_url = info.xpath('.//a/@href').extract_first()

                    play_number = info.xpath('.//div[@class="detail"]/span[1]/text()').extract_first()

                    viewing_number = info.xpath('.//div[@class="detail"]/span[2]/text()').extract_first()



                    json_data['number'] = int(number)

                    json_data['title'] = title

                    json_data['play_number'] = convertor(play_number)

                    json_data['viewing_number'] = convertor(viewing_number)



                    json_data['url'] = detail_url



                    task = asyncio.create_task(detail_list(session, detail_url, json_data))

                    # await detail_url()

                end = time.time()

                print(f'time used {end-start}')

                await asyncio.sleep(SLEEP)  # 暂停10分钟

                print(f'sleep for {SLEEP}')





async def detail_list(session, url, json_data):

    async with session.get(url, headers=headers) as response:

        response = await response.text()

        await parse_detail(response, json_data)





async def parse_detail(html, json_data=None):

    resp = Selector(text=html)

    info = resp.xpath('//div[@id="v_desc"]/div[@class="info open"]/text()').extract_first()



    if not info:

        info = '这个家伙很懒'



    json_data['info'] = info.strip()

    current = datetime.datetime.now()

    json_data['crawltime'] = current

    await db['bilibili'].update_one({'url': json_data['url']}, {'$set': json_data}, True, True)





loop = asyncio.get_event_loop()

loop.run_until_complete(home_page())

m = re.match(p1, line)

if m:

return m.group(1)

else:

m = re.match(p2, line)

if m:

return m.group(2)

else:

m = re.match(p3, line)

    ...

if m := re.match(p1, line):

return m.group(1)

elif m := re.match(p2, line):

return m.group(2)

elif m := re.match(p3, line):

  File "C:\ProgramData\Anaconda3\lib\threading.py", line 926, in _bootstrap_inner

    self.run()

  File "C:\ProgramData\Anaconda3\lib\threading.py", line 870, in run

    self._target(*self._args, **self._kwargs)

  File "C:\ProgramData\Anaconda3\lib\subprocess.py", line 1238, in _readerthread

    buffer.append(fh.read())

UnicodeDecodeError: 'gbk' codec can't decode byte 0xbd in position 52: illegal multibyte sequence

  File "C:\ProgramData\Anaconda3\lib\site-packages\execjs\_external_runtime.py", line 103, in _exec_with_pipe

    stdoutdata, stderrdata = p.communicate(input=input)

  File "C:\ProgramData\Anaconda3\lib\subprocess.py", line 939, in communicate

    stdout, stderr = self._communicate(input, endtime, timeout)

  File "C:\ProgramData\Anaconda3\lib\subprocess.py", line 1288, in _communicate

    stdout = stdout[0]

IndexError: list index out of range

    def __init__(self, args, bufsize=-1, executable=None,

                 stdin=None, stdout=None, stderr=None,

                 preexec_fn=None, close_fds=True,

                 shell=False, cwd=None, env=None, universal_newlines=None,

                 startupinfo=None, creationflags=0,

                 restore_signals=True, start_new_session=False,

                 pass_fds=(), *, encoding="None", errors=None, text=None):

  log   [17:39:09.057] [warning][data][elasticsearch] No living connections

  log   [17:39:09.058] [warning][licensing][plugins] License information could not be obtained from Elasticsearch due to Error: No Living connections error

  log   [17:39:09.635] [warning][admin][elasticsearch] Unable to revive connection: http://127.0.0.1:9200/

  log   [17:39:09.636] [warning][admin][elasticsearch] No living connections

  log   [17:39:12.137] [warning][admin][elasticsearch] Unable to revive connection: http://127.0.0.1:9200/

  log   [17:39:12.138] [warning][admin][elasticsearch] No living connections

  log   [17:39:14.640] [warning][admin][elasticsearch] Unable to revive connection: http://127.0.0.1:9200/

  log   [17:39:14.640] [warning][admin][elasticsearch] No living connections

  log   [17:39:17.143] [warning][admin][elasticsearch] Unable to revive connection: http://127.0.0.1:9200/

  log   [17:39:17.143] [warning][admin][elasticsearch] No living connections

  log   [17:39:19.645] [warning][admin][elasticsearch] Unable to revive connection: http://127.0.0.1:9200/

In [1]: import smtplib                                                                                                                                                             

   ...:                                                                                                                                                                                    



In [5]:                                                                                                                                                                                    



In [5]: In [2]: obj = smtplib.SMTP()                                                                                                                                                       

   ...:                                                                                                                                                                                    



In [6]:                                                                                                                                                                                    



In [6]: In [3]: obj.connect('smtp.qiye.aliyun.com', 25)  

$alias python4=gcc python5=rustc python6=javac python7=node

git log --graph --pretty=oneline --abbrev-commit

# -*- coding: utf-8 -*-

 

"""

PyQt5 tutorial

 

In this example, we create a custom widget.

"""

import sys

from PyQt5.QtWidgets import (QWidget, QSlider, QApplication,

                             QHBoxLayout, QVBoxLayout)

from PyQt5.QtCore import QObject, Qt, pyqtSignal

from PyQt5.QtGui import QPainter, QFont, QColor, QPen

 

 

class Communicate(QObject):

    updateBW = pyqtSignal(int)

 

 

class BurningWidget(QWidget):

    def __init__(self):

        super().__init__()

 

        self.initUI()

 

    def initUI(self):

 

        self.setMinimumSize(1, 30)

        self.value = 75

        self.num = [75, 150, 225, 300, 375, 450, 525, 600, 675]

 

    def setValue(self, value):

 

        self.value = value

 

    def paintEvent(self, e):

 

        qp = QPainter()

        qp.begin(self)

        self.drawWidget(qp)

        qp.end()

 

    def drawWidget(self, qp):

 

        font = QFont('Serif', 7, QFont.Light)

        qp.setFont(font)

 

        size = self.size()

        w = size.width()

        h = size.height()

 

        step = int(round(w / 10.0))

 

        till = int(((w / 750.0) * self.value))

        full = int(((w / 750.0) * 700))

 

        if self.value >= 700:

 

            qp.setPen(QColor(255, 255, 255))

            qp.setBrush(QColor(255, 255, 184))

            qp.drawRect(0, 0, full, h)

            qp.setPen(QColor(255, 175, 175))

            qp.setBrush(QColor(255, 175, 175))

            qp.drawRect(full, 0, till - full, h)

 

        else:

 

            qp.setPen(QColor(255, 255, 255))

            qp.setBrush(QColor(255, 255, 184))

            qp.drawRect(0, 0, till, h)

 

        pen = QPen(QColor(20, 20, 20), 1,

                   Qt.SolidLine)

 

        qp.setPen(pen)

        qp.setBrush(Qt.NoBrush)

        qp.drawRect(0, 0, w - 1, h - 1)

 

        j = 0

 

        for i in range(step, 10 * step, step):

            qp.drawLine(i, 0, i, 5)

            metrics = qp.fontMetrics()

            fw = metrics.width(str(self.num[j]))

            qp.drawText(i - fw / 2, h / 2, str(self.num[j]))

            j = j + 1

 

 

class Example(QWidget):

    def __init__(self):

        super().__init__()

 

        self.initUI()

 

    def initUI(self):

        sld = QSlider(Qt.Horizontal, self)

        sld.setFocusPolicy(Qt.NoFocus)

        sld.setRange(1, 750)

        sld.setValue(75)

        sld.setGeometry(30, 40, 150, 30)

 

        self.c = Communicate()

        self.wid = BurningWidget()

        self.c.updateBW[int].connect(self.wid.setValue)

 

        sld.valueChanged[int].connect(self.changeValue)

        hbox = QHBoxLayout()

        hbox.addWidget(self.wid)

        vbox = QVBoxLayout()

        vbox.addStretch(1)

        vbox.addLayout(hbox)

        self.setLayout(vbox)

 

        self.setGeometry(300, 300, 390, 210)

        self.setWindowTitle('Burning widget')

        self.show()

 

    def changeValue(self, value):

        self.c.updateBW.emit(value)

        self.wid.repaint()

 

 

if __name__ == '__main__':

    app = QApplication(sys.argv)

    ex = Example()

    sys.exit(app.exec_())

class BurningWidget(QWidget):

  

    def __init__(self):      

        super().__init__() 

self.setMinimumSize(1, 30)

font = QFont('Serif', 7, QFont.Light)

qp.setFont(font)

size = self.size()

w = size.width()

h = size.height()

 

step = int(round(w / 10.0))

 

 

till = int(((w / 750.0) * self.value))

full = int(((w / 750.0) * 700))

metrics = qp.fontMetrics()

fw = metrics.width(str(self.num[j]))

qp.drawText(i-fw/2, h/2, str(self.num[j]))

def changeValue(self, value):

          

    self.c.updateBW.emit(value)        

    self.wid.repaint()

Set ws = CreateObject("Wscript.Shell") 

ws.run "cmd /c D:\新建\diff.bat",vbhide

WScript.sleep 1000

from PyQt5.QtCore import Qt,QRect