## 通知设置新通知

#### scrapy rabbitmq 分布式爬虫

python爬虫李魔佛 发表了文章 • 0 个评论 • 489 次浏览 • 2019-07-17 16:59 • 来自相关话题

对于没接触过rabbitmq的同学，可以看这个文章：https://blog.csdn.net/hellozpc/article/details/81436980
rabbitmq是个不错的消息队列服务，可以配合scrapy作为消息队列.

下面是一个简单的demo：import re
import requests
import scrapy
from scrapy import Request
from rabbit_spider import settings
from scrapy.log import logger
import json
from rabbit_spider.items import RabbitSpiderItem
import datetime
from scrapy.selector import Selector
import pika

# from scrapy_rabbitmq.spiders import RabbitMQMixin
# from scrapy.contrib.spiders import CrawlSpider

class Website(scrapy.Spider):
name = "rabbit"

def start_requests(self):
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
'Host': '36kr.com',
'Referer': 'https://36kr.com/information/web_news',
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36'
}

url = 'https://36kr.com/information/web_news'

yield Request(url=url,

def parse(self, response):

connection = pika.BlockingConnection(pika.ConnectionParameters('192.168.1.101', 5672, '/', credentials))

channel = connection.channel()
channel.exchange_declare(exchange='direct_log', exchange_type='direct')

result = channel.queue_declare(exclusive=True, queue='')

queue_name = result.method.queue

# print(queue_name)
# infos = sys.argv[1:] if len(sys.argv)>1 else ['info']
info = 'info'

# 绑定多个值

channel.queue_bind(
exchange='direct_log',
routing_key=info,
queue=queue_name
)

channel.basic_consume(
on_message_callback=self.callback_func,
queue=queue_name,
auto_ack=True,
)

channel.start_consuming()

def callback_func(self, ch, method, properties, body):
print(body)
启动spider：from scrapy import cmdline
cmdline.execute('scrapy crawl rabbit'.split())
然后往rabbitmq里面推送数据：import pika
import settings

connection = pika.BlockingConnection(pika.ConnectionParameters('192.168.1.101',5672,'/',credentials))

channel = connection.channel()
channel.exchange_declare(exchange='direct_log',exchange_type='direct') # fanout 就是组播

routing_key = 'info'
message='https://36kr.com/pp/api/aggregation-entity?type=web_latest_article&b_id=59499&per_page=30'
channel.basic_publish(
exchange='direct_log',
routing_key=routing_key,
body=message
)

print('sending message {}'.format(message))
connection.close()

推送数据后，scrapy会马上接受到队里里面的数据。
注意不能在start_requests里面写等待队列的命令，因为start_requests函数需要返回一个生成器，否则程序会报错。

待续。。。
###### 2019-08-29 更新 ###################
发现一个坑，就是rabbitMQ在接受到数据后，无法在回调函数里面使用yield生成器。
查看全部
对于没接触过rabbitmq的同学，可以看这个文章：https://blog.csdn.net/hellozpc/article/details/81436980
rabbitmq是个不错的消息队列服务，可以配合scrapy作为消息队列.

下面是一个简单的demo：
import re
import requests
import scrapy
from scrapy import Request
from rabbit_spider import settings
from scrapy.log import logger
import json
from rabbit_spider.items import RabbitSpiderItem
import datetime
from scrapy.selector import Selector
import pika

# from scrapy_rabbitmq.spiders import RabbitMQMixin
# from scrapy.contrib.spiders import CrawlSpider

class Website(scrapy.Spider):
name = "rabbit"

def start_requests(self):
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
'Host': '36kr.com',
'Referer': 'https://36kr.com/information/web_news',
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36'
}

url = 'https://36kr.com/information/web_news'

yield Request(url=url,

def parse(self, response):

connection = pika.BlockingConnection(pika.ConnectionParameters('192.168.1.101', 5672, '/', credentials))

channel = connection.channel()
channel.exchange_declare(exchange='direct_log', exchange_type='direct')

result = channel.queue_declare(exclusive=True, queue='')

queue_name = result.method.queue

# print(queue_name)
# infos = sys.argv[1:] if len(sys.argv)>1 else ['info']
info = 'info'

# 绑定多个值

channel.queue_bind(
exchange='direct_log',
routing_key=info,
queue=queue_name
)

channel.basic_consume(
on_message_callback=self.callback_func,
queue=queue_name,
auto_ack=True,
)

channel.start_consuming()

def callback_func(self, ch, method, properties, body):
print(body)

启动spider：
from scrapy import cmdline
cmdline.execute('scrapy crawl rabbit'.split())

然后往rabbitmq里面推送数据：
import pika
import settings

connection = pika.BlockingConnection(pika.ConnectionParameters('192.168.1.101',5672,'/',credentials))

channel = connection.channel()
channel.exchange_declare(exchange='direct_log',exchange_type='direct') # fanout 就是组播

routing_key = 'info'
message='https://36kr.com/pp/api/aggregation-entity?type=web_latest_article&b_id=59499&per_page=30'
channel.basic_publish(
exchange='direct_log',
routing_key=routing_key,
body=message
)

print('sending message {}'.format(message))
connection.close()

推送数据后，scrapy会马上接受到队里里面的数据。
注意不能在start_requests里面写等待队列的命令，因为start_requests函数需要返回一个生成器，否则程序会报错。

待续。。。
###### 2019-08-29 更新 ###################
发现一个坑，就是rabbitMQ在接受到数据后，无法在回调函数里面使用yield生成器。

#### exchange_declare() got an unexpected keyword argument 'type'

python李魔佛 发表了文章 • 0 个评论 • 221 次浏览 • 2019-07-16 14:40 • 来自相关话题

In new version of pika, now it is using

connection = pika.BlockingConnection(pika.ConnectionParameters('192.168.1.101',5672,'/',credentials))

channel = connection.channel()

channel.exchange_declare(exchange='logs',exchange_type='fanout') 查看全部
In new version of pika, now it is using

	credentials = pika.PlainCredentials('admin','admin')
connection = pika.BlockingConnection(pika.ConnectionParameters('192.168.1.101',5672,'/',credentials))

channel = connection.channel()

channel.exchange_declare(exchange='logs',exchange_type='fanout')

#### twisted的getPage已经不建议使用，新接口为twisted.web.client.Agent

python爬虫李魔佛 发表了文章 • 0 个评论 • 307 次浏览 • 2019-07-12 11:31 • 来自相关话题

Twisted-16.7.0 is coming soon, and it deprecates twisted.web.client.getPage (and client.HTTPClientFactory). We use these in some of the unit tests, to fetch one of the HTTP WAPI/WUI pages and make sure the contents look right.

We need to change these tests to use twisted.web.client.Agent instead, or a package named "treq", which is a Twisted flavor of the excellent (but blocking) requests library.

查看全部

Twisted-16.7.0 is coming soon, and it deprecates twisted.web.client.getPage (and client.HTTPClientFactory). We use these in some of the unit tests, to fetch one of the HTTP WAPI/WUI pages and make sure the contents look right.

We need to change these tests to use twisted.web.client.Agent instead, or a package named "treq", which is a Twisted flavor of the excellent (but blocking) requests library.

python李魔佛 发表了文章 • 0 个评论 • 318 次浏览 • 2019-07-11 09:43 • 来自相关话题

代码如下：
from scrapy.selector import Selector

def get_response_callback(content):
txt = str(content,encoding='utf-8')
resp = Selector(text=txt)
title = resp.xpath('//title/text()').extract_first()
print(title)

@defer.inlineCallbacks
url = 'http://www.baidu.com'
d=getPage(url.encode('utf-8'))
yield d

def done():
reactor.stop()

def done1(*args,**kwargs):
reactor.stop()

for i in range(4):

reactor.run()
上面的代码是无法停止的，如果使用的是

done函数的定义是没有参数的。

而使用另一个done函数带参数的done(*args,**kwargs)
是可以正常退出的，done里面写了reactor.stop() 函数

原创文章
转载请注明出处：
http://30daydo.com/article/509
查看全部
代码如下：

	from scrapy.selector import Selector

def get_response_callback(content):
txt = str(content,encoding='utf-8')
resp = Selector(text=txt)
title = resp.xpath('//title/text()').extract_first()
print(title)

@defer.inlineCallbacks
url = 'http://www.baidu.com'
d=getPage(url.encode('utf-8'))
yield d

def done():
reactor.stop()

def done1(*args,**kwargs):
reactor.stop()

for i in range(4):

reactor.run()

上面的代码是无法停止的，如果使用的是

done函数的定义是没有参数的。

而使用另一个done函数带参数的done(*args,**kwargs)
是可以正常退出的，done里面写了reactor.stop() 函数

原创文章
转载请注明出处：
http://30daydo.com/article/509

#### numpy indices的用法

量化交易李魔佛 发表了文章 • 0 个评论 • 507 次浏览 • 2019-07-08 17:58 • 来自相关话题

Suppose you have a matrix M whose (i,j)-th element equals

M_ij = 2*i + 3*j
One way to define this matrix would be

i, j = np.indices((2,3))
M = 2*i + 3*j
which yields

array([[0, 3, 6],
[2, 5, 8]])
In other words, np.indices returns arrays which can be used as indices. The elements in i indicate the row index:

In [12]: i
Out[12]:
array([[0, 0, 0],
[1, 1, 1]])
The elements in j indicate the column index:

In [13]: j
Out[13]:
array([[0, 1, 2],
[0, 1, 2]])
上面是Stack Overflow的解释。 翻译一下：

np.indices((2,3))

返回的是一个行列的索引，然后可以用这个索引快速的创建二维数据。

比如我要画一个圆：
img = np.zeros((400,400))
ir,ic = np.indices(img.shape)
circle = (ir-135)**2+(ic-150)**2 < 30**2 # 半径30，圆心在135,150
img[circle]=1

img现在就是一个圆啦
查看全部
Suppose you have a matrix M whose (i,j)-th element equals

M_ij = 2*i + 3*j
One way to define this matrix would be

i, j = np.indices((2,3))
M = 2*i + 3*j
which yields

array([[0, 3, 6],
[2, 5, 8]])
In other words, np.indices returns arrays which can be used as indices. The elements in i indicate the row index:

In [12]: i
Out[12]:
array([[0, 0, 0],
[1, 1, 1]])
The elements in j indicate the column index:

In [13]: j
Out[13]:
array([[0, 1, 2],
[0, 1, 2]])

上面是Stack Overflow的解释。 翻译一下：

np.indices((2,3))

返回的是一个行列的索引，然后可以用这个索引快速的创建二维数据。

比如我要画一个圆：
img = np.zeros((400,400))
ir,ic = np.indices(img.shape)
circle = (ir-135)**2+(ic-150)**2 < 30**2 # 半径30，圆心在135,150
img[circle]=1

img现在就是一个圆啦

#### cv2 distanceTransform函数的用法 python

python李魔佛 发表了文章 • 0 个评论 • 985 次浏览 • 2019-07-08 15:35 • 来自相关话题

distanceTransform
Calculates the distance to the closest zero pixel for each pixel of the source image.

Python: cv2.distanceTransform(src, distanceType, maskSize[, dst]) → dst

Parameters:
src – 8-bit, single-channel (binary) source image.
dst – Output image with calculated distances. It is a 32-bit floating-point, single-channel image of the same size as src .
distanceType – Type of distance. It can be CV_DIST_L1, CV_DIST_L2 , or CV_DIST_C .
maskSize – Size of the distance transform mask. It can be 3, 5, or CV_DIST_MASK_PRECISE (the latter option is only supported by the first function). In case of the CV_DIST_L1 or CV_DIST_C distance type, the parameter is forced to 3 because a 3\times 3 mask gives the same result as 5\times 5 or any larger aperture.
labels – Optional output 2D array of labels (the discrete Voronoi diagram). It has the type CV_32SC1 and the same size as src . See the details below.
labelType – Type of the label array to build. If labelType==DIST_LABEL_CCOMP then each connected component of zeros in src (as well as all the non-zero pixels closest to the connected component) will be assigned the same label. If labelType==DIST_LABEL_PIXEL then each zero pixel (and all the non-zero pixels closest to it) gets its own label.
The functions distanceTransform calculate the approximate or precise distance from every binary image pixel to the nearest zero pixel. For zero image pixels, the distance will obviously be zero.

When maskSize == CV_DIST_MASK_PRECISE and distanceType == CV_DIST_L2 , the function runs the algorithm described in [Felzenszwalb04]. This algorithm is parallelized with the TBB library.

In other cases, the algorithm [Borgefors86] is used. This means that for a pixel the function finds the shortest path to the nearest zero pixel consisting of basic shifts: horizontal, vertical, diagonal, or knight’s move (the latest is available for a 5\times 5 mask). The overall distance is calculated as a sum of these basic distances. Since the distance function should be symmetric, all of the horizontal and vertical shifts must have the same cost (denoted as a ), all the diagonal shifts must have the same cost (denoted as b ), and all knight’s moves must have the same cost (denoted as c ). For the CV_DIST_C and CV_DIST_L1 types, the distance is calculated precisely, whereas for CV_DIST_L2 (Euclidean distance) the distance can be calculated only with a relative error (a 5\times 5 mask gives more accurate results). For a,b , and c , OpenCV uses the values suggested in the original paper:

CV_DIST_C (3\times 3) a = 1, b = 1
CV_DIST_L1 (3\times 3) a = 1, b = 2
CV_DIST_L2 (3\times 3) a=0.955, b=1.3693
CV_DIST_L2 (5\times 5) a=1, b=1.4, c=2.1969
Typically, for a fast, coarse distance estimation CV_DIST_L2, a 3\times 3 mask is used. For a more accurate distance estimation CV_DIST_L2 , a 5\times 5 mask or the precise algorithm is used. Note that both the precise and the approximate algorithms are linear on the number of pixels.

The second variant of the function does not only compute the minimum distance for each pixel (x, y) but also identifies the nearest connected component consisting of zero pixels (labelType==DIST_LABEL_CCOMP) or the nearest zero pixel (labelType==DIST_LABEL_PIXEL). Index of the component/pixel is stored in \texttt{labels}(x, y) . When labelType==DIST_LABEL_CCOMP, the function automatically finds connected components of zero pixels in the input image and marks them with distinct labels. When labelType==DIST_LABEL_CCOMP, the function scans through the input image and marks all the zero pixels with distinct labels.

In this mode, the complexity is still linear. That is, the function provides a very fast way to compute the Voronoi diagram for a binary image. Currently, the second variant can use only the approximate distance transform algorithm, i.e. maskSize=CV_DIST_MASK_PRECISE is not supported yet.

Note
An example on using the distance transform can be found at opencv_source_code/samples/cpp/distrans.cpp
(Python) An example on using the distance transform can be found at opencv_source/samples/python2/distrans.py

查看全部
distanceTransform
Calculates the distance to the closest zero pixel for each pixel of the source image.

Python: cv2.distanceTransform(src, distanceType, maskSize[, dst]) → dst

Python: cv.DistTransform(src, dst, distance_type=CV_DIST_L2, mask_size=3, mask=None, labels=None) → None

Parameters:
src – 8-bit, single-channel (binary) source image.
dst – Output image with calculated distances. It is a 32-bit floating-point, single-channel image of the same size as src .

distanceType – Type of distance. It can be CV_DIST_L1, CV_DIST_L2 , or CV_DIST_C .
maskSize – Size of the distance transform mask. It can be 3, 5, or CV_DIST_MASK_PRECISE (the latter option is only supported by the first function). In case of the CV_DIST_L1 or CV_DIST_C distance type, the parameter is forced to 3 because a  3\times 3 mask gives the same result as  5\times 5 or any larger aperture.

labels – Optional output 2D array of labels (the discrete Voronoi diagram). It has the type CV_32SC1 and the same size as src . See the details below.

labelType – Type of the label array to build. If labelType==DIST_LABEL_CCOMP then each connected component of zeros in src (as well as all the non-zero pixels closest to the connected component) will be assigned the same label. If labelType==DIST_LABEL_PIXEL then each zero pixel (and all the non-zero pixels closest to it) gets its own label.
The functions distanceTransform calculate the approximate or precise distance from every binary image pixel to the nearest zero pixel. For zero image pixels, the distance will obviously be zero.



When maskSize == CV_DIST_MASK_PRECISE and distanceType == CV_DIST_L2 , the function runs the algorithm described in [Felzenszwalb04]. This algorithm is parallelized with the TBB library.

In other cases, the algorithm [Borgefors86] is used. This means that for a pixel the function finds the shortest path to the nearest zero pixel consisting of basic shifts: horizontal, vertical, diagonal, or knight’s move (the latest is available for a 5\times 5 mask). The overall distance is calculated as a sum of these basic distances. Since the distance function should be symmetric, all of the horizontal and vertical shifts must have the same cost (denoted as a ), all the diagonal shifts must have the same cost (denoted as b ), and all knight’s moves must have the same cost (denoted as c ). For the CV_DIST_C and CV_DIST_L1 types, the distance is calculated precisely, whereas for CV_DIST_L2 (Euclidean distance) the distance can be calculated only with a relative error (a 5\times 5 mask gives more accurate results). For a,b , and c , OpenCV uses the values suggested in the original paper:

CV_DIST_C	(3\times 3)	a = 1, b = 1
CV_DIST_L1	(3\times 3)	a = 1, b = 2
CV_DIST_L2	(3\times 3)	a=0.955, b=1.3693
CV_DIST_L2	(5\times 5)	a=1, b=1.4, c=2.1969
Typically, for a fast, coarse distance estimation CV_DIST_L2, a 3\times 3 mask is used. For a more accurate distance estimation CV_DIST_L2 , a 5\times 5 mask or the precise algorithm is used. Note that both the precise and the approximate algorithms are linear on the number of pixels.

The second variant of the function does not only compute the minimum distance for each pixel (x, y) but also identifies the nearest connected component consisting of zero pixels (labelType==DIST_LABEL_CCOMP) or the nearest zero pixel (labelType==DIST_LABEL_PIXEL). Index of the component/pixel is stored in \texttt{labels}(x, y) . When labelType==DIST_LABEL_CCOMP, the function automatically finds connected components of zero pixels in the input image and marks them with distinct labels. When labelType==DIST_LABEL_CCOMP, the function scans through the input image and marks all the zero pixels with distinct labels.

In this mode, the complexity is still linear. That is, the function provides a very fast way to compute the Voronoi diagram for a binary image. Currently, the second variant can use only the approximate distance transform algorithm, i.e. maskSize=CV_DIST_MASK_PRECISE is not supported yet.

Note
An example on using the distance transform can be found at opencv_source_code/samples/cpp/distrans.cpp
(Python) An example on using the distance transform can be found at opencv_source/samples/python2/distrans.py

#### Django 版本不兼容报错 AuthenticationMiddleware

数据库李魔佛 发表了文章 • 0 个评论 • 840 次浏览 • 2019-07-04 15:43 • 来自相关话题

Django 2.2.ERRORS:
?: (admin.E408) 'django.contrib.auth.middleware.AuthenticationMiddleware' must be in MIDDLEWARE in order to use the admin application.
在之前的版本上没有问题，更新后就出错。
降级Django

pip install django=2.1.7

PS: 这个django的版本兼容的确是个大问题，哪天升级了下django版本，不经过严格的测试就带来灾难性的后果。 查看全部
Django 2.2.
ERRORS:
?: (admin.E408) 'django.contrib.auth.middleware.AuthenticationMiddleware' must be in MIDDLEWARE in order to use the admin application.

在之前的版本上没有问题，更新后就出错。
降级Django

pip install django=2.1.7

PS: 这个django的版本兼容的确是个大问题，哪天升级了下django版本，不经过严格的测试就带来灾难性的后果。

#### Win10下PhantomJS无法运行 【版本兼容问题】

python李魔佛 发表了文章 • 0 个评论 • 442 次浏览 • 2019-07-04 09:07 • 来自相关话题

以前在win7上运行的好好的。
在win10下就报错：
selenium.common.exceptions.WebDriverException: Message: Service C:\Tool\phantomjs-2.5.0-beta2-windows\phantomjs-2.5.0-beta2-windows\bin\phantomjs.exe unexpectedly exited. Status code was: 4294967295

后来替换了一个旧的版本，发现问题就这么解决了。
旧版本：phantomjs-2.1.1-windows

原创文章
转载请注明出处
http://30daydo.com/article/505
查看全部
以前在win7上运行的好好的。
在win10下就报错：
selenium.common.exceptions.WebDriverException: Message: Service C:\Tool\phantomjs-2.5.0-beta2-windows\phantomjs-2.5.0-beta2-windows\bin\phantomjs.exe unexpectedly exited. Status code was: 4294967295

后来替换了一个旧的版本，发现问题就这么解决了。
旧版本：phantomjs-2.1.1-windows

原创文章
转载请注明出处
http://30daydo.com/article/505

#### nunpy中的std标准差是样本差吗

量化交易李魔佛 发表了文章 • 0 个评论 • 282 次浏览 • 2019-07-01 10:08 • 来自相关话题

写个代码测试下：
# 测试一下那个方差
x=[1,2,3,4,5,6,7,8,9,10]
X = np.array(x)
X.mean()
5.5

X.std() # 标准差
2.8722813232690143

手工计算一下：
def my_fangca(X):
l=len(X)
mean=X.mean()
sum_ = 0
sum_std=0
for i in X:
sum_+=(i-mean)**2
var_=sum_/l
std_=(sum_/(l))**0.5
return var_,std_
result = my_fangca(X)
得到的result

(8.25, 2.8722813232690143)

说明numpy的std是标准差，不是样本差 查看全部
写个代码测试下：
# 测试一下那个方差
x=[1,2,3,4,5,6,7,8,9,10]
X = np.array(x)

X.mean()
5.5

X.std() # 标准差
2.8722813232690143

手工计算一下：
def my_fangca(X):
l=len(X)
mean=X.mean()
sum_ = 0
sum_std=0
for i in X:
sum_+=(i-mean)**2
var_=sum_/l
std_=(sum_/(l))**0.5
return var_,std_

result = my_fangca(X)
得到的result

(8.25, 2.8722813232690143)

说明numpy的std是标准差，不是样本差

#### 喜马拉雅app 爬取音频文件

python爬虫李魔佛 发表了文章 • 0 个评论 • 1208 次浏览 • 2019-06-30 12:24 • 来自相关话题

爬取喜马拉雅app上 杨继东的投资之道 的音频文件
运行环境：python3# -*- coding: utf-8 -*-
# website: http://30daydo.com
# @Time : 2019/6/30 12:03
# @File : main.py

import requests
import re
url = 'https://www.ximalaya.com/revision/play/album?albumId=23057324&pageNum=1&sort=1&pageSize=60'

js_data = r.json()
data_list = js_data.get('data',{}).get('tracksAudioPlay',)
for item in data_list:
trackName=item.get('trackName')
trackName=re.sub(':','',trackName)
src_url = item.get('src')
try:
except Exception as e:
print(e)
print(trackName)
else:
with open('{}.m4a'.format(trackName),'wb') as f:
f.write(r0.content)
保存为main.py
然后运行 python main.py
稍微等几分钟就自动下载好了。

附下载好的音频文件：
提取码：e3zb

原创文章
转载请注明出处
http://30daydo.com/article/503 查看全部
爬取喜马拉雅app上 杨继东的投资之道 的音频文件
运行环境：python3
# -*- coding: utf-8 -*-
# website: http://30daydo.com
# @Time : 2019/6/30 12:03
# @File : main.py

import requests
import re
url = 'https://www.ximalaya.com/revision/play/album?albumId=23057324&pageNum=1&sort=1&pageSize=60'

js_data = r.json()
data_list = js_data.get('data',{}).get('tracksAudioPlay',)
for item in data_list:
trackName=item.get('trackName')
trackName=re.sub(':','',trackName)
src_url = item.get('src')
try:
except Exception as e:
print(e)
print(trackName)
else:
with open('{}.m4a'.format(trackName),'wb') as f:
f.write(r0.content)
print('{} downloaded'.format(trackName))

保存为main.py
然后运行 python main.py
稍微等几分钟就自动下载好了。

附下载好的音频文件：
提取码：e3zb

原创文章
转载请注明出处
http://30daydo.com/article/503