db.test_tab.insert({array:[]})

db.test_tab.insert({array:[]})

db.test_tab.insert({array:[]})

db.test_tab.insert({array:[1,2,3,4,5]})

db.test_tab.insert({array:[1,2,3,4,5,6]})

db.getCollection("example").find({array:{$exists:true,$ne:[]}}); # 字段不为0

    from . import (constants, error, message, context,

ImportError: DLL load failed: 找不到指定的模块。

 # helper

    class RedisHelp(object):



        def __init__(self,channel):

            # self.pool = redis.ConnectionPool('10.18.6.46',port=6379)



            # self.conn = redis.Redis(connection_pool=self.pool)

            # 上面的方式无法使用订阅者 发布者模式



            self.conn = redis.Redis(host='10.18.6.46')

            self.publish_channel = channel

            self.subscribe_channel = channel





        def publish(self,msg):

            self.conn.publish(self.publish_channel,msg) # 1. 渠道名 ,2 信息



        def subscribe(self):

            self.pub = self.conn.pubsub()

            self.pub.subscribe(self.subscribe_channel)

            self.pub.parse_response()

            print('initial')

            return self.pub





    helper = RedisHelp('cuiqingcai')



    # 订阅者

    if sys.argv[1]=='s':

        print('in subscribe mode')

        pub = helper.subscribe()

        while 1:

            print('waiting for publish')

            pubsub.check_health()

            msg = pub.parse_response()



            s=str(msg[2],encoding='utf-8')

            print(s)

            if s=='exit':

                break





    # 发布者

    elif sys.argv[1]=='p':

        print('in publish mode')

        msg = sys.argv[2]

        print(f'msg -> {msg}')

        helper.publish(msg)

db.test.update({},{$rename:{'旧字段':'新字段'}},true,true)

db.getCollection('example').update({},{$rename:{'corp':'企业'}})

db.getCollection('example').update({},{$rename:{'corp.address':'corp.地址'}})

from random import randint



randint(0,1)

Out[25]: 1



randint(0,1)

Out[26]: 1



randint(0,1)

Out[27]: 1



randint(0,1)

Out[28]: 1



randint(0,1)

Out[29]: 0



randint(0,1)

Out[30]: 1

     file='test.txt'

     cmd = f'rsync -av  {file} root@10.18.6.46:/home/cjw/'



      p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,executable="/bin/bash")

      output, error = p.communicate()

      if p.returncode != 0:

          print("Error while running - %s" % cmd)

          print(error)

      print(output)

 p0 = multiprocessing.Process(name=str(0), target=foo, args=(0,))

    p0.start()

    p1 = multiprocessing.Process(name=str(1), target=foo, args=(1,))

    p1.start()

    p2 = multiprocessing.Process(name=str(2), target=foo, args=(2,))

    p2.start()

    p3 = multiprocessing.Process(name=str(3), target=foo, args=(3,))

    p3.start()

    p4 = multiprocessing.Process(name=str(4), target=foo, args=(4,))

    p4.start()



    p5 = multiprocessing.Process(name=str(5), target=foo, args=(5,))

    p5.start()

    

    p0.join()

    p1.join()

    p2.join()

    p3.join()

    p4.join()

    p5.join()

$ wget https://www.python.org/ftp/pyt ... 1.tgz 

$ tar zxvf Python-3.6.1.tgz $ cd Python-3.6.1

$ sudo ./configure && sudo make && sudo make install

Python 3.6.1 (default, Jul 21 2019, 14:26:28) 
[GCC 4.9.2] on linux
Type "help", "copyright", "credits" or "license" for more information.
 

from __future__ import print_function



import re



import requests



from frontera.contrib.requests.manager import RequestsFrontierManager

# from frontera.contrib.requests.manager import RequestsFrontierManager

from frontera import Settings



from six.moves.urllib.parse import urljoin





SETTINGS = Settings()

SETTINGS.BACKEND = 'frontera.contrib.backends.memory.FIFO'

# SETTINGS.BACKEND = 'frontera.contrib.backends.memory.MemoryDistributedBackend'



SETTINGS.LOGGING_MANAGER_ENABLED = True

SETTINGS.LOGGING_BACKEND_ENABLED = True

SETTINGS.MAX_REQUESTS = 100

SETTINGS.MAX_NEXT_REQUESTS = 10



SEEDS = [

    'http://www.imdb.com',

]



LINK_RE = re.compile(r'<a.+?href="(.*?)".?>', re.I)





def extract_page_links(response):

    return [urljoin(response.url, link) for link in LINK_RE.findall(response.text)]



if __name__ == '__main__':



    frontier = RequestsFrontierManager(SETTINGS)

    frontier.add_seeds([requests.Request(url=url) for url in SEEDS])

    while True:

        next_requests = frontier.get_next_requests()

        if not next_requests:

            break

        for request in next_requests:

                try:

                    response = requests.get(request.url)

                    links = [

                        requests.Request(url=url)

                        for url in extract_page_links(response)

                    ]

                    frontier.page_crawled(response)

                    print('Crawled', response.url, '(found', len(links), 'urls)')



                    if links:

                        frontier.links_extracted(request, links)

                except requests.RequestException as e:

                    error_code = type(e).__name__

                    frontier.request_error(request, error_code)

                    print('Failed to process request', request.url, 'Error:', e)

raise NameError("Module '%s' doesn't define any object named '%s'" % (module, name))

NameError: Module 'frontera.contrib.backends.memory' doesn't define any object named 'FIFO'

import re

import requests

import scrapy

from scrapy import Request

from rabbit_spider import settings

from scrapy.log import logger

import json

from rabbit_spider.items import RabbitSpiderItem

import datetime

from scrapy.selector import Selector

import pika



# from scrapy_rabbitmq.spiders import RabbitMQMixin

# from scrapy.contrib.spiders import CrawlSpider



class Website(scrapy.Spider):

    name = "rabbit"



    def start_requests(self):

        headers = {'Accept': '*/*',

                   'Accept-Encoding': 'gzip, deflate, br',

                   'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',

                   'Host': '36kr.com',

                   'Referer': 'https://36kr.com/information/web_news',

                   'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36'

                   }



        url = 'https://36kr.com/information/web_news'

        



        yield Request(url=url,

                      headers=headers)



    def parse(self, response):

       



        credentials = pika.PlainCredentials('admin', 'admin')

        connection = pika.BlockingConnection(pika.ConnectionParameters('192.168.1.101', 5672, '/', credentials))



        channel = connection.channel()

        channel.exchange_declare(exchange='direct_log', exchange_type='direct')



        result = channel.queue_declare(exclusive=True, queue='')



        queue_name = result.method.queue



        # print(queue_name)

        # infos = sys.argv[1:] if len(sys.argv)>1 else ['info']

        info = 'info'



        # 绑定多个值



        channel.queue_bind(

            exchange='direct_log',

            routing_key=info,

            queue=queue_name

        )

        print('start to receive [{}]'.format(info))



        channel.basic_consume(

            on_message_callback=self.callback_func,

            queue=queue_name,

            auto_ack=True,

        )



        channel.start_consuming()





    def callback_func(self, ch, method, properties, body):

        print(body)

from scrapy import cmdline

cmdline.execute('scrapy crawl rabbit'.split())

import pika

import settings



credentials = pika.PlainCredentials('admin','admin')

connection = pika.BlockingConnection(pika.ConnectionParameters('192.168.1.101',5672,'/',credentials))



channel = connection.channel()

channel.exchange_declare(exchange='direct_log',exchange_type='direct') # fanout 就是组播



routing_key = 'info'

message='https://36kr.com/pp/api/aggregation-entity?type=web_latest_article&b_id=59499&per_page=30'

channel.basic_publish(

	exchange='direct_log',

	routing_key=routing_key,

	body=message

	)



print('sending message {}'.format(message))

connection.close()

	credentials = pika.PlainCredentials('admin','admin')

	connection = pika.BlockingConnection(pika.ConnectionParameters('192.168.1.101',5672,'/',credentials))



	channel = connection.channel()



	channel.exchange_declare(exchange='logs',exchange_type='fanout')

Twisted-16.7.0 is coming soon, and it deprecates twisted.web.client.getPage (and client.HTTPClientFactory). We use these in some of the unit tests, to fetch one of the HTTP WAPI/WUI pages and make sure the contents look right.

We need to change these tests to use twisted.web.client.Agent instead, or a package named "treq", which is a Twisted flavor of the excellent (but blocking) requests library.

	from scrapy.selector import Selector

	

	def get_response_callback(content):

		txt = str(content,encoding='utf-8')

		resp = Selector(text=txt)

		title = resp.xpath('//title/text()').extract_first()

		print(title)

	

	@defer.inlineCallbacks

	def task():

		url = 'http://www.baidu.com'

		d=getPage(url.encode('utf-8'))

		d.addCallback(get_response_callback)

		yield d



	def done():

		reactor.stop()



	def done1(*args,**kwargs):

		reactor.stop()



	task_list = 

	for i in range(4):

		d=task()

		task_list.append(d)



	dd = defer.DeferredList(task_list)



	dd.addBoth(done)



	reactor.run()

Suppose you have a matrix M whose (i,j)-th element equals



M_ij = 2*i + 3*j

One way to define this matrix would be



i, j = np.indices((2,3))

M = 2*i + 3*j

which yields



array([[0, 3, 6],

       [2, 5, 8]])

In other words, np.indices returns arrays which can be used as indices. The elements in i indicate the row index:



In [12]: i

Out[12]: 

array([[0, 0, 0],

       [1, 1, 1]])

The elements in j indicate the column index:



In [13]: j

Out[13]: 

array([[0, 1, 2],

       [0, 1, 2]])

distanceTransform

Calculates the distance to the closest zero pixel for each pixel of the source image.





Python: cv2.distanceTransform(src, distanceType, maskSize[, dst]) → dst



Python: cv.DistTransform(src, dst, distance_type=CV_DIST_L2, mask_size=3, mask=None, labels=None) → None

Parameters:	

src – 8-bit, single-channel (binary) source image.

dst – Output image with calculated distances. It is a 32-bit floating-point, single-channel image of the same size as src .

distanceType – Type of distance. It can be CV_DIST_L1, CV_DIST_L2 , or CV_DIST_C .

maskSize – Size of the distance transform mask. It can be 3, 5, or CV_DIST_MASK_PRECISE (the latter option is only supported by the first function). In case of the CV_DIST_L1 or CV_DIST_C distance type, the parameter is forced to 3 because a  3\times 3 mask gives the same result as  5\times 5 or any larger aperture.

labels – Optional output 2D array of labels (the discrete Voronoi diagram). It has the type CV_32SC1 and the same size as src . See the details below.

labelType – Type of the label array to build. If labelType==DIST_LABEL_CCOMP then each connected component of zeros in src (as well as all the non-zero pixels closest to the connected component) will be assigned the same label. If labelType==DIST_LABEL_PIXEL then each zero pixel (and all the non-zero pixels closest to it) gets its own label.

The functions distanceTransform calculate the approximate or precise distance from every binary image pixel to the nearest zero pixel. For zero image pixels, the distance will obviously be zero.

When maskSize == CV_DIST_MASK_PRECISE and distanceType == CV_DIST_L2 , the function runs the algorithm described in [Felzenszwalb04]. This algorithm is parallelized with the TBB library.



In other cases, the algorithm [Borgefors86] is used. This means that for a pixel the function finds the shortest path to the nearest zero pixel consisting of basic shifts: horizontal, vertical, diagonal, or knight’s move (the latest is available for a 5\times 5 mask). The overall distance is calculated as a sum of these basic distances. Since the distance function should be symmetric, all of the horizontal and vertical shifts must have the same cost (denoted as a ), all the diagonal shifts must have the same cost (denoted as b ), and all knight’s moves must have the same cost (denoted as c ). For the CV_DIST_C and CV_DIST_L1 types, the distance is calculated precisely, whereas for CV_DIST_L2 (Euclidean distance) the distance can be calculated only with a relative error (a 5\times 5 mask gives more accurate results). For a,``b`` , and c , OpenCV uses the values suggested in the original paper:



CV_DIST_C	(3\times 3)	a = 1, b = 1

CV_DIST_L1	(3\times 3)	a = 1, b = 2

CV_DIST_L2	(3\times 3)	a=0.955, b=1.3693

CV_DIST_L2	(5\times 5)	a=1, b=1.4, c=2.1969

Typically, for a fast, coarse distance estimation CV_DIST_L2, a 3\times 3 mask is used. For a more accurate distance estimation CV_DIST_L2 , a 5\times 5 mask or the precise algorithm is used. Note that both the precise and the approximate algorithms are linear on the number of pixels.



The second variant of the function does not only compute the minimum distance for each pixel (x, y) but also identifies the nearest connected component consisting of zero pixels (labelType==DIST_LABEL_CCOMP) or the nearest zero pixel (labelType==DIST_LABEL_PIXEL). Index of the component/pixel is stored in \texttt{labels}(x, y) . When labelType==DIST_LABEL_CCOMP, the function automatically finds connected components of zero pixels in the input image and marks them with distinct labels. When labelType==DIST_LABEL_CCOMP, the function scans through the input image and marks all the zero pixels with distinct labels.



In this mode, the complexity is still linear. That is, the function provides a very fast way to compute the Voronoi diagram for a binary image. Currently, the second variant can use only the approximate distance transform algorithm, i.e. maskSize=CV_DIST_MASK_PRECISE is not supported yet.



Note

An example on using the distance transform can be found at opencv_source_code/samples/cpp/distrans.cpp

(Python) An example on using the distance transform can be found at opencv_source/samples/python2/distrans.py

ERRORS:

?: (admin.E408) 'django.contrib.auth.middleware.AuthenticationMiddleware' must be in MIDDLEWARE in order to use the admin application.

# 测试一下那个方差

x=[1,2,3,4,5,6,7,8,9,10]

X = np.array(x)

def my_fangca(X):

    l=len(X)

    mean=X.mean()

    sum_ = 0

    sum_std=0

    for i in X:

        sum_+=(i-mean)**2

    var_=sum_/l

    std_=(sum_/(l))**0.5

    return var_,std_

# -*- coding: utf-8 -*-

# website: http://30daydo.com

# @Time : 2019/6/30 12:03

# @File : main.py



import requests

import re

import os



url = 'http://180.153.255.6/mobile/v1/album/track/ts-1571294887744?albumId=23057324&device=android&isAsc=true&isQueryInvitationBrand=true&pageId={}&pageSize=20&pre_page=0'

headers = {'User-Agent': 'Xiaomi'}



def download():

    for i in range(1, 3):

        r = requests.get(url=url.format(i), headers=headers)

        js_data = r.json()

        data_list = js_data.get('data', {}).get('list', [])

        for item in data_list:

            trackName = item.get('title')

            trackName = re.sub('[\/\\\:\*\?\"\<\>\|]', '_', trackName)

            # trackName=re.sub(':','',trackName)

            src_url = item.get('playUrl64')

            filename = '{}.mp3'.format(trackName)

            if not os.path.exists(filename):



                try:

                    r0 = requests.get(src_url, headers=headers)

                except Exception as e:

                    print(e)

                    print(trackName)

                    r0 = requests.get(src_url, headers=headers)





                else:

                    with open(filename, 'wb') as f:

                        f.write(r0.content)



                    print('{} downloaded'.format(trackName))



            else:

                print(f'{filename}已经下载过了')



import shutil



def rename_():

    for i in range(1, 3):

        r = requests.get(url=url.format(i), headers=headers)

        js_data = r.json()

        data_list = js_data.get('data', {}).get('list', [])

        for item in data_list:

            trackName = item.get('title')

            trackName = re.sub('[\/\\\:\*\?\"\<\>\|]', '_', trackName)

            src_url = item.get('playUrl64')



            orderNo=item.get('orderNo')



            filename = '{}.mp3'.format(trackName)

            try:



                if os.path.exists(filename):

                    new_file='{}_{}.mp3'.format(orderNo,trackName)

                    shutil.move(filename,new_file)

            except Exception as e:

                print(e)











if __name__=='__main__':

    rename_()

# -*- coding: utf-8 -*-

# website: http://30daydo.com

# @Time : 2019/6/30 12:03

# @File : main.py



import requests

import re

url = 'https://www.ximalaya.com/revision/play/album?albumId=23057324&pageNum=1&sort=1&pageSize=60'

headers={'User-Agent':'Xiaomi'}



r = requests.get(url=url,headers=headers)

js_data = r.json()

data_list = js_data.get('data',{}).get('tracksAudioPlay',)

for item in data_list:

    trackName=item.get('trackName')

    trackName=re.sub(':','',trackName)

    src_url = item.get('src')

    try:

        r0=requests.get(src_url,headers=headers)

    except Exception as e:

        print(e)

        print(trackName)

    else:

        with open('{}.m4a'.format(trackName),'wb') as f:

            f.write(r0.content)

        print('{} downloaded'.format(trackName))

def iter_demo():



    class DefineIter(object):



        def __init__(self,length):

            self.length = length

            self.data = range(self.length)

            self.index=0



        def __iter__(self):

            return self





        def __next__(self):



            if self.index >=self.length:

                # return None

                raise StopIteration



            d = self.data[self.index]*50

            self.index =self.index + 1



            return d



    a = DefineIter(10)

    print(type(a))

    for i in a:

        print(i)

方式

通过 Live Template 快速添加时间



步骤

1、添加一个 Template Group 命名为 Common

2、添加一个 Live Template 设置如下

Abbreviation： time

Description ： current time

Template Text:  $time$



Edit Variables ->  Expresssion : date("yyyy-MM-dd HH:mm:ss")







3、让设置生效

Define->Everywhere



4、使用

输入 time 后 按下tab键 就能转换为当前时间了

pywinauto.application.AppStartError: Could not create the process "C:\全能行证券交易终端\xiadan.exe"
Error returned by CreateProcess: (2, 'CreateProcess', '系统找不到指定的文件。')

class GJ(CommonConfig):

    DEFAULT_EXE_PATH = "C:\\Tool\\xiadan.exe"

  File "C:\ProgramData\Anaconda3\lib\site-packages\pymongo\helpers.py", line 155, in _check_command_response

    raise OperationFailure(msg % errmsg, code, response)

pymongo.errors.OperationFailure: Query for sharded findAndModify must contain the shard key

2019-06-10 16:14:32 [scrapy.core.engine] INFO: Closing spider (finished)

2019-06-10 16:14:32 [scrapy.statscollectors] INFO: Dumping Scrapy stats:

Targeted Operations vs. Broadcast Operations

Generally, the fastest queries in a sharded environment are those that mongos route to a single shard, using the shard key and the cluster meta data from the config server. These targeted operations use the shard key value to locate the shard or subset of shards that satisfy the query document.

For queries that don’t include the shard key, mongos must query all shards, wait for their responses and then return the result to the application. These “scatter/gather” queries can be long running operations.

Broadcast Operations

mongos instances broadcast queries to all shards for the collection unless the mongos can determine which shard or subset of shards stores this data.



After the mongos receives responses from all shards, it merges the data and returns the result document. The performance of a broadcast operation depends on the overall load of the cluster, as well as variables like network latency, individual shard load, and number of documents returned per shard. Whenever possible, favor operations that result in targeted operation over those that result in a broadcast operation.

Multi-update operations are always broadcast operations.

The updateMany() and deleteMany() methods are broadcast operations, unless the query document specifies the shard key in full.

Targeted Operations

mongos can route queries that include the shard key or the prefix of a compound shard key a specific shard or set of shards. mongos uses the shard key value to locate the chunk whose range includes the shard key value and directs the query at the shard containing that chunk.



For example, if the shard key is:

copy

{ a: 1, b: 1, c: 1 }



The mongos program can route queries that include the full shard key or either of the following shard key prefixes at a specific shard or set of shards:

copy

{ a: 1 }

{ a: 1, b: 1 }



All insertOne() operations target to one shard. Each document in the insertMany() array targets to a single shard, but there is no guarantee all documents in the array insert into a single shard.

All updateOne(), replaceOne() and deleteOne() operations must include the shard key or _id in the query document. MongoDB returns an error if these methods are used without the shard key or _id.

Depending on the distribution of data in the cluster and the selectivity of the query, mongos may still perform a broadcast operation to fulfill these queries.

Index Use

If the query does not include the shard key, the mongos must send the query to all shards as a “scatter/gather” operation. Each shard will, in turn, use either the shard key index or another more efficient index to fulfill the query.

If the query includes multiple sub-expressions that reference the fields indexed by the shard key and the secondary index, the mongos can route the queries to a specific shard and the shard will use the index that will allow it to fulfill most efficiently.

Sharded Cluster Security

Use Internal Authentication to enforce intra-cluster security and prevent unauthorized cluster components from accessing the cluster. You must start each mongod or mongos in the cluster with the appropriate security settings in order to enforce internal authentication.

See Deploy Sharded Cluster with Keyfile Access Control for a tutorial on deploying a secured shardedcluster.

Cluster Users

Sharded clusters support Role-Based Access Control (RBAC) for restricting unauthorized access to cluster data and operations. You must start each mongod in the cluster, including the config servers, with the --auth option in order to enforce RBAC. Alternatively, enforcing Internal Authentication for inter-cluster security also enables user access controls via RBAC.

With RBAC enforced, clients must specify a --username, --password, and --authenticationDatabase when connecting to the mongos in order to access cluster resources.

Each cluster has its own cluster users. These users cannot be used to access individual shards.

See Enable Access Control for a tutorial on enabling adding users to an RBAC-enabled MongoDB deployment.