numpy.asfarray(a, dtype=<class 'numpy.float64'>)



Return an array converted to a float type.



Parameters:	

a : array_like

The input array.



dtype : str or dtype object, optional

Float type code to coerce input array a. If dtype is one of the ‘int’ dtypes, it is replaced with float64.



Returns:	

out : ndarray

The input a as a float ndarray.

Examples



>>>

>>> np.asfarray([2, 3])

array([ 2.,  3.])

>>> np.asfarray([2, 3], dtype='float')

array([ 2.,  3.])

>>> np.asfarray([2, 3], dtype='int8')

array([ 2.,  3.])

import sys

import cv2

from matplotlib import pyplot as plt

import matplotlib

%matplotlib inline

img = cv2.imread('forest.jpg')

plt.imshow(img)

# -*- coding: utf-8 -*-

import re

import scrapy

from scrapy import Request, FormRequest

from jsl.items import JslItem

from jsl import config

import logging



class AllcontentSpider(scrapy.Spider):

    name = 'allcontent'



    headers = {

        'Host': 'www.jisilu.cn', 'Connection': 'keep-alive', 'Pragma': 'no-cache',

        'Cache-Control': 'no-cache', 'Accept': 'application/json,text/javascript,*/*;q=0.01',

        'Origin': 'https://www.jisilu.cn', 'X-Requested-With': 'XMLHttpRequest',

        'User-Agent': 'Mozilla/5.0(WindowsNT6.1;WOW64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/67.0.3396.99Safari/537.36',

        'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8',

        'Referer': 'https://www.jisilu.cn/login/',

        'Accept-Encoding': 'gzip,deflate,br',

        'Accept-Language': 'zh,en;q=0.9,en-US;q=0.8'

    }



    def start_requests(self):

        login_url = 'https://www.jisilu.cn/login/'

        headers = {

            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',

            'Accept-Encoding': 'gzip,deflate,br', 'Accept-Language': 'zh,en;q=0.9,en-US;q=0.8',

            'Cache-Control': 'no-cache', 'Connection': 'keep-alive',

            'Host': 'www.jisilu.cn', 'Pragma': 'no-cache', 'Referer': 'https://www.jisilu.cn/',

            'Upgrade-Insecure-Requests': '1',

            'User-Agent': 'Mozilla/5.0(WindowsNT6.1;WOW64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/67.0.3396.99Safari/537.36'}



        yield Request(url=login_url, headers=headers, callback=self.login,dont_filter=True)



    def login(self, response):

        url = 'https://www.jisilu.cn/account/ajax/login_process/'

        data = {

            'return_url': 'https://www.jisilu.cn/',

            'user_name': config.username,

            'password': config.password,

            'net_auto_login': '1',

            '_post_type': 'ajax',

        }



        yield FormRequest(

            url=url,

            headers=self.headers,

            formdata=data,

            callback=self.parse,

            dont_filter=True

        )



    def parse(self, response):

        for i in range(1,3726):

            focus_url = 'https://www.jisilu.cn/home/explore/sort_type-new__day-0__page-{}'.format(i)

            yield Request(url=focus_url, headers=self.headers, callback=self.parse_page,dont_filter=True)



    def parse_page(self, response):

        nodes = response.xpath('//div[@class="aw-question-list"]/div')

        for node in nodes:

            each_url=node.xpath('.//h4/a/@href').extract_first()

            yield Request(url=each_url,headers=self.headers,callback=self.parse_item,dont_filter=True)



    def parse_item(self,response):

        item = JslItem()

        title = response.xpath('//div[@class="aw-mod-head"]/h1/text()').extract_first()

        s = response.xpath('//div[@class="aw-question-detail-txt markitup-box"]').xpath('string(.)').extract_first()

        ret = re.findall('(.*?)\.donate_user_avatar', s, re.S)



        try:

            content = ret[0].strip()

        except:

            content = None



        createTime = response.xpath('//div[@class="aw-question-detail-meta"]/span/text()').extract_first()



        resp_no = response.xpath('//div[@class="aw-mod aw-question-detail-box"]//ul/h2/text()').re_first('\d+')



        url = response.url

        item['title'] = title.strip()

        item['content'] = content

        try:

            item['resp_no']=int(resp_no)

        except Exception as e:

            logging.warning('e')

            item['resp_no']=None



        item['createTime'] = createTime

        item['url'] = url.strip()

        resp = 

        for index,reply in enumerate(response.xpath('//div[@class="aw-mod-body aw-dynamic-topic"]/div[@class="aw-item"]')):

            replay_user = reply.xpath('.//div[@class="pull-left aw-dynamic-topic-content"]//p/a/text()').extract_first()

            rep_content = reply.xpath(

                './/div[@class="pull-left aw-dynamic-topic-content"]//div[@class="markitup-box"]/text()').extract_first()

            # print rep_content

            agree=reply.xpath('.//em[@class="aw-border-radius-5 aw-vote-count pull-left"]/text()').extract_first()

            resp.append({replay_user.strip()+'_{}'.format(index): [int(agree),rep_content.strip()]})



        item['resp'] = resp

        yield item

import pymongo

from collections import OrderedDict

class JslPipeline(object):

    def __init__(self):

        self.db = pymongo.MongoClient(host='10.18.6.1',port=27017)

        # self.user = u'neo牛3' # 修改为指定的用户名 如 毛之川 ，然后找到用户的id，在用户也的源码哪里可以找到 比如持有封基是8132

        self.collection = self.db['db_parker']['jsl']

    def process_item(self, item, spider):

        self.collection.insert(OrderedDict(item))

        return item

C:\Program Files\MongoDB\Server\3.4\bin>mongoexport.exe /h 10.18.6.102 /d stock

/c company /o company.json /type json

2018-08-31T14:13:47.841+0800    connected to: 10.18.6.102

2018-08-31T14:13:47.854+0800    Failed: Failed to parse: { find: "company", filt

er: {}, sort: {}, skip: 0, snapshot: true, $readPreference: { mode: "secondaryPr

eferred" }, $db: "stock" }. Unrecognized field 'snapshot'.



C:\Program Files\MongoDB\Server\3.4\bin>

yield scrapy.Request(

                url=self.base_url.format(i),

                meta={'page':str(i),

                    'splash': {

                        'args': {

                            'images':0,

                            'wait': 15,

                            'proxy': self.get_proxy(),

                        },

                        'endpoint': 'render.html',

                        },

            },

            )

for i in doc.find({})

# -*-coding=utf-8-*-

import pandas as pd

import json

import pymongo

from sqlalchemy import create_engine



# 将mongo数据转移到mysql



client = pymongo.MongoClient('xxx')

doc = client['spider']['meituan']

engine = create_engine('mysql+pymysql://xxx:xxx@xxx:/xxx?charset=utf8')





def classic_method():

    temp = 

    start = 0

    # 数据太大还是会爆内存,或者游标丢失

    for i in doc.find().batch_size(500):

        start += 1

        del i['_id']

        temp.append(i)

        print(start)



    print('start to save to mysql')

    df = pd.read_json(json.dumps(temp))

    df = df.set_index('poiid', drop=True)

    df.to_sql('meituan', con=engine, if_exists='replace')

    print('done')





def chunksize_move():

    block = 10000

    total = doc.find({}).count()

    iter_number = total // block



    for i in range(iter_number + 1):

        small_part = doc.find({}).limit(block).skip(i * block)

        list_data = 



        for item in small_part:

            del item['_id']

            del item['crawl_time']

            item['poiid'] = int(item['poiid'])

            for k, v in item.items():

                if isinstance(v, dict) or isinstance(v, list):



                    item[k] = json.dumps(v, ensure_ascii=False)



            list_data.append(item)



        df = pd.DataFrame(list_data)

        df = df.set_index('poiid', drop=True)



        try:

            df.to_sql('meituan', con=engine, if_exists='append')

            print('to sql {}'.format(i))

        except Exception as e:

            print(e)



chunksize_move()

import pymongo

from setting import get_engine



# 将mongo数据转移到mysql



client = pymongo.MongoClient('10.18.6.101')

doc = client['spider']['meituan']

engine = create_engine('mysql+pymysql://localhost:1234@10.18.4.211/spider?charset=utf8')

temp=[]



for i in doc.find({}):

    del i['_id']

    temp.append(i)

print('start to save to mysql')

df = pd.read_json(json.dumps(temp))

df = df.set_index('poiid',drop=True)

df.to_sql('meituan',con=engine,if_exists='replace')

print('done')

python json.loads 文件中的字典不能用单引号

只能改成双引号,或者使用



with open('cookies', 'r') as f:

    # js = json.load(f)

    js=eval(f.read())

# cookie=js.get('Cookie','')

headers = js.get('headers', '')



#content为文件的内容

from scrapy import log

log.msg("This is a warning", level=log.WARING)

[py.warnings] WARNING: E:\git\CrawlMan\bilibili\bilibili\spiders\bili.py:14: ScrapyDeprecationWarning: log.msg has been deprecated, create a python logger and log through it instead

  log.msg

# -*- coding: utf-8 -*-

import scrapy

from scrapy_splash import SplashRequest

import logging

# from scrapy import log

class BiliSpider(scrapy.Spider):

    name = 'ordinary'  # 这个名字就是上面连接中那个启动应用的名字

    allowed_domain = ["bilibili.com"]

    start_urls = [

        "https://www.bilibili.com/"

    ]



    def parse(self, response):

        logging.info('====================================================')

        content = response.xpath("//div[@class='num-wrap']").extract_first()

        logging.info(content)

        logging.info('====================================================')

#!/bin/env python

# -*- coding: utf-8 -*-



from sys import platform

import json

import codecs



with codecs.open('data.json', 'r', encoding='utf8') as json_data:

    city = json.load(json_data)



def check_valid(idcard):

    # 城市编码, 出生日期, 归属地

    city_id = idcard[:6]

    print(city_id)

    birth = idcard[6:14]



    city_name = city.get(city_id,'Not found')



    # 根据规则校验身份证是否符合规则

    idcard_tuple = [int(num) for num in list(idcard[:-1])]

    coefficient = [7, 9, 10, 5, 8, 4, 2, 1, 6, 3, 7, 9, 10, 5, 8, 4, 2]

    sum_value = sum([idcard_tuple[i] * coefficient[i] for i in range(17)])



    remainder = sum_value % 11



    maptable = {0: '1', 1: '0', 2: 'x', 3: '9', 4: '8', 5: '7', 6: '6', 7: '5', 8: '4', 9: '3', 10: '2'}



    if maptable[remainder] == idcard[17]:

        print('<身份证合法>')

        sex = int(idcard[16]) % 2

        sex = '男' if sex == 1 else '女'

        print('性别：' + sex)

        birth_format="{}年{}月{}日".format(birth[:4],birth[4:6],birth[6:8])

        print('出生日期:' + birth_format)

        print('归属地:' + city_name)

        return True

    else:

        print('<身份证不合法>')

        return False





if __name__=='__main__':

    idcard = str(input('请输入身份证号码：'))

    check_valid(idcard)[/i]

Error: error: {

	"ok" : 0,

	"errmsg" : "Executor error during find command: OperationFailed: Sort operation used more than the maximum 33554432 bytes of RAM. Add an index, or specify a smaller limit.",

	"code" : 96,

	"codeName" : "OperationFailed"

}

db.getCollection('老布').find({}).sort({'created_at':-1}).limit(1000)

class CreditRecord(Base):

    __tablename__ = 'tb_PersonPunishment'



    id = Column(Integer, primary_key=True, autoincrement=True)

    name = Column(String(180),comment='名字')

all_stocks = DataAPI.SecTypeRegionRelGet(secID=u"",ticker=u"",typeID=u"",field=u"",pandas="1")

fbl = 



for code in all_stocks['secID']:

    df = DataAPI.MktEqudGet(secID=code,ticker=u"",tradeDate=u"",beginDate=u"20120101",endDate=u"",isOpen="",field=u"",pandas="1")

    df['ztj']=map(lambda x:round(x,2),df['preClosePrice']*1.1)

    df['chgPct']=df['chgPct']*100

    

    # 非一字板

    zt = df[(df['ztj']==df['highestPrice']) & (df['openPrice']!=df['highestPrice'])]

    fz= df[(df['ztj']==df['highestPrice']) & (df['openPrice']!=df['highestPrice'])&(df['closePrice']==df['highestPrice'])]

    try:

        f = len(fz)*1.00/len(zt)*100

        fbl.append((code,f))

    except Exception,e:

        print e

        print code

dx= dict(fbl)

x = np.array(dx.values())

print x.mean()

from abc import ABCMeta,abstractmethod



class Server(metaclass=ABCMeta):

	

	@abstractmethod

	def __init__(self):

		pass



	def __str__(self):

		return self.name



	@abstractmethod

	def boot(self):

		pass



	@abstractmethod

	def kill(self):

		pass

from abc import ABCMeta,abstractmethod



class Server(object):

	__metaclass__=ABCMeta

	@abstractmethod

	def __init__(self):

		pass



	def __str__(self):

		return self.name



	@abstractmethod

	def boot(self):

		pass



	@abstractmethod

	def kill(self):

		pass