yield scrapy.Request(

                url=self.base_url.format(i),

                meta={'page':str(i),

                    'splash': {

                        'args': {

                            'images':0,

                            'wait': 15,

                            'proxy': self.get_proxy(),

                        },

                        'endpoint': 'render.html',

                        },

            },

            )

for i in doc.find({})

# -*-coding=utf-8-*-

import pandas as pd

import json

import pymongo

from sqlalchemy import create_engine



# 将mongo数据转移到mysql



client = pymongo.MongoClient('xxx')

doc = client['spider']['meituan']

engine = create_engine('mysql+pymysql://xxx:xxx@xxx:/xxx?charset=utf8')





def classic_method():

    temp = 

    start = 0

    # 数据太大还是会爆内存,或者游标丢失

    for i in doc.find().batch_size(500):

        start += 1

        del i['_id']

        temp.append(i)

        print(start)



    print('start to save to mysql')

    df = pd.read_json(json.dumps(temp))

    df = df.set_index('poiid', drop=True)

    df.to_sql('meituan', con=engine, if_exists='replace')

    print('done')





def chunksize_move():

    block = 10000

    total = doc.find({}).count()

    iter_number = total // block



    for i in range(iter_number + 1):

        small_part = doc.find({}).limit(block).skip(i * block)

        list_data = 



        for item in small_part:

            del item['_id']

            del item['crawl_time']

            item['poiid'] = int(item['poiid'])

            for k, v in item.items():

                if isinstance(v, dict) or isinstance(v, list):



                    item[k] = json.dumps(v, ensure_ascii=False)



            list_data.append(item)



        df = pd.DataFrame(list_data)

        df = df.set_index('poiid', drop=True)



        try:

            df.to_sql('meituan', con=engine, if_exists='append')

            print('to sql {}'.format(i))

        except Exception as e:

            print(e)



chunksize_move()

import pymongo

from setting import get_engine



# 将mongo数据转移到mysql



client = pymongo.MongoClient('10.18.6.101')

doc = client['spider']['meituan']

engine = create_engine('mysql+pymysql://localhost:1234@10.18.4.211/spider?charset=utf8')

temp=[]



for i in doc.find({}):

    del i['_id']

    temp.append(i)

print('start to save to mysql')

df = pd.read_json(json.dumps(temp))

df = df.set_index('poiid',drop=True)

df.to_sql('meituan',con=engine,if_exists='replace')

print('done')

python json.loads 文件中的字典不能用单引号

只能改成双引号,或者使用



with open('cookies', 'r') as f:

    # js = json.load(f)

    js=eval(f.read())

# cookie=js.get('Cookie','')

headers = js.get('headers', '')



#content为文件的内容

from scrapy import log

log.msg("This is a warning", level=log.WARING)

[py.warnings] WARNING: E:\git\CrawlMan\bilibili\bilibili\spiders\bili.py:14: ScrapyDeprecationWarning: log.msg has been deprecated, create a python logger and log through it instead

  log.msg

# -*- coding: utf-8 -*-

import scrapy

from scrapy_splash import SplashRequest

import logging

# from scrapy import log

class BiliSpider(scrapy.Spider):

    name = 'ordinary'  # 这个名字就是上面连接中那个启动应用的名字

    allowed_domain = ["bilibili.com"]

    start_urls = [

        "https://www.bilibili.com/"

    ]



    def parse(self, response):

        logging.info('====================================================')

        content = response.xpath("//div[@class='num-wrap']").extract_first()

        logging.info(content)

        logging.info('====================================================')

#!/bin/env python

# -*- coding: utf-8 -*-



from sys import platform

import json

import codecs



with codecs.open('data.json', 'r', encoding='utf8') as json_data:

    city = json.load(json_data)



def check_valid(idcard):

    # 城市编码, 出生日期, 归属地

    city_id = idcard[:6]

    print(city_id)

    birth = idcard[6:14]



    city_name = city.get(city_id,'Not found')



    # 根据规则校验身份证是否符合规则

    idcard_tuple = [int(num) for num in list(idcard[:-1])]

    coefficient = [7, 9, 10, 5, 8, 4, 2, 1, 6, 3, 7, 9, 10, 5, 8, 4, 2]

    sum_value = sum([idcard_tuple[i] * coefficient[i] for i in range(17)])



    remainder = sum_value % 11



    maptable = {0: '1', 1: '0', 2: 'x', 3: '9', 4: '8', 5: '7', 6: '6', 7: '5', 8: '4', 9: '3', 10: '2'}



    if maptable[remainder] == idcard[17]:

        print('<身份证合法>')

        sex = int(idcard[16]) % 2

        sex = '男' if sex == 1 else '女'

        print('性别：' + sex)

        birth_format="{}年{}月{}日".format(birth[:4],birth[4:6],birth[6:8])

        print('出生日期:' + birth_format)

        print('归属地:' + city_name)

        return True

    else:

        print('<身份证不合法>')

        return False





if __name__=='__main__':

    idcard = str(input('请输入身份证号码：'))

    check_valid(idcard)[/i]

Error: error: {

	"ok" : 0,

	"errmsg" : "Executor error during find command: OperationFailed: Sort operation used more than the maximum 33554432 bytes of RAM. Add an index, or specify a smaller limit.",

	"code" : 96,

	"codeName" : "OperationFailed"

}

db.getCollection('老布').find({}).sort({'created_at':-1}).limit(1000)

class CreditRecord(Base):

    __tablename__ = 'tb_PersonPunishment'



    id = Column(Integer, primary_key=True, autoincrement=True)

    name = Column(String(180),comment='名字')

all_stocks = DataAPI.SecTypeRegionRelGet(secID=u"",ticker=u"",typeID=u"",field=u"",pandas="1")

fbl = 



for code in all_stocks['secID']:

    df = DataAPI.MktEqudGet(secID=code,ticker=u"",tradeDate=u"",beginDate=u"20120101",endDate=u"",isOpen="",field=u"",pandas="1")

    df['ztj']=map(lambda x:round(x,2),df['preClosePrice']*1.1)

    df['chgPct']=df['chgPct']*100

    

    # 非一字板

    zt = df[(df['ztj']==df['highestPrice']) & (df['openPrice']!=df['highestPrice'])]

    fz= df[(df['ztj']==df['highestPrice']) & (df['openPrice']!=df['highestPrice'])&(df['closePrice']==df['highestPrice'])]

    try:

        f = len(fz)*1.00/len(zt)*100

        fbl.append((code,f))

    except Exception,e:

        print e

        print code

dx= dict(fbl)

x = np.array(dx.values())

print x.mean()

from abc import ABCMeta,abstractmethod



class Server(metaclass=ABCMeta):

	

	@abstractmethod

	def __init__(self):

		pass



	def __str__(self):

		return self.name



	@abstractmethod

	def boot(self):

		pass



	@abstractmethod

	def kill(self):

		pass

from abc import ABCMeta,abstractmethod



class Server(object):

	__metaclass__=ABCMeta

	@abstractmethod

	def __init__(self):

		pass



	def __str__(self):

		return self.name



	@abstractmethod

	def boot(self):

		pass



	@abstractmethod

	def kill(self):

		pass

sgdc = SGDClassifier()

sgdc.fit(X_train,y_train)

sgdc_predict_y = sgdc.predict(X_test)

print 'Accuary of SGD classifier ', sgdc.score(X_test,y_test)

print classification_report(y_test,sgdc_predict_y,target_names=['Benign','Malignant'])