网上都是一个炒一个，没有通过验证的。
百度出来的csdn上的结果：https://blog.csdn.net/xudailong_blog/article/details/78313568
就是不对的，我把django降级到1.10，也是报错，明显不对嘛。
 
官方上说的1.8之后不建议使用，所以应该降级到1.8才可以。
 
降级命令：
pip install django==1.8
 
即可。
  查看全部

import sys
import cv2
from matplotlib import pyplot as plt
import matplotlib
%matplotlib inlineimg = cv2.imread('forest.jpg')
plt.imshow(img)效果如图：





  查看全部

import sys

import cv2

from matplotlib import pyplot as plt

import matplotlib

%matplotlib inline

img = cv2.imread('forest.jpg')

plt.imshow(img)

## 2019-03-19更新 问题已解决
 很无语。 目前还找不到原因。
 
docker里面运行的mongodb， mongodb的数据挂载到宿主机。 开放了27017端口。
在windows下使用mongoexport工具导出数据：
 
错误信息：C:\Program Files\MongoDB\Server\3.4\bin>mongoexport.exe /h 10.18.6.102 /d stock
/c company /o company.json /type json
2018-08-31T14:13:47.841+0800 connected to: 10.18.6.102
2018-08-31T14:13:47.854+0800 Failed: Failed to parse: { find: "company", filt
er: {}, sort: {}, skip: 0, snapshot: true, $readPreference: { mode: "secondaryPr
eferred" }, $db: "stock" }. Unrecognized field 'snapshot'.

C:\Program Files\MongoDB\Server\3.4\bin> 
目前这个问题已经解决：
需要进去docker容器里面，然后在容器里面操作，把数据导出来到挂载的目录下，然后可以直接获取到数据了。 查看全部

C:\Program Files\MongoDB\Server\3.4\bin>mongoexport.exe /h 10.18.6.102 /d stock

/c company /o company.json /type json

2018-08-31T14:13:47.841+0800    connected to: 10.18.6.102

2018-08-31T14:13:47.854+0800    Failed: Failed to parse: { find: "company", filt

er: {}, sort: {}, skip: 0, snapshot: true, $readPreference: { mode: "secondaryPr

eferred" }, $db: "stock" }. Unrecognized field 'snapshot'.



C:\Program Files\MongoDB\Server\3.4\bin>

对于新人来说太坑爹，不同版本，即使是一个小版本，很多函数都作了修改，或者直接被移除。好坑。
 
 

数据约为5GB左右,如果直接用for i in doc.find({})进行逐行遍历的话,游标就会超时,而且越到后面速度越慢.
 
 于是使用了分段遍历的方法.# -*-coding=utf-8-*-
import pandas as pd
import json
import pymongo
from sqlalchemy import create_engine

# 将mongo数据转移到mysql

client = pymongo.MongoClient('xxx')
doc = client['spider']['meituan']
engine = create_engine('mysql+pymysql://xxx:xxx@xxx:/xxx?charset=utf8')


def classic_method():
temp =
start = 0
# 数据太大还是会爆内存,或者游标丢失
for i in doc.find().batch_size(500):
start += 1
del i['_id']
temp.append(i)
print(start)

print('start to save to mysql')
df = pd.read_json(json.dumps(temp))
df = df.set_index('poiid', drop=True)
df.to_sql('meituan', con=engine, if_exists='replace')
print('done')


def chunksize_move():
block = 10000
total = doc.find({}).count()
iter_number = total // block

for i in range(iter_number + 1):
small_part = doc.find({}).limit(block).skip(i * block)
list_data =

for item in small_part:
del item['_id']
del item['crawl_time']
item['poiid'] = int(item['poiid'])
for k, v in item.items():
if isinstance(v, dict) or isinstance(v, list):

item[k] = json.dumps(v, ensure_ascii=False)

list_data.append(item)

df = pd.DataFrame(list_data)
df = df.set_index('poiid', drop=True)

try:
df.to_sql('meituan', con=engine, if_exists='append')
print('to sql {}'.format(i))
except Exception as e:
print(e)

chunksize_move()

 





速度比一次批量的要快不少. 查看全部

for i in doc.find({})

# -*-coding=utf-8-*-

import pandas as pd

import json

import pymongo

from sqlalchemy import create_engine



# 将mongo数据转移到mysql



client = pymongo.MongoClient('xxx')

doc = client['spider']['meituan']

engine = create_engine('mysql+pymysql://xxx:xxx@xxx:/xxx?charset=utf8')





def classic_method():

    temp = 

    start = 0

    # 数据太大还是会爆内存,或者游标丢失

    for i in doc.find().batch_size(500):

        start += 1

        del i['_id']

        temp.append(i)

        print(start)



    print('start to save to mysql')

    df = pd.read_json(json.dumps(temp))

    df = df.set_index('poiid', drop=True)

    df.to_sql('meituan', con=engine, if_exists='replace')

    print('done')





def chunksize_move():

    block = 10000

    total = doc.find({}).count()

    iter_number = total // block



    for i in range(iter_number + 1):

        small_part = doc.find({}).limit(block).skip(i * block)

        list_data = 



        for item in small_part:

            del item['_id']

            del item['crawl_time']

            item['poiid'] = int(item['poiid'])

            for k, v in item.items():

                if isinstance(v, dict) or isinstance(v, list):



                    item[k] = json.dumps(v, ensure_ascii=False)



            list_data.append(item)



        df = pd.DataFrame(list_data)

        df = df.set_index('poiid', drop=True)



        try:

            df.to_sql('meituan', con=engine, if_exists='append')

            print('to sql {}'.format(i))

        except Exception as e:

            print(e)



chunksize_move()

代码如下: 很简短.
import pymongo
from setting import get_engine

# 将mongo数据转移到mysql

client = pymongo.MongoClient('10.18.6.101')
doc = client['spider']['meituan']
engine = create_engine('mysql+pymysql://localhost:1234@10.18.4.211/spider?charset=utf8')
temp=[]

for i in doc.find({}):
del i['_id']
temp.append(i)
print('start to save to mysql')
df = pd.read_json(json.dumps(temp))
df = df.set_index('poiid',drop=True)
df.to_sql('meituan',con=engine,if_exists='replace')
print('done')





 
居然CPU飙到了90%
  查看全部

import pymongo

from setting import get_engine



# 将mongo数据转移到mysql



client = pymongo.MongoClient('10.18.6.101')

doc = client['spider']['meituan']

engine = create_engine('mysql+pymysql://localhost:1234@10.18.4.211/spider?charset=utf8')

temp=[]



for i in doc.find({}):

    del i['_id']

    temp.append(i)

print('start to save to mysql')

df = pd.read_json(json.dumps(temp))

df = df.set_index('poiid',drop=True)

df.to_sql('meituan',con=engine,if_exists='replace')

print('done')

python json.loads 文件中的字典不能用单引号
只能改成双引号,或者使用

with open('cookies', 'r') as f:
# js = json.load(f)
js=eval(f.read())
# cookie=js.get('Cookie','')
headers = js.get('headers', '')

#content为文件的内容 查看全部

python json.loads 文件中的字典不能用单引号

只能改成双引号,或者使用



with open('cookies', 'r') as f:

    # js = json.load(f)

    js=eval(f.read())

# cookie=js.get('Cookie','')

headers = js.get('headers', '')



#content为文件的内容

Introduction to Twisted Enterprise
Abstract

Twisted is an asynchronous networking framework, but most database API implementations unfortunately have blocking interfaces -- for this reason, twisted.enterprise.adbapi was created. It is a non-blocking interface to the standardized DB-API 2.0 API, which allows you to access a number of different RDBMSes.

What you should already know

Python :-)
How to write a simple Twisted Server (see this tutorial to learn how)
Familiarity with using database interfaces (see the documentation for DBAPI 2.0 or this article by Andrew Kuchling)

Quick Overview

Twisted is an asynchronous framework. This means standard database modules cannot be used directly, as they typically work something like:# Create connection... db = dbmodule.connect('mydb', 'andrew', 'password') # ...which blocks for an unknown amount of time # Create a cursor cursor = db.cursor() # Do a query... resultset = cursor.query('SELECT * FROM table WHERE ...') # ...which could take a long time, perhaps even minutes.Those delays are unacceptable when using an asynchronous framework such as Twisted. For this reason, twisted provides twisted.enterprise.adbapi, an asynchronous wrapper for any DB-API 2.0-compliant module. It is currently best tested with the pyPgSQL module for PostgreSQL.

enterprise.adbapi will do blocking database operations in seperate threads, which trigger callbacks in the originating thread when they complete. In the meantime, the original thread can continue doing normal work, like servicing other requests.

How do I use adbapi?

Rather than creating a database connection directly, use the adbapi.ConnectionPool class to manage a connections for you. This allows enterprise.adbapi to use multiple connections, one per thread. This is easy:# Using the "dbmodule" from the previous example, create a ConnectionPool from twisted.enterprise import adbapi dbpool = adbapi.ConnectionPool("dbmodule", 'mydb', 'andrew', 'password')Things to note about doing this:

There is no need to import dbmodule directly. You just pass the name to adbapi.ConnectionPool's constructor.
The parameters you would pass to dbmodule.connect are passed as extra arguments to adbapi.ConnectionPool's constructor. Keyword parameters work as well.
You may also control the size of the connection pool with the keyword parameters cp_min and cp_max. The default minimum and maximum values are 3 and 5.

So, now you need to be able to dispatch queries to your ConnectionPool. We do this by subclassing adbapi.Augmentation. Here's an example:class AgeDatabase(adbapi.Augmentation): """A simple example that can retrieve an age from the database""" def getAge(self, name): # Define the query sql = """SELECT Age FROM People WHERE name = ?""" # Run the query, and return a Deferred to the caller to add # callbacks to. return self.runQuery(sql, name) def gotAge(resultlist, name): """Callback for handling the result of the query""" age = resultlist[0][0] # First field of first record print "%s is %d years old" % (name, age) db = AgeDatabase(dbpool) # These will *not* block. Hooray! db.getAge("Andrew").addCallbacks(gotAge, db.operationError, callbackArgs=("Andrew",)) db.getAge("Glyph").addCallbacks(gotAge, db.operationError, callbackArgs=("Glyph",)) # Of course, nothing will happen until the reactor is started from twisted.internet import reactor reactor.run()This is straightforward, except perhaps for the return value of getAge. It returns a twisted.internet.defer.Deferred, which allows arbitrary callbacks to be called upon completion (or upon failure). More documentation on Deferred is available here.

Also worth noting is that this example assumes that dbmodule uses the qmarks paramstyle (see the DB-API specification). If your dbmodule uses a different paramstyle (e.g. pyformat) then use that. Twisted doesn't attempt to offer any sort of magic paramater munging -- runQuery(query, params, ...) maps directly onto cursor.execute(query, params, ...).

And that's it!

That's all you need to know to use a database from within Twisted. You probably should read the adbapi module's documentation to get an idea of the other functions it has, but hopefully this document presents the core ideas. 查看全部

输入身份证号码, 判断18位身份证号码是否合法, 并查询信息(性别, 年龄, 所在地)

验证原理

将前面的身份证号码17位数分别乘以不同的系数, 从第一位到第十七位的系数分别为: 7 9 10 5 8 4 2 1 6 3 7 9 10 5 8 4 2
将这17位数字和系数相乘的结果相加.
用加出来和除以11, 看余数是多少?
余数只可能有<0 1 2 3 4 5 6 7 8 9 10>这11个数字, 其分别对应的最后一位身份证的号码为<1 0 X 9 8 7 6 5 4 3 2>.
通过上面得知如果余数是2，就会在身份证的第18位数字上出现罗马数字的Ⅹ。如果余数是10，身份证的最后一位号码就是2.

例如: 某男性的身份证号码是34052419800101001X, 我们要看看这个身份证是不是合法的身份证.

首先: 我们得出, 前17位的乘积和是189.

然后: 用189除以11得出的余数是2.

最后: 通过对应规则就可以知道余数2对应的数字是x. 所以, 这是一个合格的身份证号码.
 
代码如下：#!/bin/env python
# -*- coding: utf-8 -*-

from sys import platform
import json
import codecs

with codecs.open('data.json', 'r', encoding='utf8') as json_data:
city = json.load(json_data)

def check_valid(idcard):
# 城市编码, 出生日期, 归属地
city_id = idcard[:6]
print(city_id)
birth = idcard[6:14]

city_name = city.get(city_id,'Not found')

# 根据规则校验身份证是否符合规则
idcard_tuple = [int(num) for num in list(idcard[:-1])]
coefficient = [7, 9, 10, 5, 8, 4, 2, 1, 6, 3, 7, 9, 10, 5, 8, 4, 2]
sum_value = sum([idcard_tuple[i] * coefficient[i] for i in range(17)])

remainder = sum_value % 11

maptable = {0: '1', 1: '0', 2: 'x', 3: '9', 4: '8', 5: '7', 6: '6', 7: '5', 8: '4', 9: '3', 10: '2'}

if maptable[remainder] == idcard[17]:
print('<身份证合法>')
sex = int(idcard[16]) % 2
sex = '男' if sex == 1 else '女'
print('性别：' + sex)
birth_format="{}年{}月{}日".format(birth[:4],birth[4:6],birth[6:8])
print('出生日期:' + birth_format)
print('归属地:' + city_name)
return True
else:
print('<身份证不合法>')
return False


if __name__=='__main__':
idcard = str(input('请输入身份证号码：'))
check_valid(idcard)[/i]

github源码：https://github.com/Rockyzsu/IdentityCheck
原创文章，转载请注明 
http://30daydo.com/article/340
  查看全部

#!/bin/env python

# -*- coding: utf-8 -*-



from sys import platform

import json

import codecs



with codecs.open('data.json', 'r', encoding='utf8') as json_data:

    city = json.load(json_data)



def check_valid(idcard):

    # 城市编码, 出生日期, 归属地

    city_id = idcard[:6]

    print(city_id)

    birth = idcard[6:14]



    city_name = city.get(city_id,'Not found')



    # 根据规则校验身份证是否符合规则

    idcard_tuple = [int(num) for num in list(idcard[:-1])]

    coefficient = [7, 9, 10, 5, 8, 4, 2, 1, 6, 3, 7, 9, 10, 5, 8, 4, 2]

    sum_value = sum([idcard_tuple[i] * coefficient[i] for i in range(17)])



    remainder = sum_value % 11



    maptable = {0: '1', 1: '0', 2: 'x', 3: '9', 4: '8', 5: '7', 6: '6', 7: '5', 8: '4', 9: '3', 10: '2'}



    if maptable[remainder] == idcard[17]:

        print('<身份证合法>')

        sex = int(idcard[16]) % 2

        sex = '男' if sex == 1 else '女'

        print('性别：' + sex)

        birth_format="{}年{}月{}日".format(birth[:4],birth[4:6],birth[6:8])

        print('出生日期:' + birth_format)

        print('归属地:' + city_name)

        return True

    else:

        print('<身份证不合法>')

        return False





if __name__=='__main__':

    idcard = str(input('请输入身份证号码：'))

    check_valid(idcard)[/i]

需要更新sqlalchemy到最新版本，旧版本会不支持。
 
在定义ORM对象的时候，
class CreditRecord(Base):
__tablename__ = 'tb_PersonPunishment'

id = Column(Integer, primary_key=True, autoincrement=True)
name = Column(String(180),comment='名字')
添加一个comment参数即可。
 
  查看全部

class CreditRecord(Base):

    __tablename__ = 'tb_PersonPunishment'



    id = Column(Integer, primary_key=True, autoincrement=True)

    name = Column(String(180),comment='名字')