import scipy.misc

	lena=scipy.misc.lena()

	plt.gray()

	plt.imshow(lena)

	plt.colorbar()

	plt.show()

def profit(code,start,end):

    try:

        df=ts.get_k_data(code,start=start,end=end)

    except Exception,e:

        print e

        return None

    try:

        p=(df['close'].iloc[-1]-df['close'].iloc[0])/df['close'].iloc[0]*100.00

    except Exception,e:

        print e

        return None 

    return round(p,2)

profit('300678','2016-12-31','2017-12-31')

def price_change():

    basic=ts.get_stock_basics()

    pro=



    for code in basic.index.values:

        print code

        p=profit(code,'2016-12-31','2017-12-31')

        pro.append(p)

    basic['price_change']=pro

    basic.to_csv('2017_all_price_change.xls',encoding='gbk')

    df=pd.read_csv('2017_all_price_change.xls',encoding='gbk')

    df.to_excel('2017_all_price_change.xls',encoding='gbk')

def analysis():

    df=pd.read_excel('2017_all_price_change.xls',encoding='gbk')

    df=df.sort_values(by='price_change',ascending=False)

    df.to_excel('2017-year.xls',encoding='gbk')

engine=get_engine('stock')

df.to_sql('2017years',engine)

def get_engine(db):

    engine = create_engine('mysql+pymysql://{}:{}@{}:{}/{}?charset=utf8'.format(MYSQL_USER, MYSQL_PASSWORD, MYSQL_HOST, MYSQL_PORT, db))

    return engine

• 2017-year.zip

    df = pd.DataFrame({'A': [1, 2, 3, 4, 5], 'B': [10, 20, 30, 40, 50]})

    df2 = pd.DataFrame({'A': [6], 'B': [60]})

    print('df', df)

    print('df2', df2)



    df_x = [df, df2]

    result = pd.concat(df_x)

    print('first result ', result)

result2= result.reset_index()

result2 = result.reset_index(drop=True)

result3 = result.reindex(columns=['A','C'])

    df = pd.DataFrame({'A': [1, 2, 3, 4, 5], 'B': [10, 20, 30, 40, 50]})

    df2 = pd.DataFrame({'A': [6], 'B': [60]})

    print 'df\n', df

    print 'df2\n', df2

    df_x = [df, df2]

    result = pd.concat(df_x)

    print 'first result\n', result

    basic=ts.get_stock_basics()

    

    print basic.info()

    print basic.head(10)



    basic.to_excel('basic2017.xls',encoding='gbk')

    basic.to_csv('2017_all_price_change.xls',encoding='gbk')

    df=pd.read_csv('2017_all_price_change.xls',encoding='gbk')

    df.to_excel('2017_all_price_change.xls',encoding='gbk')

	df=pd.read_excel('600050.xls',dtype={'datetime':np.datetime64})

	df=df.set_index('datetime')

	df['close'].plot(use_index=True,grid=True)

	plt.show()

Server has startup warnings: 

2017-12-17T22:56:19.702+0800 I STORAGE  [initandlisten] 

2017-12-17T22:56:19.702+0800 I STORAGE  [initandlisten] ** WARNING: Using the XFS filesystem is strongly recommended with the WiredTiger storage engine

2017-12-17T22:56:19.702+0800 I STORAGE  [initandlisten] **          See http://dochub.mongodb.org/core ... ystem

2017-12-17T22:56:20.600+0800 I CONTROL  [initandlisten] 

2017-12-17T22:56:20.600+0800 I CONTROL  [initandlisten] ** WARNING: Access control is not enabled for the database.

2017-12-17T22:56:20.600+0800 I CONTROL  [initandlisten] **          Read and write access to data and configuration is unrestricted.

2017-12-17T22:56:20.600+0800 I CONTROL  [initandlisten] 

2017-12-17T22:56:20.600+0800 I CONTROL  [initandlisten] ** WARNING: This server is bound to localhost.

2017-12-17T22:56:20.600+0800 I CONTROL  [initandlisten] **          Remote systems will be unable to connect to this server. 

2017-12-17T22:56:20.600+0800 I CONTROL  [initandlisten] **          Start the server with --bind_ip <address> to specify which IP 

2017-12-17T22:56:20.600+0800 I CONTROL  [initandlisten] **          addresses it should serve responses from, or with --bind_ip_all to

2017-12-17T22:56:20.600+0800 I CONTROL  [initandlisten] **          bind to all interfaces. If this behavior is desired, start the

2017-12-17T22:56:20.600+0800 I CONTROL  [initandlisten] **          server with --bind_ip 127.0.0.1 to disable this warning.

2017-12-17T22:56:20.600+0800 I CONTROL  [initandlisten] 

2017-12-17T22:56:20.601+0800 I CONTROL  [initandlisten] 

2017-12-17T22:56:20.601+0800 I CONTROL  [initandlisten] ** WARNING: /sys/kernel/mm/transparent_hugepage/defrag is 'always'.

2017-12-17T22:56:20.601+0800 I CONTROL  [initandlisten] **        We suggest setting it to 'never'

2017-12-17T22:56:20.601+0800 I CONTROL  [initandlisten] 

scrapy startproject headerchange

scrapy genspider headervalidation helloacm.com

class HeadervalidationSpider(scrapy.Spider):

    name = 'headervalidation'

    allowed_domains = ['helloacm.com']

    start_urls = ['http://helloacm.com/api/user-agent/']



    def parse(self, response):

        print '*'*20

        print response.body

        print '*'*20

scrapy crawl headervalidation

# Crawl responsibly by identifying yourself (and your website) on the user-agent

USER_AGENT = 'Hello World'

scrapy crawl headervalidation

2017-12-14 16:17:35 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023

2017-12-14 16:17:35 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://helloacm.com/api/us

er-agent/> from <GET http://helloacm.com/api/user-agent/>

2017-12-14 16:17:36 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://helloacm.com/api/user-agent/> (referer: None)



[b]********************

"Hello World"

********************

[/b]2017-12-14 16:17:37 [scrapy.core.engine] INFO: Closing spider (finished)

2017-12-14 16:17:37 [scrapy.statscollectors] INFO: Dumping Scrapy stats:

{'downloader/request_bytes': 406,

 'downloader/request_count': 2,

 'downloader/request_method_count/GET': 2,

 'downloader/response_bytes': 796,

 'downloader/response_count': 2,

 'downloader/response_status_count/200': 1,

 'downloader/response_status_count/301': 1,

 'finish_reason': 'finished',

 'finish_time': datetime.datetime(2017, 12, 14, 8, 17, 37, 29000),

 'log_count/DEBUG': 3,

 'log_count/INFO': 7,

 'response_received_count': 1,

 'scheduler/dequeued': 2,

 'scheduler/dequeued/memory': 2,

 'scheduler/enqueued': 2,

 'scheduler/enqueued/memory': 2,

 'start_time': datetime.datetime(2017, 12, 14, 8, 17, 35, 137000)}

2017-12-14 16:17:37 [scrapy.core.engine] INFO: Spider closed (finished)

# Override the default request headers:

 DEFAULT_REQUEST_HEADERS = {

    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',

    'Accept-Language': 'en',

    'User-Agent':'Hello World'

 }

class HeadervalidationSpider(scrapy.Spider):

    name = 'headervalidation'

    allowed_domains = ['helloacm.com']





    def start_requests(self):

        header={'User-Agent':'Hello World'}

        yield scrapy.Request(url='http://helloacm.com/api/user-agent/',headers=header)



    def parse(self, response):

        print '*'*20

        print response.body

        print '*'*20

2017-12-14 16:17:35 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023

2017-12-14 16:17:35 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://helloacm.com/api/us

er-agent/> from <GET http://helloacm.com/api/user-agent/>

2017-12-14 16:17:36 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://helloacm.com/api/user-agent/> (referer: None)



********************

"Hello World"

********************

2017-12-14 16:17:37 [scrapy.core.engine] INFO: Closing spider (finished)

2017-12-14 16:17:37 [scrapy.statscollectors] INFO: Dumping Scrapy stats:

#-*-coding=utf-8-*-

from scrapy.contrib.downloadermiddleware.useragent import UserAgentMiddleware



class CustomerUserAgent(UserAgentMiddleware):

    def process_request(self, request, spider):

        ua='HELLO World?????????'

        request.headers.setdefault('User-Agent',ua)

DOWNLOADER_MIDDLEWARES = {

    'headerchange.customerMiddleware.customMiddleware.CustomerUserAgent':10

#    'headerchange.middlewares.MyCustomDownloaderMiddleware': 543,

}

  File "C:\Python27\lib\threading.py", line 736, in start

    _start_new_thread(self.__bootstrap, ())

thread.error: can't start new thread

fastboot flash recovery recovery.img

fastboot flash boot boot.img

fastboot flash system system.img

fastboot erase userdata

fastboot flash staging blob

fastboot flash boot boot.img

fastboot flash system system.img

fastboot erase userdata

这个策略是我做虚拟货币以来的主要策略，后面经过不断完善和修改，复杂了很多，但主要思想并没有改变，分享的这个版本是无明显bug的 最初版本，最为简单清晰，没有仓位管理，每次交易都是满仓，没有卡死后重启等等，但也足够说明问题。

function GetPrice(Type) {

   //_C()是平台的容错函数

    var depth=_C(exchange.GetDepth);

    var amountBids=0;

    var amountAsks=0;

    //计算买价，获取累计深度达到预设的价格

    if(Type=="Buy"){

       for(var i=0;i<20;i++){

           amountBids+=depth.Bids[i].Amount;

           //参数floatamountbuy是预设的累计深度

           if (amountBids>floatamountbuy){

               //稍微加0.01，使得订单排在前面

              return depth.Bids[i].Price+0.01;}

        }

    }

    //同理计算卖价

    if(Type=="Sell"){

       for(var j=0; j<20; j++){

    	   amountAsks+=depth.Asks[j].Amount;

            if (amountAsks>floatamountsell){

            return depth.Asks[j].Price-0.01;}

        }

    }

    //遍历了全部深度仍未满足需求，就返回一个价格，以免出现bug

    return depth.Asks[0].Price

}



// 每个循环的主函数onTick(),这里定的循环时间3.5s，每次循环都会把原来的单子撤销，重新挂单，越简单越不会遇到bug.



function onTick() {

    var buyPrice = GetPrice("Buy");

    var sellPrice= GetPrice("Sell");

    //diffprice是预设差价，买卖价差如果小于预设差价，就会挂一个相对更深的价格

    if ((sellPrice - buyPrice) <= diffprice){

            buyPrice-=10;

            sellPrice+=10;}

    //把原有的单子全部撤销，实际上经常出现新的价格和已挂单价格相同的情况，此时不需要撤销

    CancelPendingOrders() 

    //获取账户信息，确定目前账户存在多少钱和多少币

    var account=_C(exchange.GetAccount);

    //可买的比特币量，_N()是平台的精度函数

    var amountBuy = _N((account.Balance / buyPrice-0.1),2); 

    //可卖的比特币量，注意到没有仓位的限制，有多少就买卖多少，因为我当时的钱很少

    var amountSell = _N((account.Stocks),2); 

    if (amountSell > 0.02) {

        exchange.Sell(sellPrice,amountSell);}

    if (amountBuy > 0.02) {

        exchange.Buy(buyPrice, amountBuy);}

    //休眠，进入下一轮循环

    Sleep(sleeptime);

}

date = date + '-01-01'

        cmd = 'select * from `{}` where datetime > \'{}\''.format(code, date)



        try:

            df = pd.read_sql(cmd, history_engine,index_col='index')

        except Exception,e:

            print e

            return None

        

        # 不知道为啥，这里的类型发生改变

        idx= df['low'].idxmin()

datetime    datetime64[ns]

code                object

name                object

open                object

close               object

high                object

low                 object

vol                float64

amount             float64

dtype: object

df['low']=df['low'].astype('float64')

    X=[1,1,1,1,1,1,1]

    labels = [u'百度',u'京东',u'陆金所',u'工行',u'招行',u'华泰',u'国金',u'广发',u'QQ']

    plt.figure()

    p = plt.pie(X,labels=labels)

    for front in p[1]:

        front.set_fontproperties(mpl.font_manager.FontProperties(

            fname='C:\Windows\winsxs\amd64_microsoft-windows-font-truetype-simfang_31bf3856ad364e35_6.1.7600.16385_none_e417159f3b4eb1b7\simfang.ttf'))

#!/usr/bin/env python

#coding:utf-8

"""a demo of matplotlib"""

import matplotlib as  mpl

from matplotlib  import pyplot as plt

mpl.rcParams[u'font.sans-serif'] = ['simhei']

mpl.rcParams['axes.unicode_minus'] = False

years = [1950, 1960, 1970, 1980, 1990, 2000, 2010]

gdp = [300.2, 543.3, 1075.9, 2862.5, 5979.6, 10289.7, 14958.3]

#创建一副线图,x轴是年份,y轴是gdp

plt.plot(years, gdp, color='green', marker='o', linestyle='solid')

#添加一个标题

plt.title(u'名义GDP')

#给y轴加标记

plt.ylabel(u'十亿美元')

plt.show()

from django import template

def template_usage():

    t = template.Template('My name is {{ name }}')

    c = template.Context({'name':'Rocky'})

    print t.render(c)



template_usage()

session = requests.Session()

def getContent(url):

    try:

        ret = session.get(url)

    except:

        return None

    if ret.status_code==200:

        return ret.text

    else:

        return None

    content = getContent(url)

    root = etree.HTML(content)

    elements=root.xpath('//div[@data-current-count="1"]//li/a/@href')

    for items in elements:

        song_url = items.replace('//','')

        song_url=song_url.strip()

        print(song_url)

        p=subprocess.Popen('python you-get -d --format=HD {}'.format(song_url),stderr=subprocess.PIPE,stdout=subprocess.PIPE,shell=True)

        output,error = p.communicate()

        print(output)

        print(error)

        p.wait()

#-*-coding=utf-8-*-

import requests

from lxml import etree

import subprocess

session = requests.Session()

def getContent(url):

    # url='http://www.iqiyi.com/v_19rrkwcx6w.html'

    try:

        ret = session.get(url)

    # except Exception,e:

    except:

        # print e

        return None

    if ret.status_code==200:

        return ret.text

    else:

        return None



def getUrl():

    url='http://www.iqiyi.com/v_19rrkwcx6w.html'

    url2='http://www.iqiyi.com/v_19rrl2td7g.html' # 31-61

    content = getContent(url)

    root = etree.HTML(content)

    elements=root.xpath('//div[@data-current-count="1"]//li/a/@href')

    for items in elements:

        song_url = items.replace('//','')

        song_url=song_url.strip()

        print(song_url)

        p=subprocess.Popen('python you-get -d --format=HD {}'.format(song_url),stderr=subprocess.PIPE,stdout=subprocess.PIPE,shell=True)

        output,error = p.communicate()

        print(output)

        print(error)

        p.wait()



def main():

    getUrl()



if __name__ == '__main__':

    main()

#-*-coding=utf-8-*-

import sys,os

import requests

from lxml import etree

import subprocess

session = requests.Session()

def getContent(url):

    # url='http://www.iqiyi.com/v_19rrkwcx6w.html'

    try:

        ret = requests.get(url)

        ret.encoding='utf-8'

    # except Exception,e:

    except:

        # print e

        return None

    if ret.status_code==200:

        return ret.text

    else:

        return None



def getUrl():

    url='http://www.iqiyi.com/v_19rrkwcx6w.html'

    url2='http://www.iqiyi.com/v_19rrl2td7g.html' # 31-61

    content = getContent(url)

    if not content:

        print "network issue, retry"

        exit(0)

    root = etree.HTML(content,parser=etree.HTMLParser(encoding='utf-8'))

    elements=root.xpath('//div[@data-current-count="1"]//li')

    for items in elements:

        url_item=items.xpath('.//a/@href')[0]

        song_url = url_item.replace('//','')

        song_url=song_url.strip()

        print(song_url)

        # name=items.xpath('.//span[@class="item-num"]/text()')[0]

        name=items.xpath('.//span[@class="item-num"]/text()')[0].encode('utf-8').strip()+\

             ' '+items.xpath('.//span[@class="item-txt"]/text()')[0].encode('utf-8').strip()+'.mp4'

        name= '儿歌多多 '+name

        name=name.decode('utf-8')

        filename=os.path.join(os.getcwd(),name)

        print filename

        if os.path.exists(filename):

            continue

        p=subprocess.Popen('python you-get -d --format=HD {}'.format(song_url),stderr=subprocess.PIPE,stdout=subprocess.PIPE,shell=True)

        output,error = p.communicate()

        print(output)

        print(error)

        p.wait()





def main():

    getUrl()



if __name__ == '__main__':

    main()

# coding: utf-8

# http://30daydo.com

import urllib



import os

import requests

import time

from lxml import etree

from header_toolkit import getheader





def spider():

    curr=os.getcwd()

    target_dir=os.path.join(curr,'data')

    if not os.path.exists(target_dir):

        os.mkdir(target_dir)

    for i in range(1, 100, 10):

        url = 'http://www.lrts.me/ajax/playlist/2/32551/%d' % i

        headers = {

            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'}

        s = requests.get(url=url, headers=headers)

        tree = etree.HTML(s.text)

        nodes = tree.xpath('//*[starts-with(@class,"clearfix section-item section")]')

        print len(nodes)

        for node in nodes:

            filename = node.xpath('.//div[@class="column1 nowrap"]/span/text()')[0]

            link = node.xpath('.//input[@name="source" and @type="hidden"]/@value')[0]



            print link

            post_fix=link.split('.')[-1]

            full_path= filename+'.'+post_fix

            urllib.urlretrieve(link, filename=os.path.join(target_dir,full_path))

            time.sleep(1)





if __name__ == '__main__':

    spider()

{

    "_id" : "GuqXmAkkARqhBDqhy",

    "beatmapset_id" : "342537",

    "version" : "MX",

    "diff_approach" : "5",

    "artist" : "Yousei Teikoku",

    "title" : "Kokou no Sousei",

    "difficultyrating" : "3.5552737712860107"

}

{

    "_id" : "oHLT7KqsB7bztBGvu",

    "beatmapset_id" : "342537",

    "version" : "HD",

    "diff_approach" : "5",

    "artist" : "Yousei Teikoku",

    "title" : "Kokou no Sousei",

    "difficultyrating" : "2.7515676021575928"

}

{

    "_id" : "GbotZfrPEwW69FkGD",

    "beatmapset_id" : "342537",

    "version" : "NM",

    "diff_approach" : "5",

    "artist" : "Yousei Teikoku",

    "title" : "Kokou no Sousei",

    "difficultyrating" : "0"

}

db.getCollection('dup_case').aggregate(

[

{$group:{

    _id:{diff_approach:'$diff_approach'},

    count:{$sum:2}

    }},

{$match:{count:{$gt:1}}}

]

)