# -*- coding: utf-8 -*-

# website: http://30daydo.com

# @Time : 2019/2/23 19:34

# @File : youdao.py

# 解密有道词典的JS





import hashlib

import random

import requests

import time





def md5_(word):

    s = bytes(word, encoding='utf8')

    m = hashlib.md5()

    m.update(s)

    ret = m.hexdigest()

    return ret



def get_sign(word, salt):

    ret = md5_('fanyideskweb' + word + salt + 'p09@Bn{h02_BIEe]$P^nG')

    return ret





def youdao(word):

    url = 'http://fanyi.youdao.com/translate_o?smartresult=dict&smartresult=rule'

    headers = {

        'Host': 'fanyi.youdao.com',

        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0',

        'Accept': 'application/json, text/javascript, */*; q=0.01',

        'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',

        'Accept-Encoding': 'gzip, deflate',

        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',

        'X-Requested-With': 'XMLHttpRequest',

        'Referer': 'http://fanyi.youdao.com/',

        'Content-Length': '252',

        'Cookie': 'YOUDAO_MOBILE_ACCESS_TYPE=1; OUTFOX_SEARCH_USER_ID=1672542763@10.169.0.83; JSESSIONID=aaaWzxpjeDu1gbhopLzKw; ___rl__test__cookies=1550913722828; OUTFOX_SEARCH_USER_ID_NCOO=372126049.6326876',

        'Connection': 'keep-alive',

        'Pragma': 'no-cache',

        'Cache-Control': 'no-cache',

    }



    ts = str(int(time.time()*1000))

    salt=ts+str(random.randint(0,10))

    bv = md5_("5.0 (Windows)")

    sign= get_sign(word,salt)



    post_data = {

        'i': word,

        'from': 'AUTO', 'to': 'AUTO', 'smartresult': 'dict', 'client': 'fanyideskweb', 'salt': salt,

        'sign': sign, 'ts': ts, 'bv': bv, 'doctype': 'json', 'version': '2.1',

        'keyfrom': 'fanyi.web', 'action': 'FY_BY_REALTIME', 'typoResult': 'false'

    }



    r = requests.post(

        url=url,

        headers=headers,

        data=post_data

    )



    for item in r.json().get('smartResult',{}).get('entries'):

        print(item)



word='student'

youdao(word)

   headers = {'Accept': 'application/json,text/javascript,*/*;q=0.01', 'Accept-Encoding':

        'gzip,deflate,br',

               'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', 'Cache-Control': 'no-cache',

               # 'Connection': 'keep-alive',

               'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8',

               'Cookie': 'JSESSIONID=ABAAABAABEEAAJAACF8F22F99AFA35F9EEC28F2D0E46A41;_ga=GA1.2.331323650.1548204973;_gat=1;Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1548204973;user_trace_token=20190123085612-adf35b62-1ea9-11e9-b744-5254005c3644;LGSID=20190123085612-adf35c69-1ea9-11e9-b744-5254005c3644;PRE_UTM=;PRE_HOST=;PRE_SITE=;PRE_LAND=https%3A%2F%2Fwww.lagou.com%2F;LGUID=20190123085612-adf35ed5-1ea9-11e9-b744-5254005c3644;_gid=GA1.2.1809874038.1548204973;index_location_city=%E6%B7%B1%E5%9C%B3;TG-TRACK-CODE=index_search;SEARCH_ID=169bf76c08b548f8830967a1968d10ca;Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1548204985;LGRID=20190123085624-b52a0555-1ea9-11e9-b744-5254005c3644',

               'Host': 'www.lagou.com', 'Origin': 'https://www.lagou.com', 'Pragma': 'no-cache',

               'Referer': 'https://www.lagou.com/jobs/list_%E7%88%AC%E8%99%AB?labelWords=&fromSearch=true&suginput=',

               'User-Agent': 'Mozilla/5.0(WindowsNT6.3;WOW64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/71.0.3578.98Safari/537.36',

               'X-Anit-Forge-Code': '0',

               'X-Anit-Forge-Token': 'None',

               'X-Requested-With': 'XMLHttpRequest'

               }

    # data='中国人'

    data='hello'

  

    ret = CryptoJS.AES.encrypt(data,'secret key 123')

    

    content = ret.toString()

    result = CryptoJS.AES.decrypt(content,'secret key 123')

    print(result.toString(CryptoJS.enc.Utf8))

s='耐克球鞋'

s =s.encode('gb2312')

 data = {'__VIEWSTATE': view_state,

        '__EVENTVALIDATION': event_validation,

        'txtTKeyword': s,

        'btQuery.x': 41,

        'btQuery.y': 24,

        }



r = session.post(url=self.base_url, headers=headers,

                     data=data,proxies=self.get_proxy()

import grequests

import requests

import bs4



def simple_request(url):

    page = requests.get(url)

    return page



urls = [

    'http://www.heroku.com',

    'http://python-tablib.org',

    'http://httpbin.org',

    'http://python-requests.org',

    'http://kennethreitz.com'

]



rs = [grequests.get(simple_request(u)) for u in urls]





grequests.map(rs)

rs = (grequests.get(u) for u in urls)

requests = grequests.map(rs)

for response in requests:

    market_watch(response.content)

# -*-coding=utf-8-*-



# @Time : 2018/10/23 9:26

# @File : money_reward.py

import requests

from collections import OrderedDict

import time

import datetime

import pymongo

import config



session = requests.Session()

def get_proxy(retry=10):

    proxyurl = 'http://{}:8081/dynamicIp/common/getDynamicIp.do'.format(config.PROXY)

    count = 0

    for i in range(retry):

        try:

            r = requests.get(proxyurl, timeout=10)

        except Exception as e:

            print(e)

            count += 1

            print('代理获取失败,重试' + str(count))

            time.sleep(1)



        else:

            js = r.json()

            proxyServer = 'http://{0}:{1}'.format(js.get('ip'), js.get('port'))

            proxies_random = {

                'http': proxyServer

            }

            return proxies_random





def get_content(url):

    headers = {

        # 此处添加个人的header信息

    }

    try:

        proxy = get_proxy()

    except Exception as e:

        print(e)

        proxy = get_proxy()



    try:

        r = session.get(url=url, headers=headers,proxies=proxy,timeout=10)

    except Exception as e:

        print(e)

        proxy = get_proxy()

        r = session.get(url=url, headers=headers,proxies=proxy,timeout=10)



    return r





def parse_content(post_id):

    url = 'https://xueqiu.com/statuses/reward/list_by_user.json?status_id={}&page=1&size=99999999'.format(post_id)

    r = get_content(url)

    print(r.text)

    if r.status_code != 200:

        print('status code != 200')

        failed_doc.insert({'post_id':post_id,'status':0})

        return None



    try:



        js_data = r.json()

    except Exception as e:

        print(e)

        print('can not parse to json')

        print(post_id)

        failed_doc.insert({'post_id': post_id, 'status': 0})

        return



    ret = 

    been_reward_user = '元卫南'

    for item in js_data.get('items'):

        name = item.get('name')

        amount = item.get('amount')

        description = item.get('description')

        user_id = item.get('user_id')

        created_at = item.get('created_at')

        if created_at:

            created_at = datetime.datetime.fromtimestamp(int(created_at) / 1000).strftime('%Y-%m-%d %H:%M:%S')



        d = OrderedDict()

        d['name'] = name

        d['user_id'] = user_id

        d['amount'] = amount / 100

        d['description'] = description

        d['created_at'] = created_at

        d['been_reward'] = been_reward_user

        ret.append(d)



    print(ret)

    if ret:

        doc.insert_many(ret)

        failed_doc.insert({'post_id':post_id,'status':1})







def get_all_page_id(user_id):

    doc = db['db_parker']['xueqiu_zhuanglan']



    get_page_url = 'https://xueqiu.com/statuses/original/timeline.json?user_id={}&page=1'.format(user_id)

    r = get_content(get_page_url)

    max_page = int(r.json().get('maxPage'))



    for i in range(1, max_page + 1):

        url = 'https://xueqiu.com/statuses/original/timeline.json?user_id=2227798650&page={}'.format(i)

        r = get_content(url)

        js_data = r.json()

        ret = 



        for item in js_data.get('list'):

            d = OrderedDict()



            d['article_id'] = item.get('id')

            d['title'] = item.get('title')

            d['description'] = item.get('description')

            d['view_count'] = item.get('view_count')

            d['target'] = 'https://xueqiu.com/' + item.get('target')

            d['user_id']= item.get('user_id')

            d['created_at'] = datetime.datetime.fromtimestamp(int(item.get('created_at')) / 1000).strftime(

                '%Y-%m-%d %H:%M:%S')



            ret.append(d)

        print(d)

        doc.insert_many(ret)



def loop_page_id():

    doc = db['db_parker']['xueqiu_zhuanglan']

    ret = doc.find({},{'article_id':1})

    failed_doc = db['db_parker']['xueqiu_reward_status']

    failed_ret = failed_doc.find({'status':1})

    article_id_list =

    for i in failed_ret:

        article_id_list.append(i.get('article_id'))



    for item in ret:

        article_id = item.get('article_id')

        print(article_id)

        if article_id in article_id_list:

            continue

        else:

            parse_content(article_id)



loop_page_id()

from PIL import Image

im = Image.open('test_0.jpg')

pytesseract.image_to_string(im)

# -*- coding: utf-8 -*-

import re

import scrapy

from scrapy import Request, FormRequest

from jsl.items import JslItem

from jsl import config

import logging



class AllcontentSpider(scrapy.Spider):

    name = 'allcontent'



    headers = {

        'Host': 'www.jisilu.cn', 'Connection': 'keep-alive', 'Pragma': 'no-cache',

        'Cache-Control': 'no-cache', 'Accept': 'application/json,text/javascript,*/*;q=0.01',

        'Origin': 'https://www.jisilu.cn', 'X-Requested-With': 'XMLHttpRequest',

        'User-Agent': 'Mozilla/5.0(WindowsNT6.1;WOW64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/67.0.3396.99Safari/537.36',

        'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8',

        'Referer': 'https://www.jisilu.cn/login/',

        'Accept-Encoding': 'gzip,deflate,br',

        'Accept-Language': 'zh,en;q=0.9,en-US;q=0.8'

    }



    def start_requests(self):

        login_url = 'https://www.jisilu.cn/login/'

        headers = {

            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',

            'Accept-Encoding': 'gzip,deflate,br', 'Accept-Language': 'zh,en;q=0.9,en-US;q=0.8',

            'Cache-Control': 'no-cache', 'Connection': 'keep-alive',

            'Host': 'www.jisilu.cn', 'Pragma': 'no-cache', 'Referer': 'https://www.jisilu.cn/',

            'Upgrade-Insecure-Requests': '1',

            'User-Agent': 'Mozilla/5.0(WindowsNT6.1;WOW64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/67.0.3396.99Safari/537.36'}



        yield Request(url=login_url, headers=headers, callback=self.login,dont_filter=True)



    def login(self, response):

        url = 'https://www.jisilu.cn/account/ajax/login_process/'

        data = {

            'return_url': 'https://www.jisilu.cn/',

            'user_name': config.username,

            'password': config.password,

            'net_auto_login': '1',

            '_post_type': 'ajax',

        }



        yield FormRequest(

            url=url,

            headers=self.headers,

            formdata=data,

            callback=self.parse,

            dont_filter=True

        )



    def parse(self, response):

        for i in range(1,3726):

            focus_url = 'https://www.jisilu.cn/home/explore/sort_type-new__day-0__page-{}'.format(i)

            yield Request(url=focus_url, headers=self.headers, callback=self.parse_page,dont_filter=True)



    def parse_page(self, response):

        nodes = response.xpath('//div[@class="aw-question-list"]/div')

        for node in nodes:

            each_url=node.xpath('.//h4/a/@href').extract_first()

            yield Request(url=each_url,headers=self.headers,callback=self.parse_item,dont_filter=True)



    def parse_item(self,response):

        item = JslItem()

        title = response.xpath('//div[@class="aw-mod-head"]/h1/text()').extract_first()

        s = response.xpath('//div[@class="aw-question-detail-txt markitup-box"]').xpath('string(.)').extract_first()

        ret = re.findall('(.*?)\.donate_user_avatar', s, re.S)



        try:

            content = ret[0].strip()

        except:

            content = None



        createTime = response.xpath('//div[@class="aw-question-detail-meta"]/span/text()').extract_first()



        resp_no = response.xpath('//div[@class="aw-mod aw-question-detail-box"]//ul/h2/text()').re_first('\d+')



        url = response.url

        item['title'] = title.strip()

        item['content'] = content

        try:

            item['resp_no']=int(resp_no)

        except Exception as e:

            logging.warning('e')

            item['resp_no']=None



        item['createTime'] = createTime

        item['url'] = url.strip()

        resp = 

        for index,reply in enumerate(response.xpath('//div[@class="aw-mod-body aw-dynamic-topic"]/div[@class="aw-item"]')):

            replay_user = reply.xpath('.//div[@class="pull-left aw-dynamic-topic-content"]//p/a/text()').extract_first()

            rep_content = reply.xpath(

                './/div[@class="pull-left aw-dynamic-topic-content"]//div[@class="markitup-box"]/text()').extract_first()

            # print rep_content

            agree=reply.xpath('.//em[@class="aw-border-radius-5 aw-vote-count pull-left"]/text()').extract_first()

            resp.append({replay_user.strip()+'_{}'.format(index): [int(agree),rep_content.strip()]})



        item['resp'] = resp

        yield item

import pymongo

from collections import OrderedDict

class JslPipeline(object):

    def __init__(self):

        self.db = pymongo.MongoClient(host='10.18.6.1',port=27017)

        # self.user = u'neo牛3' # 修改为指定的用户名 如 毛之川 ，然后找到用户的id，在用户也的源码哪里可以找到 比如持有封基是8132

        self.collection = self.db['db_parker']['jsl']

    def process_item(self, item, spider):

        self.collection.insert(OrderedDict(item))

        return item

yield scrapy.Request(

                url=self.base_url.format(i),

                meta={'page':str(i),

                    'splash': {

                        'args': {

                            'images':0,

                            'wait': 15,

                            'proxy': self.get_proxy(),

                        },

                        'endpoint': 'render.html',

                        },

            },

            )

from scrapy import log

log.msg("This is a warning", level=log.WARING)

[py.warnings] WARNING: E:\git\CrawlMan\bilibili\bilibili\spiders\bili.py:14: ScrapyDeprecationWarning: log.msg has been deprecated, create a python logger and log through it instead

  log.msg

# -*- coding: utf-8 -*-

import scrapy

from scrapy_splash import SplashRequest

import logging

# from scrapy import log

class BiliSpider(scrapy.Spider):

    name = 'ordinary'  # 这个名字就是上面连接中那个启动应用的名字

    allowed_domain = ["bilibili.com"]

    start_urls = [

        "https://www.bilibili.com/"

    ]



    def parse(self, response):

        logging.info('====================================================')

        content = response.xpath("//div[@class='num-wrap']").extract_first()

        logging.info(content)

        logging.info('====================================================')

<div id="content">

    <div class="login-wrap">

		<div class="w">

            <div class="login-form">

                <div class="login-tab login-tab-l">

                    <a href="javascript:void(0)" clstag="pageclick|keycount|201607144|1"> 扫码登录</a>

                </div>

                <div class="login-tab login-tab-r">

                    <a href="javascript:void(0)" clstag="pageclick|keycount|201607144|2">账户登录</a>

                </div>

                <div class="login-box">

                    <div class="mt tab-h">

                    </div>

                    <div class="msg-wrap">

						                        <div class="msg-error hide"><b></b></div>

                    </div>

# -*-coding=utf-8-*-



import requests

session = requests.Session()

user = ''

password = ''



def getCode():

    headers = {

        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36'}

    url = 'http://vexx.pro/verify/code.html'

    s = session.get(url=url, headers=headers)



    with open('code.png', 'wb') as f:

        f.write(s.content)



    code = raw_input('input the code: ')

    print 'code is ', code



    login_url = 'http://vexx.pro/login/up_login.html'

    post_data = {

        'moble': user,

        'mobles': '+86',

        'password': password,

        'verify': code,

        'login_token': ''}



    login_s = session.post(url=login_url, headers=header, data=post_data)

    print login_s.status_code



    zzc_url = 'http://vexx.pro/ajax/check_zzc/'

    zzc_s = session.get(url=zzc_url, headers=headers)

    print zzc_s.text



def main():

    getCode()



if __name__ == '__main__':

    main()

scrapy startproject headerchange

scrapy genspider headervalidation helloacm.com

class HeadervalidationSpider(scrapy.Spider):

    name = 'headervalidation'

    allowed_domains = ['helloacm.com']

    start_urls = ['http://helloacm.com/api/user-agent/']



    def parse(self, response):

        print '*'*20

        print response.body

        print '*'*20

scrapy crawl headervalidation

# Crawl responsibly by identifying yourself (and your website) on the user-agent

USER_AGENT = 'Hello World'

scrapy crawl headervalidation

2017-12-14 16:17:35 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023

2017-12-14 16:17:35 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://helloacm.com/api/us

er-agent/> from <GET http://helloacm.com/api/user-agent/>

2017-12-14 16:17:36 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://helloacm.com/api/user-agent/> (referer: None)



[b]********************

"Hello World"

********************

[/b]2017-12-14 16:17:37 [scrapy.core.engine] INFO: Closing spider (finished)

2017-12-14 16:17:37 [scrapy.statscollectors] INFO: Dumping Scrapy stats:

{'downloader/request_bytes': 406,

 'downloader/request_count': 2,

 'downloader/request_method_count/GET': 2,

 'downloader/response_bytes': 796,

 'downloader/response_count': 2,

 'downloader/response_status_count/200': 1,

 'downloader/response_status_count/301': 1,

 'finish_reason': 'finished',

 'finish_time': datetime.datetime(2017, 12, 14, 8, 17, 37, 29000),

 'log_count/DEBUG': 3,

 'log_count/INFO': 7,

 'response_received_count': 1,

 'scheduler/dequeued': 2,

 'scheduler/dequeued/memory': 2,

 'scheduler/enqueued': 2,

 'scheduler/enqueued/memory': 2,

 'start_time': datetime.datetime(2017, 12, 14, 8, 17, 35, 137000)}

2017-12-14 16:17:37 [scrapy.core.engine] INFO: Spider closed (finished)

# Override the default request headers:

 DEFAULT_REQUEST_HEADERS = {

    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',

    'Accept-Language': 'en',

    'User-Agent':'Hello World'

 }

class HeadervalidationSpider(scrapy.Spider):

    name = 'headervalidation'

    allowed_domains = ['helloacm.com']





    def start_requests(self):

        header={'User-Agent':'Hello World'}

        yield scrapy.Request(url='http://helloacm.com/api/user-agent/',headers=header)



    def parse(self, response):

        print '*'*20

        print response.body

        print '*'*20

2017-12-14 16:17:35 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023

2017-12-14 16:17:35 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://helloacm.com/api/us

er-agent/> from <GET http://helloacm.com/api/user-agent/>

2017-12-14 16:17:36 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://helloacm.com/api/user-agent/> (referer: None)



********************

"Hello World"

********************

2017-12-14 16:17:37 [scrapy.core.engine] INFO: Closing spider (finished)

2017-12-14 16:17:37 [scrapy.statscollectors] INFO: Dumping Scrapy stats:

#-*-coding=utf-8-*-

from scrapy.contrib.downloadermiddleware.useragent import UserAgentMiddleware



class CustomerUserAgent(UserAgentMiddleware):

    def process_request(self, request, spider):

        ua='HELLO World?????????'

        request.headers.setdefault('User-Agent',ua)

DOWNLOADER_MIDDLEWARES = {

    'headerchange.customerMiddleware.customMiddleware.CustomerUserAgent':10

#    'headerchange.middlewares.MyCustomDownloaderMiddleware': 543,

}

session = requests.Session()

def getContent(url):

    try:

        ret = session.get(url)

    except:

        return None

    if ret.status_code==200:

        return ret.text

    else:

        return None

    content = getContent(url)

    root = etree.HTML(content)

    elements=root.xpath('//div[@data-current-count="1"]//li/a/@href')

    for items in elements:

        song_url = items.replace('//','')

        song_url=song_url.strip()

        print(song_url)

        p=subprocess.Popen('python you-get -d --format=HD {}'.format(song_url),stderr=subprocess.PIPE,stdout=subprocess.PIPE,shell=True)

        output,error = p.communicate()

        print(output)

        print(error)

        p.wait()

#-*-coding=utf-8-*-

import requests

from lxml import etree

import subprocess

session = requests.Session()

def getContent(url):

    # url='http://www.iqiyi.com/v_19rrkwcx6w.html'

    try:

        ret = session.get(url)

    # except Exception,e:

    except:

        # print e

        return None

    if ret.status_code==200:

        return ret.text

    else:

        return None



def getUrl():

    url='http://www.iqiyi.com/v_19rrkwcx6w.html'

    url2='http://www.iqiyi.com/v_19rrl2td7g.html' # 31-61

    content = getContent(url)

    root = etree.HTML(content)

    elements=root.xpath('//div[@data-current-count="1"]//li/a/@href')

    for items in elements:

        song_url = items.replace('//','')

        song_url=song_url.strip()

        print(song_url)

        p=subprocess.Popen('python you-get -d --format=HD {}'.format(song_url),stderr=subprocess.PIPE,stdout=subprocess.PIPE,shell=True)

        output,error = p.communicate()

        print(output)

        print(error)

        p.wait()



def main():

    getUrl()



if __name__ == '__main__':

    main()

#-*-coding=utf-8-*-

import sys,os

import requests

from lxml import etree

import subprocess

session = requests.Session()

def getContent(url):

    # url='http://www.iqiyi.com/v_19rrkwcx6w.html'

    try:

        ret = requests.get(url)

        ret.encoding='utf-8'

    # except Exception,e:

    except:

        # print e

        return None

    if ret.status_code==200:

        return ret.text

    else:

        return None



def getUrl():

    url='http://www.iqiyi.com/v_19rrkwcx6w.html'

    url2='http://www.iqiyi.com/v_19rrl2td7g.html' # 31-61

    content = getContent(url)

    if not content:

        print "network issue, retry"

        exit(0)

    root = etree.HTML(content,parser=etree.HTMLParser(encoding='utf-8'))

    elements=root.xpath('//div[@data-current-count="1"]//li')

    for items in elements:

        url_item=items.xpath('.//a/@href')[0]

        song_url = url_item.replace('//','')

        song_url=song_url.strip()

        print(song_url)

        # name=items.xpath('.//span[@class="item-num"]/text()')[0]

        name=items.xpath('.//span[@class="item-num"]/text()')[0].encode('utf-8').strip()+\

             ' '+items.xpath('.//span[@class="item-txt"]/text()')[0].encode('utf-8').strip()+'.mp4'

        name= '儿歌多多 '+name

        name=name.decode('utf-8')

        filename=os.path.join(os.getcwd(),name)

        print filename

        if os.path.exists(filename):

            continue

        p=subprocess.Popen('python you-get -d --format=HD {}'.format(song_url),stderr=subprocess.PIPE,stdout=subprocess.PIPE,shell=True)

        output,error = p.communicate()

        print(output)

        print(error)

        p.wait()





def main():

    getUrl()



if __name__ == '__main__':

    main()

# coding: utf-8

# http://30daydo.com

import urllib



import os

import requests

import time

from lxml import etree

from header_toolkit import getheader





def spider():

    curr=os.getcwd()

    target_dir=os.path.join(curr,'data')

    if not os.path.exists(target_dir):

        os.mkdir(target_dir)

    for i in range(1, 100, 10):

        url = 'http://www.lrts.me/ajax/playlist/2/32551/%d' % i

        headers = {

            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'}

        s = requests.get(url=url, headers=headers)

        tree = etree.HTML(s.text)

        nodes = tree.xpath('//*[starts-with(@class,"clearfix section-item section")]')

        print len(nodes)

        for node in nodes:

            filename = node.xpath('.//div[@class="column1 nowrap"]/span/text()')[0]

            link = node.xpath('.//input[@name="source" and @type="hidden"]/@value')[0]



            print link

            post_fix=link.split('.')[-1]

            full_path= filename+'.'+post_fix

            urllib.urlretrieve(link, filename=os.path.join(target_dir,full_path))

            time.sleep(1)





if __name__ == '__main__':

    spider()

#Get your range of csdn

'''

http://30daydo.com

contact: weigesysu@qq.com

'''

import urllib2,re

import time

link='http://blog.csdn.net/[b]用户名[/b]/article/details/52858314'

user_agent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)"

header = {"User-Agent": user_agent}

req = urllib2.Request(link, headers=header)

resp = urllib2.urlopen(req)

content = resp.read()

#print content

p=re.compile(r'<li>排名：<span>第(\d+)名</span></li>')

result=p.findall(content)

print result[0]



today=time.strftime("%Y-%m-%d")

print today



f=open("data/csdn_range.txt",'a')

contents=today+'\t'+result[0]+'\n'

f.write(contents)

f.close()