# -*-coding=utf-8-*-



# @Time : 2018/10/23 9:26

# @File : money_reward.py

import requests

from collections import OrderedDict

import time

import datetime

import pymongo

import config



session = requests.Session()

def get_proxy(retry=10):

    proxyurl = 'http://{}:8081/dynamicIp/common/getDynamicIp.do'.format(config.PROXY)

    count = 0

    for i in range(retry):

        try:

            r = requests.get(proxyurl, timeout=10)

        except Exception as e:

            print(e)

            count += 1

            print('代理获取失败,重试' + str(count))

            time.sleep(1)



        else:

            js = r.json()

            proxyServer = 'http://{0}:{1}'.format(js.get('ip'), js.get('port'))

            proxies_random = {

                'http': proxyServer

            }

            return proxies_random





def get_content(url):

    headers = {

        # 此处添加个人的header信息

    }

    try:

        proxy = get_proxy()

    except Exception as e:

        print(e)

        proxy = get_proxy()



    try:

        r = session.get(url=url, headers=headers,proxies=proxy,timeout=10)

    except Exception as e:

        print(e)

        proxy = get_proxy()

        r = session.get(url=url, headers=headers,proxies=proxy,timeout=10)



    return r





def parse_content(post_id):

    url = 'https://xueqiu.com/statuses/reward/list_by_user.json?status_id={}&page=1&size=99999999'.format(post_id)

    r = get_content(url)

    print(r.text)

    if r.status_code != 200:

        print('status code != 200')

        failed_doc.insert({'post_id':post_id,'status':0})

        return None



    try:



        js_data = r.json()

    except Exception as e:

        print(e)

        print('can not parse to json')

        print(post_id)

        failed_doc.insert({'post_id': post_id, 'status': 0})

        return



    ret = 

    been_reward_user = '元卫南'

    for item in js_data.get('items'):

        name = item.get('name')

        amount = item.get('amount')

        description = item.get('description')

        user_id = item.get('user_id')

        created_at = item.get('created_at')

        if created_at:

            created_at = datetime.datetime.fromtimestamp(int(created_at) / 1000).strftime('%Y-%m-%d %H:%M:%S')



        d = OrderedDict()

        d['name'] = name

        d['user_id'] = user_id

        d['amount'] = amount / 100

        d['description'] = description

        d['created_at'] = created_at

        d['been_reward'] = been_reward_user

        ret.append(d)



    print(ret)

    if ret:

        doc.insert_many(ret)

        failed_doc.insert({'post_id':post_id,'status':1})







def get_all_page_id(user_id):

    doc = db['db_parker']['xueqiu_zhuanglan']



    get_page_url = 'https://xueqiu.com/statuses/original/timeline.json?user_id={}&page=1'.format(user_id)

    r = get_content(get_page_url)

    max_page = int(r.json().get('maxPage'))



    for i in range(1, max_page + 1):

        url = 'https://xueqiu.com/statuses/original/timeline.json?user_id=2227798650&page={}'.format(i)

        r = get_content(url)

        js_data = r.json()

        ret = 



        for item in js_data.get('list'):

            d = OrderedDict()



            d['article_id'] = item.get('id')

            d['title'] = item.get('title')

            d['description'] = item.get('description')

            d['view_count'] = item.get('view_count')

            d['target'] = 'https://xueqiu.com/' + item.get('target')

            d['user_id']= item.get('user_id')

            d['created_at'] = datetime.datetime.fromtimestamp(int(item.get('created_at')) / 1000).strftime(

                '%Y-%m-%d %H:%M:%S')



            ret.append(d)

        print(d)

        doc.insert_many(ret)



def loop_page_id():

    doc = db['db_parker']['xueqiu_zhuanglan']

    ret = doc.find({},{'article_id':1})

    failed_doc = db['db_parker']['xueqiu_reward_status']

    failed_ret = failed_doc.find({'status':1})

    article_id_list =

    for i in failed_ret:

        article_id_list.append(i.get('article_id'))



    for item in ret:

        article_id = item.get('article_id')

        print(article_id)

        if article_id in article_id_list:

            continue

        else:

            parse_content(article_id)



loop_page_id()

from PIL import Image

im = Image.open('test_0.jpg')

pytesseract.image_to_string(im)

# -*- coding: utf-8 -*-

import re

import scrapy

from scrapy import Request, FormRequest

from jsl.items import JslItem

from jsl import config

import logging



class AllcontentSpider(scrapy.Spider):

    name = 'allcontent'



    headers = {

        'Host': 'www.jisilu.cn', 'Connection': 'keep-alive', 'Pragma': 'no-cache',

        'Cache-Control': 'no-cache', 'Accept': 'application/json,text/javascript,*/*;q=0.01',

        'Origin': 'https://www.jisilu.cn', 'X-Requested-With': 'XMLHttpRequest',

        'User-Agent': 'Mozilla/5.0(WindowsNT6.1;WOW64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/67.0.3396.99Safari/537.36',

        'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8',

        'Referer': 'https://www.jisilu.cn/login/',

        'Accept-Encoding': 'gzip,deflate,br',

        'Accept-Language': 'zh,en;q=0.9,en-US;q=0.8'

    }



    def start_requests(self):

        login_url = 'https://www.jisilu.cn/login/'

        headers = {

            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',

            'Accept-Encoding': 'gzip,deflate,br', 'Accept-Language': 'zh,en;q=0.9,en-US;q=0.8',

            'Cache-Control': 'no-cache', 'Connection': 'keep-alive',

            'Host': 'www.jisilu.cn', 'Pragma': 'no-cache', 'Referer': 'https://www.jisilu.cn/',

            'Upgrade-Insecure-Requests': '1',

            'User-Agent': 'Mozilla/5.0(WindowsNT6.1;WOW64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/67.0.3396.99Safari/537.36'}



        yield Request(url=login_url, headers=headers, callback=self.login,dont_filter=True)



    def login(self, response):

        url = 'https://www.jisilu.cn/account/ajax/login_process/'

        data = {

            'return_url': 'https://www.jisilu.cn/',

            'user_name': config.username,

            'password': config.password,

            'net_auto_login': '1',

            '_post_type': 'ajax',

        }



        yield FormRequest(

            url=url,

            headers=self.headers,

            formdata=data,

            callback=self.parse,

            dont_filter=True

        )



    def parse(self, response):

        for i in range(1,3726):

            focus_url = 'https://www.jisilu.cn/home/explore/sort_type-new__day-0__page-{}'.format(i)

            yield Request(url=focus_url, headers=self.headers, callback=self.parse_page,dont_filter=True)



    def parse_page(self, response):

        nodes = response.xpath('//div[@class="aw-question-list"]/div')

        for node in nodes:

            each_url=node.xpath('.//h4/a/@href').extract_first()

            yield Request(url=each_url,headers=self.headers,callback=self.parse_item,dont_filter=True)



    def parse_item(self,response):

        item = JslItem()

        title = response.xpath('//div[@class="aw-mod-head"]/h1/text()').extract_first()

        s = response.xpath('//div[@class="aw-question-detail-txt markitup-box"]').xpath('string(.)').extract_first()

        ret = re.findall('(.*?)\.donate_user_avatar', s, re.S)



        try:

            content = ret[0].strip()

        except:

            content = None



        createTime = response.xpath('//div[@class="aw-question-detail-meta"]/span/text()').extract_first()



        resp_no = response.xpath('//div[@class="aw-mod aw-question-detail-box"]//ul/h2/text()').re_first('\d+')



        url = response.url

        item['title'] = title.strip()

        item['content'] = content

        try:

            item['resp_no']=int(resp_no)

        except Exception as e:

            logging.warning('e')

            item['resp_no']=None



        item['createTime'] = createTime

        item['url'] = url.strip()

        resp = 

        for index,reply in enumerate(response.xpath('//div[@class="aw-mod-body aw-dynamic-topic"]/div[@class="aw-item"]')):

            replay_user = reply.xpath('.//div[@class="pull-left aw-dynamic-topic-content"]//p/a/text()').extract_first()

            rep_content = reply.xpath(

                './/div[@class="pull-left aw-dynamic-topic-content"]//div[@class="markitup-box"]/text()').extract_first()

            # print rep_content

            agree=reply.xpath('.//em[@class="aw-border-radius-5 aw-vote-count pull-left"]/text()').extract_first()

            resp.append({replay_user.strip()+'_{}'.format(index): [int(agree),rep_content.strip()]})



        item['resp'] = resp

        yield item

import pymongo

from collections import OrderedDict

class JslPipeline(object):

    def __init__(self):

        self.db = pymongo.MongoClient(host='10.18.6.1',port=27017)

        # self.user = u'neo牛3' # 修改为指定的用户名 如 毛之川 ，然后找到用户的id，在用户也的源码哪里可以找到 比如持有封基是8132

        self.collection = self.db['db_parker']['jsl']

    def process_item(self, item, spider):

        self.collection.insert(OrderedDict(item))

        return item

yield scrapy.Request(

                url=self.base_url.format(i),

                meta={'page':str(i),

                    'splash': {

                        'args': {

                            'images':0,

                            'wait': 15,

                            'proxy': self.get_proxy(),

                        },

                        'endpoint': 'render.html',

                        },

            },

            )

from scrapy import log

log.msg("This is a warning", level=log.WARING)

[py.warnings] WARNING: E:\git\CrawlMan\bilibili\bilibili\spiders\bili.py:14: ScrapyDeprecationWarning: log.msg has been deprecated, create a python logger and log through it instead

  log.msg

# -*- coding: utf-8 -*-

import scrapy

from scrapy_splash import SplashRequest

import logging

# from scrapy import log

class BiliSpider(scrapy.Spider):

    name = 'ordinary'  # 这个名字就是上面连接中那个启动应用的名字

    allowed_domain = ["bilibili.com"]

    start_urls = [

        "https://www.bilibili.com/"

    ]



    def parse(self, response):

        logging.info('====================================================')

        content = response.xpath("//div[@class='num-wrap']").extract_first()

        logging.info(content)

        logging.info('====================================================')

<div id="content">

    <div class="login-wrap">

		<div class="w">

            <div class="login-form">

                <div class="login-tab login-tab-l">

                    <a href="javascript:void(0)" clstag="pageclick|keycount|201607144|1"> 扫码登录</a>

                </div>

                <div class="login-tab login-tab-r">

                    <a href="javascript:void(0)" clstag="pageclick|keycount|201607144|2">账户登录</a>

                </div>

                <div class="login-box">

                    <div class="mt tab-h">

                    </div>

                    <div class="msg-wrap">

						                        <div class="msg-error hide"><b></b></div>

                    </div>

# -*-coding=utf-8-*-



import requests

session = requests.Session()

user = ''

password = ''



def getCode():

    headers = {

        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36'}

    url = 'http://vexx.pro/verify/code.html'

    s = session.get(url=url, headers=headers)



    with open('code.png', 'wb') as f:

        f.write(s.content)



    code = raw_input('input the code: ')

    print 'code is ', code



    login_url = 'http://vexx.pro/login/up_login.html'

    post_data = {

        'moble': user,

        'mobles': '+86',

        'password': password,

        'verify': code,

        'login_token': ''}



    login_s = session.post(url=login_url, headers=header, data=post_data)

    print login_s.status_code



    zzc_url = 'http://vexx.pro/ajax/check_zzc/'

    zzc_s = session.get(url=zzc_url, headers=headers)

    print zzc_s.text



def main():

    getCode()



if __name__ == '__main__':

    main()

scrapy startproject headerchange

scrapy genspider headervalidation helloacm.com

class HeadervalidationSpider(scrapy.Spider):

    name = 'headervalidation'

    allowed_domains = ['helloacm.com']

    start_urls = ['http://helloacm.com/api/user-agent/']



    def parse(self, response):

        print '*'*20

        print response.body

        print '*'*20

scrapy crawl headervalidation

# Crawl responsibly by identifying yourself (and your website) on the user-agent

USER_AGENT = 'Hello World'

scrapy crawl headervalidation

2017-12-14 16:17:35 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023

2017-12-14 16:17:35 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://helloacm.com/api/us

er-agent/> from <GET http://helloacm.com/api/user-agent/>

2017-12-14 16:17:36 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://helloacm.com/api/user-agent/> (referer: None)



[b]********************

"Hello World"

********************

[/b]2017-12-14 16:17:37 [scrapy.core.engine] INFO: Closing spider (finished)

2017-12-14 16:17:37 [scrapy.statscollectors] INFO: Dumping Scrapy stats:

{'downloader/request_bytes': 406,

 'downloader/request_count': 2,

 'downloader/request_method_count/GET': 2,

 'downloader/response_bytes': 796,

 'downloader/response_count': 2,

 'downloader/response_status_count/200': 1,

 'downloader/response_status_count/301': 1,

 'finish_reason': 'finished',

 'finish_time': datetime.datetime(2017, 12, 14, 8, 17, 37, 29000),

 'log_count/DEBUG': 3,

 'log_count/INFO': 7,

 'response_received_count': 1,

 'scheduler/dequeued': 2,

 'scheduler/dequeued/memory': 2,

 'scheduler/enqueued': 2,

 'scheduler/enqueued/memory': 2,

 'start_time': datetime.datetime(2017, 12, 14, 8, 17, 35, 137000)}

2017-12-14 16:17:37 [scrapy.core.engine] INFO: Spider closed (finished)

# Override the default request headers:

 DEFAULT_REQUEST_HEADERS = {

    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',

    'Accept-Language': 'en',

    'User-Agent':'Hello World'

 }

class HeadervalidationSpider(scrapy.Spider):

    name = 'headervalidation'

    allowed_domains = ['helloacm.com']





    def start_requests(self):

        header={'User-Agent':'Hello World'}

        yield scrapy.Request(url='http://helloacm.com/api/user-agent/',headers=header)



    def parse(self, response):

        print '*'*20

        print response.body

        print '*'*20

2017-12-14 16:17:35 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023

2017-12-14 16:17:35 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://helloacm.com/api/us

er-agent/> from <GET http://helloacm.com/api/user-agent/>

2017-12-14 16:17:36 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://helloacm.com/api/user-agent/> (referer: None)



********************

"Hello World"

********************

2017-12-14 16:17:37 [scrapy.core.engine] INFO: Closing spider (finished)

2017-12-14 16:17:37 [scrapy.statscollectors] INFO: Dumping Scrapy stats:

#-*-coding=utf-8-*-

from scrapy.contrib.downloadermiddleware.useragent import UserAgentMiddleware



class CustomerUserAgent(UserAgentMiddleware):

    def process_request(self, request, spider):

        ua='HELLO World?????????'

        request.headers.setdefault('User-Agent',ua)

DOWNLOADER_MIDDLEWARES = {

    'headerchange.customerMiddleware.customMiddleware.CustomerUserAgent':10

#    'headerchange.middlewares.MyCustomDownloaderMiddleware': 543,

}

session = requests.Session()

def getContent(url):

    try:

        ret = session.get(url)

    except:

        return None

    if ret.status_code==200:

        return ret.text

    else:

        return None

    content = getContent(url)

    root = etree.HTML(content)

    elements=root.xpath('//div[@data-current-count="1"]//li/a/@href')

    for items in elements:

        song_url = items.replace('//','')

        song_url=song_url.strip()

        print(song_url)

        p=subprocess.Popen('python you-get -d --format=HD {}'.format(song_url),stderr=subprocess.PIPE,stdout=subprocess.PIPE,shell=True)

        output,error = p.communicate()

        print(output)

        print(error)

        p.wait()

#-*-coding=utf-8-*-

import requests

from lxml import etree

import subprocess

session = requests.Session()

def getContent(url):

    # url='http://www.iqiyi.com/v_19rrkwcx6w.html'

    try:

        ret = session.get(url)

    # except Exception,e:

    except:

        # print e

        return None

    if ret.status_code==200:

        return ret.text

    else:

        return None



def getUrl():

    url='http://www.iqiyi.com/v_19rrkwcx6w.html'

    url2='http://www.iqiyi.com/v_19rrl2td7g.html' # 31-61

    content = getContent(url)

    root = etree.HTML(content)

    elements=root.xpath('//div[@data-current-count="1"]//li/a/@href')

    for items in elements:

        song_url = items.replace('//','')

        song_url=song_url.strip()

        print(song_url)

        p=subprocess.Popen('python you-get -d --format=HD {}'.format(song_url),stderr=subprocess.PIPE,stdout=subprocess.PIPE,shell=True)

        output,error = p.communicate()

        print(output)

        print(error)

        p.wait()



def main():

    getUrl()



if __name__ == '__main__':

    main()

#-*-coding=utf-8-*-

import sys,os

import requests

from lxml import etree

import subprocess

session = requests.Session()

def getContent(url):

    # url='http://www.iqiyi.com/v_19rrkwcx6w.html'

    try:

        ret = requests.get(url)

        ret.encoding='utf-8'

    # except Exception,e:

    except:

        # print e

        return None

    if ret.status_code==200:

        return ret.text

    else:

        return None



def getUrl():

    url='http://www.iqiyi.com/v_19rrkwcx6w.html'

    url2='http://www.iqiyi.com/v_19rrl2td7g.html' # 31-61

    content = getContent(url)

    if not content:

        print "network issue, retry"

        exit(0)

    root = etree.HTML(content,parser=etree.HTMLParser(encoding='utf-8'))

    elements=root.xpath('//div[@data-current-count="1"]//li')

    for items in elements:

        url_item=items.xpath('.//a/@href')[0]

        song_url = url_item.replace('//','')

        song_url=song_url.strip()

        print(song_url)

        # name=items.xpath('.//span[@class="item-num"]/text()')[0]

        name=items.xpath('.//span[@class="item-num"]/text()')[0].encode('utf-8').strip()+\

             ' '+items.xpath('.//span[@class="item-txt"]/text()')[0].encode('utf-8').strip()+'.mp4'

        name= '儿歌多多 '+name

        name=name.decode('utf-8')

        filename=os.path.join(os.getcwd(),name)

        print filename

        if os.path.exists(filename):

            continue

        p=subprocess.Popen('python you-get -d --format=HD {}'.format(song_url),stderr=subprocess.PIPE,stdout=subprocess.PIPE,shell=True)

        output,error = p.communicate()

        print(output)

        print(error)

        p.wait()





def main():

    getUrl()



if __name__ == '__main__':

    main()

# coding: utf-8

# http://30daydo.com

import urllib



import os

import requests

import time

from lxml import etree

from header_toolkit import getheader





def spider():

    curr=os.getcwd()

    target_dir=os.path.join(curr,'data')

    if not os.path.exists(target_dir):

        os.mkdir(target_dir)

    for i in range(1, 100, 10):

        url = 'http://www.lrts.me/ajax/playlist/2/32551/%d' % i

        headers = {

            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'}

        s = requests.get(url=url, headers=headers)

        tree = etree.HTML(s.text)

        nodes = tree.xpath('//*[starts-with(@class,"clearfix section-item section")]')

        print len(nodes)

        for node in nodes:

            filename = node.xpath('.//div[@class="column1 nowrap"]/span/text()')[0]

            link = node.xpath('.//input[@name="source" and @type="hidden"]/@value')[0]



            print link

            post_fix=link.split('.')[-1]

            full_path= filename+'.'+post_fix

            urllib.urlretrieve(link, filename=os.path.join(target_dir,full_path))

            time.sleep(1)





if __name__ == '__main__':

    spider()

#Get your range of csdn

'''

http://30daydo.com

contact: weigesysu@qq.com

'''

import urllib2,re

import time

link='http://blog.csdn.net/[b]用户名[/b]/article/details/52858314'

user_agent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)"

header = {"User-Agent": user_agent}

req = urllib2.Request(link, headers=header)

resp = urllib2.urlopen(req)

content = resp.read()

#print content

p=re.compile(r'<li>排名：<span>第(\d+)名</span></li>')

result=p.findall(content)

print result[0]



today=time.strftime("%Y-%m-%d")

print today



f=open("data/csdn_range.txt",'a')

contents=today+'\t'+result[0]+'\n'

f.write(contents)

f.close()

session=requests.session()

#先创建一个seession



s=session.get('http://xueqiu.com',headers=self.headers)

#随意建立一个访问雪球的session，此时的session就自带了雪球的cookies



url='https://xueqiu.com/snowmart/push/stocks.json?product_id=19&page=3&count=5'

headers['Referer']='https://xueqiu.com/strategy/19'

headers['X-Requested-With']='XMLHttpRequest'

headers['DNT']='1'



data={'product_id':19,'page':3,'count':5}



resp=session.get(url,headers=self.headers,params=data).text

#如果这里用的request.get的话，就不能获取到网页返回的正确内容

print resp

def csdn():

    session=requests.session()

    header={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36'}

    url='[url]http://msg.csdn.net/'[/url] 

    header['Cookie']='uuid_tt_dd=-5697318013068753627_20160111; _ga=GA1.2.795042232.1452766190; _message_m=quvq2wle24wa4awe UserInfo=zMhiNgesgIlEBQ3TOqLCtx4nUI360IIq3ciBzg4EKH%2FW8mSpTANpu5cTlRFLj2Tqh%2FZzQr2rNqDtT1SZz%2Be2%2FpDkGoQxDK3IVUZXvwZ%2FEP1I4UTg6MoZkH7LDO3sjrJJ; UserNick=%E9%87%8D%E5%A4%8D%E7%9A%84%E7%94%9F%E6%B4%BB; AU=0F1; UD=%E8%AE%B0%E5%BD%95%E8%87%AA%E5%AD%A6%E7%9A%84%E5%8E%86%E7%A8%8B+%E5%88%83%E8%8D%; BT=1490707396344; access-token=c2e12bff-5b27-4a91-953b-448ff6f6beac; _csdn_notify_admin_session=VE41a0d3TitrVGY2bGtXY09pZENwR1lHenhUU1NVaWc1b04wL1I3dCtDQVdadWpjMXBzdGRJL0RZR04wYldvZDBhTU96b2oycVVKeVI1UEVyUHFKbG1yNnB2b2pHRWVnWG1uc2JMM2R3YWthakRyTXZNaEpVU1NtUy9zQUJrNjd3R2lpbG5PK0paMnlyc1dyK0lTZUtRPT0tLXlPQUE1QzF5UmhDNjEvSFdtRFlQS2c9PQ%3D%3D--4569c5a32916dcf969a8b7e007c37abeb90be4f3; dc_tos=onj1dg; dc_session_id=1490707393375; Hm_lvt_6bcd52f51e9b3dce32bec4a3997715ac=1490024083,1490024559,1490374375,1490707368; Hm_lpvt_6bcd52f51e9b3dce32bec4a3997715ac=1490707638'

    header['Cache-Control']='max-age=0'

    header['Host']='msg.csdn.net'

    header['Referer']='http://blog.csdn.net/username'

    resp=requests.get(url,headers=header)

    print resp.text

 
上面这段代码，只要你改一下cookie和 Refer 的连接，改成你自己的用户名。 你就可以访问你的csdn的消息，也就是别人在你的csdn博客访问的的留言，评论。
 
只是这个cookie只能在某个时间段生效，也就是存活期大概就一个星期左右。

            content=requests.get(url,headers=self.header)

            content.encoding='gbk'

            print content.text

#-*-coding=utf-8-*-

__author__ = 'rocky'

#获取每天深圳一手房，二手房的成交套数与面积，并且写入数据库

#主要就是正则表达抓取几个数字

import urllib2,re

import database

def getContent():

    url="http://ris.szpl.gov.cn/"

    one_hand="credit/showcjgs/ysfcjgs.aspx"

    second_hand="credit/showcjgs/esfcjgs.aspx"

    req=urllib2.Request(url+one_hand)

    content=urllib2.urlopen(req).read()

    #返回的就是网页的源码，没有做任何防爬虫的处理，zf网站，呵呵

    #print content

    date=re.compile(r'<SPAN class=titleblue><span id=\"lblCurTime5\">(.*)</span>')

    reg=re.compile(r'<td width="14%"><b>(\d+)</b>')

    result=reg.findall(content)

    current_date=date.findall(content)



    reg2=re.compile(r'<td align="right"><b>(.*?)</b>')

    yishou_area=reg2.findall(content)





    print current_date[0]

    print '一手商品房成交套数：%s'  % result[0]

    print '一手商品房成交面积： %s'  % yishou_area[0]





    sec_req=urllib2.Request(url+second_hand)

    sec_content=urllib2.urlopen(sec_req).read()



    sec_quantity=re.compile(r'<td width="30%">(\d+)</td>')

    sec_result=sec_quantity.findall(sec_content)

    second_area=re.findall(r'<td align="right">(.*?)</td>',sec_content)



    print '二手商品房成交套数：%s'  % sec_result[1]

    print '二手商品房成交面积： %s'  % second_area[2]

    database.create_table()

    database.insert(current_date[0],result[0],yishou_area[0],sec_result[1],second_area[2])



getContent()

class getProxy():



    def __init__(self):

        self.user_agent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)"

        self.header = {"User-Agent": self.user_agent}

        self.dbname="proxy.db"

        self.now = time.strftime("%Y-%m-%d")



    def getContent(self, num):

        nn_url = "http://www.xicidaili.com/nn/" + str(num)

        #国内高匿

        req = urllib2.Request(nn_url, headers=self.header)

        resp = urllib2.urlopen(req, timeout=10)

        content = resp.read()

        et = etree.HTML(content)

        result_even = et.xpath('//tr[@class=""]')

        result_odd = et.xpath('//tr[@class="odd"]')

        #因为网页源码中class 分开了奇偶两个class，所以使用lxml最方便的方式就是分开获取。

        #刚开始我使用一个方式获取，因而出现很多不对称的情况，估计是网站会经常修改源码，怕被其他爬虫的抓到

        #使用上面的方法可以不管网页怎么改，都可以抓到ip 和port

        for i in result_even:

            t1 = i.xpath("./td/text()")[:2]

            print "IP:%s\tPort:%s" % (t1[0], t1[1])

            if self.isAlive(t1[0], t1[1]):



                self.insert_db(self.now,t1[0],t1[1])

        for i in result_odd:

            t2 = i.xpath("./td/text()")[:2]

            print "IP:%s\tPort:%s" % (t2[0], t2[1])

            if self.isAlive(t2[0], t2[1]):

                self.insert_db(self.now,t2[0],t2[1])

def insert_db(self,date,ip,port):

        dbname=self.dbname

        try:

            conn=sqlite3.connect(dbname)

        except:

            print "Error to open database%" %self.dbname

        create_tb='''

        CREATE TABLE IF NOT EXISTS PROXY

        (DATE TEXT,

        IP TEXT,

        PORT TEXT

        );

        '''

        conn.execute(create_tb)

        insert_db_cmd='''

        INSERT INTO PROXY (DATE,IP,PORT) VALUES ('%s','%s','%s');

        ''' %(date,ip,port) #写入时间，ip和端口

        conn.execute(insert_db_cmd)

        conn.commit() #记得commit

        conn.close()

 #查看爬到的代理IP是否还能用

    def isAlive(self,ip,port):

        proxy={'http':ip+':'+port}

        print proxy



        #使用这个方式是全局方法。

        proxy_support=urllib2.ProxyHandler(proxy)

        opener=urllib2.build_opener(proxy_support)

        urllib2.install_opener(opener)

        #使用代理访问腾讯官网，进行验证代理是否有效

        test_url="http://www.qq.com"

        req=urllib2.Request(test_url,headers=self.header)

        try:

            #timeout 设置为10，如果你不能忍受你的代理延时超过10，就修改timeout的数字

            resp=urllib2.urlopen(req,timeout=10)



            if resp.code==200:

                print "work"

                return True

            else:

                print "not work"

                return False

        except :

            print "Not work"

            return False

    def loop(self,page):

        for i in range(1,page):

            self.getContent(i)

if __name__ == "__main__":

    now = datetime.datetime.now()

    print "Start at %s" % now

    obj=getProxy()

    obj.loop(5)

__author__ = 'rocchen'

# working v1.0

from bs4 import BeautifulSoup

import urllib2, datetime, time, codecs, cookielib, random, threading

import os,sys





def getInfo(max_index_user=5):

    stock_news_site =

"http://ggjd.cnstock.com/gglist/search/ggkx/"

 

    my_userAgent = [

        'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',

        'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',

        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0',

        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)',

        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)',

        'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',

        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',

        'Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',

        'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11',

        'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11',

        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)',

        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',

        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)',

        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)']

    index = 0

    max_index = max_index_user

    num = 1

    temp_time = time.strftime("[%Y-%m-%d]-[%H-%M]", time.localtime())



    store_filename = "StockNews-%s.log" % temp_time

    fOpen = codecs.open(store_filename, 'w', 'utf-8')



    while index < max_index:

        user_agent = random.choice(my_userAgent)

        # print user_agent

        company_news_site = stock_news_site + str(index)

        # content = urllib2.urlopen(company_news_site)

        headers = {'User-Agent': user_agent, 'Host': "ggjd.cnstock.com", 'DNT': '1',

                   'Accept': 'text/html, application/xhtml+xml, */*', }

        req = urllib2.Request(url=company_news_site, headers=headers)

        resp = None

        raw_content = ""

        try:

            resp = urllib2.urlopen(req, timeout=30)



        except urllib2.HTTPError as e:

            e.fp.read()

        except urllib2.URLError as e:

            if hasattr(e, 'code'):

                print "error code %d" % e.code

            elif hasattr(e, 'reason'):

                print "error reason %s " % e.reason



        finally:

            if resp:

                raw_content = resp.read()

                time.sleep(2)

                resp.close()



        soup = BeautifulSoup(raw_content, "html.parser")

        all_content = soup.find_all("span", "time")



        for i in all_content:

            news_time = i.string

            node = i.next_sibling

            str_temp = "No.%s \n%s\t%s\n---> %s \n\n" % (str(num), news_time, node['title'], node['href'])

            #print "inside %d" %num

            #print str_temp

            fOpen.write(str_temp)

            num = num + 1



        #print "index %d" %index

        index = index + 1



    fOpen.close()





def execute_task(n=60):

    period = int(n)

    while True:

        print datetime.datetime.now()

        getInfo(3)

        

        time.sleep(60 * period)

        





if __name__ == "__main__":



    sub_folder = os.path.join(os.getcwd(), "stock")

    if not os.path.exists(sub_folder):

        os.mkdir(sub_folder)

    os.chdir(sub_folder)

    start_time = time.time()  # user can change the max index number getInfo(10), by default is getInfo(5)

    if len(sys.argv) <2:

        n = raw_input("Input Period : ? mins to download every cycle")

    else:

        n=int(sys.argv[1])

    execute_task(n)

    end_time = time.time()

    print "Total time: %s s." % str(round((end_time - start_time), 4))

#-*-coding=utf-8-*-

__author__ = 'rocky chen'

from bs4 import BeautifulSoup

import urllib2,sys,StringIO,gzip,time,random,re,urllib,os

reload(sys)

sys.setdefaultencoding('utf-8')

class Xitek():

    def __init__(self):

        self.url="http://photo.xitek.com/"

        user_agent="Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)"

        self.headers={"User-Agent":user_agent}

        self.last_page=self.__get_last_page()





    def __get_last_page(self):

        html=self.__getContentAuto(self.url)

        bs=BeautifulSoup(html,"html.parser")

        page=bs.find_all('a',class_="blast")

        last_page=page[0]['href'].split('/')[-1]

        return int(last_page)





    def __getContentAuto(self,url):

        req=urllib2.Request(url,headers=self.headers)

        resp=urllib2.urlopen(req)

        #time.sleep(2*random.random())

        content=resp.read()

        info=resp.info().get("Content-Encoding")

        if info==None:

            return content

        else:

            t=StringIO.StringIO(content)

            gziper=gzip.GzipFile(fileobj=t)

            html = gziper.read()

            return html



    #def __getFileName(self,stream):





    def __download(self,url):

        p=re.compile(r'href="(/photoid/\d+)"')

        #html=self.__getContentNoZip(url)



        html=self.__getContentAuto(url)



        content = p.findall(html)

        for i in content:

            print i



            photoid=self.__getContentAuto(self.url+i)

            bs=BeautifulSoup(photoid,"html.parser")

            final_link=bs.find('img',class_="mimg")['src']

            print final_link

            #pic_stream=self.__getContentAuto(final_link)

            title=bs.title.string.strip()

            filename = re.sub('[\/:*?"<>|]', '-', title)

            filename=filename+'.jpg'

            urllib.urlretrieve(final_link,filename)

            #f=open(filename,'w')

            #f.write(pic_stream)

            #f.close()

        #print html

        #bs=BeautifulSoup(html,"html.parser")

        #content=bs.find_all(p)

        #for i in content:

        #    print i

        '''

        print bs.title

        element_link=bs.find_all('div',class_="element")

        print len(element_link)

        k=1

        for href in element_link:



            #print type(href)

            #print href.tag

        '''

        '''

            if href.children[0]:

                print href.children[0]

        '''

        '''

            t=0



            for i in href.children:

                #if i.a:

                if t==0:

                    #print k

                    if i['href']

                    print link



                        if p.findall(link):

                            full_path=self.url[0:len(self.url)-1]+link

                            sub_html=self.__getContent(full_path)

                            bs=BeautifulSoup(sub_html,"html.parser")

                            final_link=bs.find('img',class_="mimg")['src']

                            #time.sleep(2*random.random())

                            print final_link

                    #k=k+1

                #print type(i)

                #print i.tag

                #if hasattr(i,"href"):

                    #print i['href']

                #print i.tag

                t=t+1

                #print "*"



        '''



        '''

            if href:

                if href.children:

                    print href.children[0]

        '''

            #print "one element link"







    def getPhoto(self):



        start=0

        #use style/0

        photo_url="http://photo.xitek.com/style/0/p/"

        for i in range(start,self.last_page+1):

            url=photo_url+str(i)

            print url

            #time.sleep(1)

            self.__download(url)



        '''

        url="http://photo.xitek.com/style/0/p/10"

        self.__download(url)

        '''

        #url="http://photo.xitek.com/style/0/p/0"

        #html=self.__getContent(url)

        #url="http://photo.xitek.com/"

        #html=self.__getContentNoZip(url)

        #print html

        #'''

def main():

    sub_folder = os.path.join(os.getcwd(), "content")

    if not os.path.exists(sub_folder):

        os.mkdir(sub_folder)

    os.chdir(sub_folder)

    obj=Xitek()

    obj.getPhoto()





if __name__=="__main__":

    main()

# -*- coding=utf-8 -*-

__author__ = 'rocky @ www.30daydo.com'

import urllib2, re, os, codecs,sys,datetime

from bs4 import BeautifulSoup

# example https://zhhrb.sinaapp.com/index.php?date=20160610

from mail_template import MailAtt

reload(sys)

sys.setdefaultencoding('utf-8')



def save2file(filename, content):

    filename = filename + ".txt"

    f = codecs.open(filename, 'a', encoding='utf-8')

    f.write(content)

    f.close()





def getPost(date_time, filter_p):

    url = 'https://zhhrb.sinaapp.com/index.php?date=' + date_time

    user_agent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)"

    header = {"User-Agent": user_agent}

    req = urllib2.Request(url, headers=header)

    resp = urllib2.urlopen(req)

    content = resp.read()

    p = re.compile('<h2 class="question-title">(.*)</h2></br></a>')

    result = re.findall(p, content)

    count = -1

    row = -1

    for i in result:

        #print i

        return_content = re.findall(filter_p, i)



        if return_content:

            row = count

            break

            #print return_content[0]

        count = count + 1

    #print row

    if row == -1:

        return 0

    link_p = re.compile('<a href="(.*)" target="_blank" rel="nofollow">')

    link_result = re.findall(link_p, content)[row + 1]

    print link_result

    result_req = urllib2.Request(link_result, headers=header)

    result_resp = urllib2.urlopen(result_req)

    #result_content= result_resp.read()

    #print result_content



    bs = BeautifulSoup(result_resp, "html.parser")

    title = bs.title.string.strip()

    #print title

    filename = re.sub('[\/:*?"<>|]', '-', title)

    print filename

    print date_time

    save2file(filename, title)

    save2file(filename, "\n\n\n\n--------------------%s Detail----------------------\n\n" %date_time)



    detail_content = bs.find_all('div', class_='content')



    for i in detail_content:

        #print i

        save2file(filename,"\n\n-------------------------answer  -------------------------\n\n")

        for j in i.strings:



            save2file(filename, j)



    smtp_server = 'smtp.126.com'

    from_mail = sys.argv[1]

    password = sys.argv[2]

    to_mail = 'jinweizsu@kindle.cn'

    send_kindle = MailAtt(smtp_server, from_mail, password, to_mail)

    send_kindle.send_txt(filename)





def main():

    sub_folder = os.path.join(os.getcwd(), "content")

    if not os.path.exists(sub_folder):

        os.mkdir(sub_folder)

    os.chdir(sub_folder)





    date_time = '20160611'

    filter_p = re.compile('大误.*')

    ori_day=datetime.date(datetime.date.today().year,01,01)

    t=datetime.date(datetime.date.today().year,datetime.date.today().month,datetime.date.today().day)

    delta=(t-ori_day).days

    print delta

    for i in range(delta):

        day=datetime.date(datetime.date.today().year,01,01)+datetime.timedelta(i)

        getPost(day.strftime("%Y%m%d"),filter_p)

    #getPost(date_time, filter_p)



if __name__ == "__main__":

    main()

# -*-coding=utf-8-*-

#抓取雪球的收藏文章

__author__ = 'Rocky'

import requests,cookielib,re,json,time

from toolkit import Toolkit

from lxml import etree

url='https://xueqiu.com/snowman/login'

session = requests.session()



session.cookies = cookielib.LWPCookieJar(filename="cookies")

try:

    session.cookies.load(ignore_discard=True)

except:

    print "Cookie can't load"



agent = 'Mozilla/5.0 (Windows NT 5.1; rv:33.0) Gecko/20100101 Firefox/33.0'

headers = {'Host': 'xueqiu.com',

           'Referer': 'https://xueqiu.com/',

           'Origin':'https://xueqiu.com',

           'User-Agent': agent}

account=Toolkit.getUserData('data.cfg')

print account['snowball_user']

print account['snowball_password']



data={'username':account['snowball_user'],'password':account['snowball_password']}

s=session.post(url,data=data,headers=headers)

print s.status_code

#print s.text

session.cookies.save()

fav_temp='https://xueqiu.com/favs?page=1'

collection=session.get(fav_temp,headers=headers)

fav_content= collection.text

p=re.compile('"maxPage":(\d+)')

maxPage=p.findall(fav_content)[0]

print maxPage

print type(maxPage)

maxPage=int(maxPage)

print type(maxPage)

for i in range(1,maxPage+1):

    fav='https://xueqiu.com/favs?page=%d' %i

    collection=session.get(fav,headers=headers)

    fav_content= collection.text

    #print fav_content

    p=re.compile('var favs = {(.*?)};',re.S|re.M)

    result=p.findall(fav_content)[0].strip()



    new_result='{'+result+'}'

    #print type(new_result)

    #print new_result

    data=json.loads(new_result)

    use_data= data['list']

    host='https://xueqiu.com'

    for i in use_data:

        url=host+ i['target']

        print url

        txt_content=session.get(url,headers=headers).text

        #print txt_content.text



        tree=etree.HTML(txt_content)

        title=tree.xpath('//title/text()')[0]



        filename = re.sub('[\/:*?"<>|]', '-', title)

        print filename



        content=tree.xpath('//div[@class="detail"]')

        for i in content:

            Toolkit.save2filecn(filename, i.xpath('string(.)'))

        #print content

        #Toolkit.save2file(filename,)

        time.sleep(10)