如果你因心愿没实现，而正在心灰意冷，为宏大的目标而拼搏过的这段经历，将来一定会成为你的力量。



所以，用你那总是满不在乎的笑脸，请继续挺起胸膛充满信心地走下去。

#-*-coding=utf-8-*-

__author__ = 'rocky chen'

from uiautomator import device as d

import time,subprocess,re,os



def zip_log(filename):

    filename="count_%d.log" %filename

    cmd='zip %s.zip %s' %(filename,filename)

    p=subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)

    out,err=p.communicate()

    p.wait()



    os.remove(filename)



def move_operation(action_key):

    cmd='adb shell input keyevent %s' %action_key

    p=subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)

    out,err=p.communicate()

    p.wait()

    print out,err



def basic_info():

    for k, v in d.info.items():

        print k,v

    #print info



def check_connect():

    cmd='adb shell ping www.qq.com'

    p=subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)

    out,err=p.communicate()

    p.wait()

    print out,err



def wifi_connect(count):

    print "WIFI connect in LOOP %d" %count

    d.press.home()

    #move_operation('KEYCODE_HOME')

    d.press.down()



    #move_operation('KEYCODE_DPAD_DOWN')



    d.press.down()

    #move_operation('KEYCODE_DPAD_DOWN')



    d.press.down()

    #move_operation('KEYCODE_DPAD_DOWN')



    d.press.right()

    #move_operation('KEYCODE_DPAD_RIGHT')



    d.press.enter()

    #move_operation('KEYCODE_ENTER')



    d.press.enter()

    #move_operation('KEYCODE_ENTER')

    d.press.down()

    #move_operation('KEYCODE_DPAD_DOWN')

    d.press.down()

    #move_operation('KEYCODE_DPAD_DOWN')

    d.press.down()

    #move_operation('KEYCODE_DPAD_DOWN')

    d.press.down()

    #move_operation('KEYCODE_DPAD_DOWN')

    d.press.down()

    #move_operation('KEYCODE_DPAD_DOWN')

    d.press.enter()

    #move_operation('KEYCODE_ENTER')



    cmd='adb shell input text xiaomi2g'

    p=subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)

    out,err=p.communicate()

    cmd='adb shell input keyevent KEYCODE_ESCAPE'

    p=subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)

    out,err=p.communicate()

    cmd='adb shell input keyevent KEYCODE_ENTER'

    p=subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)

    out,err=p.communicate()



    d.press.down()

    d.press.down()

    #move_operation('KEYCODE_DPAD_DOWN')

    #d.press.down()

    d.press.enter()

    #move_operation('KEYCODE_ENTER')



    cmd='adb shell input text test1234'

    p=subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)

    out,err=p.communicate()

    p.wait()



    cmd='adb shell input keyevent KEYCODE_ENTER'

    p=subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)

    out,err=p.communicate()

    p.wait()

    time.sleep(25)







def check_wifi_list(count):

    print "check wifi list in loop %d" %count

    d.press.up()

    d.press.up()

    d.press.up()

    d.press.up()

    d.press.up()

    d.press.enter()

    d.press.enter()

    time.sleep(3)

    print "########### IN LOOP %d  #################" %count

    cmd='adb shell screencap -p /sdcard/%d.png' %count

    p=subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)

    out,err=p.communicate()

    p.wait()



    cmd='adb pull /sdcard/%d.png .' %count

    p=subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)

    out,err=p.communicate()

    p.wait()







def check_connection(count):

    print "check connection %d" %count

    fp=open("capture.log",'w')

    fp.write('\n')

    cmd='adb shell ping -c 4 www.baidu.com'

    p=subprocess.Popen(cmd,stdout=fp,stderr=subprocess.PIPE,shell=True)

    out,err=p.communicate()

    print out

    print err

    p.wait()

    fp.flush()

    fp.close()

    fp=open("capture.log",'r')

    return_data=fp.read()

    #print return_data

    pettern=re.compile(r'100\% packet loss')

    t=pettern.search(return_data)

    #print t

    if t:

        print "Failed"

    else:

        print "Passed"



    try:

        os.remove("capture.log")



    except:

        print "Delete capture.log failed"





def reboot_device(count):

    print "reboot device in loop %d" %count

    cmd='adb reboot'

    p=subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)

    out,err=p.communicate()

    print out,err

    p.wait()



    cmd='adb wait-for-device'

    p=subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)

    out,err=p.communicate()

    print out,err

    p.wait()



    print "Reboot done"

    time.sleep(60)



def forget_password(count):

    print "Forget wifi in loop %d" %count

    d.press.home()

    d.press.down()

    d.press.down()

    d.press.down()

    d.press.right()

    d.press.enter()

    d.press.enter()

    d.press.enter()

    d.press.down()

    d.press.down()

    d.press.enter()

    d.press.enter()

    time.sleep(1)



def get_log(count):



    cmd="adb logcat -v time >count_%d.log &" %count

    '''p=subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)

    out,err=p.communicate()

    print out,err

    '''

    os.system(cmd)



if __name__=="__main__":



    #check_connection()

    #get_log(2)

    #basic_info()





    for i in range(3000):

        reboot_device(i)

        wifi_connect(i)

        get_log(i)

        check_wifi_list(i)

        check_connection(i)

        forget_password(i)

        zip_log(i)



    #move_operation('KEYCODE_HOME')

    #move_operation('KEYCODE_DPAD_DOWN')

#-*-coding=utf-8-*-

__author__ = 'rocky'

#获取每天深圳一手房，二手房的成交套数与面积，并且写入数据库

#主要就是正则表达抓取几个数字

import urllib2,re

import database

def getContent():

    url="http://ris.szpl.gov.cn/"

    one_hand="credit/showcjgs/ysfcjgs.aspx"

    second_hand="credit/showcjgs/esfcjgs.aspx"

    req=urllib2.Request(url+one_hand)

    content=urllib2.urlopen(req).read()

    #返回的就是网页的源码，没有做任何防爬虫的处理，zf网站，呵呵

    #print content

    date=re.compile(r'<SPAN class=titleblue><span id=\"lblCurTime5\">(.*)</span>')

    reg=re.compile(r'<td width="14%"><b>(\d+)</b>')

    result=reg.findall(content)

    current_date=date.findall(content)



    reg2=re.compile(r'<td align="right"><b>(.*?)</b>')

    yishou_area=reg2.findall(content)





    print current_date[0]

    print '一手商品房成交套数：%s'  % result[0]

    print '一手商品房成交面积： %s'  % yishou_area[0]





    sec_req=urllib2.Request(url+second_hand)

    sec_content=urllib2.urlopen(sec_req).read()



    sec_quantity=re.compile(r'<td width="30%">(\d+)</td>')

    sec_result=sec_quantity.findall(sec_content)

    second_area=re.findall(r'<td align="right">(.*?)</td>',sec_content)



    print '二手商品房成交套数：%s'  % sec_result[1]

    print '二手商品房成交面积： %s'  % second_area[2]

    database.create_table()

    database.insert(current_date[0],result[0],yishou_area[0],sec_result[1],second_area[2])



getContent()

class getProxy():



    def __init__(self):

        self.user_agent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)"

        self.header = {"User-Agent": self.user_agent}

        self.dbname="proxy.db"

        self.now = time.strftime("%Y-%m-%d")



    def getContent(self, num):

        nn_url = "http://www.xicidaili.com/nn/" + str(num)

        #国内高匿

        req = urllib2.Request(nn_url, headers=self.header)

        resp = urllib2.urlopen(req, timeout=10)

        content = resp.read()

        et = etree.HTML(content)

        result_even = et.xpath('//tr[@class=""]')

        result_odd = et.xpath('//tr[@class="odd"]')

        #因为网页源码中class 分开了奇偶两个class，所以使用lxml最方便的方式就是分开获取。

        #刚开始我使用一个方式获取，因而出现很多不对称的情况，估计是网站会经常修改源码，怕被其他爬虫的抓到

        #使用上面的方法可以不管网页怎么改，都可以抓到ip 和port

        for i in result_even:

            t1 = i.xpath("./td/text()")[:2]

            print "IP:%s\tPort:%s" % (t1[0], t1[1])

            if self.isAlive(t1[0], t1[1]):



                self.insert_db(self.now,t1[0],t1[1])

        for i in result_odd:

            t2 = i.xpath("./td/text()")[:2]

            print "IP:%s\tPort:%s" % (t2[0], t2[1])

            if self.isAlive(t2[0], t2[1]):

                self.insert_db(self.now,t2[0],t2[1])

def insert_db(self,date,ip,port):

        dbname=self.dbname

        try:

            conn=sqlite3.connect(dbname)

        except:

            print "Error to open database%" %self.dbname

        create_tb='''

        CREATE TABLE IF NOT EXISTS PROXY

        (DATE TEXT,

        IP TEXT,

        PORT TEXT

        );

        '''

        conn.execute(create_tb)

        insert_db_cmd='''

        INSERT INTO PROXY (DATE,IP,PORT) VALUES ('%s','%s','%s');

        ''' %(date,ip,port) #写入时间，ip和端口

        conn.execute(insert_db_cmd)

        conn.commit() #记得commit

        conn.close()

 #查看爬到的代理IP是否还能用

    def isAlive(self,ip,port):

        proxy={'http':ip+':'+port}

        print proxy



        #使用这个方式是全局方法。

        proxy_support=urllib2.ProxyHandler(proxy)

        opener=urllib2.build_opener(proxy_support)

        urllib2.install_opener(opener)

        #使用代理访问腾讯官网，进行验证代理是否有效

        test_url="http://www.qq.com"

        req=urllib2.Request(test_url,headers=self.header)

        try:

            #timeout 设置为10，如果你不能忍受你的代理延时超过10，就修改timeout的数字

            resp=urllib2.urlopen(req,timeout=10)



            if resp.code==200:

                print "work"

                return True

            else:

                print "not work"

                return False

        except :

            print "Not work"

            return False

    def loop(self,page):

        for i in range(1,page):

            self.getContent(i)

if __name__ == "__main__":

    now = datetime.datetime.now()

    print "Start at %s" % now

    obj=getProxy()

    obj.loop(5)

#判断表存不存在来创建表

def create_table():



    conn = sqlite3.connect(db_name)

    try:

        create_tb_cmd='''

        CREATE TABLE IF NOT EXISTS USER

        (NAME TEXT,

        AGE INT,

        SALARY REAL);

        '''

        #主要就是上面的语句

        conn.execute(create_tb_cmd)

    except:

        print "Create table failed"

        return False

    insert_dt_cmd='''

    INSERT INTO USER (NAME,AGE,SALARY) VALUES ("Jack",10,20.1);

    '''

    conn.execute(insert_dt_cmd)

    conn.commit()

    conn.close()