#-*-coding=utf-8-*-

__author__ = 'rocky chen'

from bs4 import BeautifulSoup

import urllib2,sys,StringIO,gzip,time,random,re,urllib,os

reload(sys)

sys.setdefaultencoding('utf-8')

class Xitek():

    def __init__(self):

        self.url="http://photo.xitek.com/"

        user_agent="Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)"

        self.headers={"User-Agent":user_agent}

        self.last_page=self.__get_last_page()





    def __get_last_page(self):

        html=self.__getContentAuto(self.url)

        bs=BeautifulSoup(html,"html.parser")

        page=bs.find_all('a',class_="blast")

        last_page=page[0]['href'].split('/')[-1]

        return int(last_page)





    def __getContentAuto(self,url):

        req=urllib2.Request(url,headers=self.headers)

        resp=urllib2.urlopen(req)

        #time.sleep(2*random.random())

        content=resp.read()

        info=resp.info().get("Content-Encoding")

        if info==None:

            return content

        else:

            t=StringIO.StringIO(content)

            gziper=gzip.GzipFile(fileobj=t)

            html = gziper.read()

            return html



    #def __getFileName(self,stream):





    def __download(self,url):

        p=re.compile(r'href="(/photoid/\d+)"')

        #html=self.__getContentNoZip(url)



        html=self.__getContentAuto(url)



        content = p.findall(html)

        for i in content:

            print i



            photoid=self.__getContentAuto(self.url+i)

            bs=BeautifulSoup(photoid,"html.parser")

            final_link=bs.find('img',class_="mimg")['src']

            print final_link

            #pic_stream=self.__getContentAuto(final_link)

            title=bs.title.string.strip()

            filename = re.sub('[\/:*?"<>|]', '-', title)

            filename=filename+'.jpg'

            urllib.urlretrieve(final_link,filename)

            #f=open(filename,'w')

            #f.write(pic_stream)

            #f.close()

        #print html

        #bs=BeautifulSoup(html,"html.parser")

        #content=bs.find_all(p)

        #for i in content:

        #    print i

        '''

        print bs.title

        element_link=bs.find_all('div',class_="element")

        print len(element_link)

        k=1

        for href in element_link:



            #print type(href)

            #print href.tag

        '''

        '''

            if href.children[0]:

                print href.children[0]

        '''

        '''

            t=0



            for i in href.children:

                #if i.a:

                if t==0:

                    #print k

                    if i['href']

                    print link



                        if p.findall(link):

                            full_path=self.url[0:len(self.url)-1]+link

                            sub_html=self.__getContent(full_path)

                            bs=BeautifulSoup(sub_html,"html.parser")

                            final_link=bs.find('img',class_="mimg")['src']

                            #time.sleep(2*random.random())

                            print final_link

                    #k=k+1

                #print type(i)

                #print i.tag

                #if hasattr(i,"href"):

                    #print i['href']

                #print i.tag

                t=t+1

                #print "*"



        '''



        '''

            if href:

                if href.children:

                    print href.children[0]

        '''

            #print "one element link"







    def getPhoto(self):



        start=0

        #use style/0

        photo_url="http://photo.xitek.com/style/0/p/"

        for i in range(start,self.last_page+1):

            url=photo_url+str(i)

            print url

            #time.sleep(1)

            self.__download(url)



        '''

        url="http://photo.xitek.com/style/0/p/10"

        self.__download(url)

        '''

        #url="http://photo.xitek.com/style/0/p/0"

        #html=self.__getContent(url)

        #url="http://photo.xitek.com/"

        #html=self.__getContentNoZip(url)

        #print html

        #'''

def main():

    sub_folder = os.path.join(os.getcwd(), "content")

    if not os.path.exists(sub_folder):

        os.mkdir(sub_folder)

    os.chdir(sub_folder)

    obj=Xitek()

    obj.getPhoto()





if __name__=="__main__":

    main()

    list=[1,2,3,5,4,6,434,2323,333,99999]

    print "max of list is ",

    print max(list)

#-*-coding=utf-8-*-

__author__ = 'rocchen'

from lxml import html

from lxml import etree

import urllib2



def lxml_test():

    url="http://www.caixunzz.com"

    req=urllib2.Request(url=url)

    resp=urllib2.urlopen(req)

    #print resp.read()



    tree=etree.HTML(resp.read())

    href=tree.xpath('//a[@class="label"]/@href')

    #print href.tag

    for i in href:

        #print html.tostring(i)

        #print type(i)

        print i



    print type(href)



lxml_test()

可以在终端里敲入以下命令



alec@PC$ git config --global color.status auto  

alec@PC$ git config --global color.diff auto  

alec@PC$ git config --global color.branch auto  

alec@PC$ git config --global color.interactive auto 

>>> x = { }

>>> y = x

>>> x = { 'a' : 'b' }

>>> y

>>> { }

>>> x = { }

>>> y = x

>>> x['a'] = 'b'

>>> y

>>> { 'a' : 'b' }

# Copyright (c) 2009 rupa deadwyler under the WTFPL license



# maintains a jump-list of the directories you actually use

#

# INSTALL:

#     * put something like this in your .bashrc/.zshrc:

#         . /path/to/z.sh

#     * cd around for a while to build up the db

#     * PROFIT!!

#     * optionally:

#         set $_Z_CMD in .bashrc/.zshrc to change the command (default z).

#         set $_Z_DATA in .bashrc/.zshrc to change the datafile (default ~/.z).

#         set $_Z_NO_RESOLVE_SYMLINKS to prevent symlink resolution.

#         set $_Z_NO_PROMPT_COMMAND if you're handling PROMPT_COMMAND yourself.

#         set $_Z_EXCLUDE_DIRS to an array of directories to exclude.

#         set $_Z_OWNER to your username if you want use z while sudo with $HOME kept

#

# USE:

#     * z foo     # cd to most frecent dir matching foo

#     * z foo bar # cd to most frecent dir matching foo and bar

#     * z -r foo  # cd to highest ranked dir matching foo

#     * z -t foo  # cd to most recently accessed dir matching foo

#     * z -l foo  # list matches instead of cd

#     * z -c foo  # restrict matches to subdirs of $PWD



[ -d "${_Z_DATA:-$HOME/.z}" ] && {

    echo "ERROR: z.sh's datafile (${_Z_DATA:-$HOME/.z}) is a directory."

}



_z() {



    local datafile="${_Z_DATA:-$HOME/.z}"



    # bail if we don't own ~/.z and $_Z_OWNER not set

    [ -z "$_Z_OWNER" -a -f "$datafile" -a ! -O "$datafile" ] && return



    # add entries

    if [ "$1" = "--add" ]; then

        shift



        # $HOME isn't worth matching

        [ "$*" = "$HOME" ] && return



        # don't track excluded directory trees

        local exclude

        for exclude in "${_Z_EXCLUDE_DIRS[@]}"; do

            case "$*" in "$exclude*") return;; esac

        done



        # maintain the data file

        local tempfile="$datafile.$RANDOM"

        while read line; do

            # only count directories

            [ -d "${line%%\|*}" ] && echo $line

        done < "$datafile" | awk -v path="$*" -v now="$(date +%s)" -F"|" '

            BEGIN {

                rank[path] = 1

                time[path] = now

            }

            $2 >= 1 {

                # drop ranks below 1

                if( $1 == path ) {

                    rank[$1] = $2 + 1

                    time[$1] = now

                } else {

                    rank[$1] = $2

                    time[$1] = $3

                }

                count += $2

            }

            END {

                if( count > 9000 ) {

                    # aging

                    for( x in rank ) print x "|" 0.99*rank[x] "|" time[x]

                } else for( x in rank ) print x "|" rank[x] "|" time[x]

            }

        ' 2>/dev/null >| "$tempfile"

        # do our best to avoid clobbering the datafile in a race condition

        if [ $? -ne 0 -a -f "$datafile" ]; then

            env rm -f "$tempfile"

        else

            [ "$_Z_OWNER" ] && chown $_Z_OWNER:$(id -ng $_Z_OWNER) "$tempfile"

            env mv -f "$tempfile" "$datafile" || env rm -f "$tempfile"

        fi



    # tab completion

    elif [ "$1" = "--complete" -a -s "$datafile" ]; then

        while read line; do

            [ -d "${line%%\|*}" ] && echo $line

        done < "$datafile" | awk -v q="$2" -F"|" '

            BEGIN {

                if( q == tolower(q) ) imatch = 1

                q = substr(q, 3)

                gsub(" ", ".*", q)

            }

            {

                if( imatch ) {

                    if( tolower($1) ~ tolower(q) ) print $1

                } else if( $1 ~ q ) print $1

            }

        ' 2>/dev/null



    else

        # list/go

        while [ "$1" ]; do case "$1" in

            --) while [ "$1" ]; do shift; local fnd="$fnd${fnd:+ }$1";done;;

            -*) local opt=${1:1}; while [ "$opt" ]; do case ${opt:0:1} in

                    c) local fnd="^$PWD $fnd";;

                    h) echo "${_Z_CMD:-z} [-chlrtx] args" >&2; return;;

                    x) sed -i -e "\:^${PWD}|.*:d" "$datafile";;

                    l) local list=1;;

                    r) local typ="rank";;

                    t) local typ="recent";;

                esac; opt=${opt:1}; done;;

             *) local fnd="$fnd${fnd:+ }$1";;

        esac; local last=$1; [ "$#" -gt 0 ] && shift; done

        [ "$fnd" -a "$fnd" != "^$PWD " ] || local list=1



        # if we hit enter on a completion just go there

        case "$last" in

            # completions will always start with /

            /*) [ -z "$list" -a -d "$last" ] && cd "$last" && return;;

        esac



        # no file yet

        [ -f "$datafile" ] || return



        local cd

        cd="$(while read line; do

            [ -d "${line%%\|*}" ] && echo $line

        done < "$datafile" | awk -v t="$(date +%s)" -v list="$list" -v typ="$typ" -v q="$fnd" -F"|" '

            function frecent(rank, time) {

                # relate frequency and time

                dx = t - time

                if( dx < 3600 ) return rank * 4

                if( dx < 86400 ) return rank * 2

                if( dx < 604800 ) return rank / 2

                return rank / 4

            }

            function output(files, out, common) {

                # list or return the desired directory

                if( list ) {

                    cmd = "sort -n >&2"

                    for( x in files ) {

                        if( files[x] ) printf "%-10s %s\n", files[x], x | cmd

                    }

                    if( common ) {

                        printf "%-10s %s\n", "common:", common > "/dev/stderr"

                    }

                } else {

                    if( common ) out = common

                    print out

                }

            }

            function common(matches) {

                # find the common root of a list of matches, if it exists

                for( x in matches ) {

                    if( matches[x] && (!short || length(x) < length(short)) ) {

                        short = x

                    }

                }

                if( short == "/" ) return

                # use a copy to escape special characters, as we want to return

                # the original. yeah, this escaping is awful.

                clean_short = short

                gsub(/\[\(\)\[\]\|\]/, "\\\\&", clean_short)

                for( x in matches ) if( matches[x] && x !~ clean_short ) return

                return short

            }

            BEGIN {

                gsub(" ", ".*", q)

                hi_rank = ihi_rank = -9999999999

            }

            {

                if( typ == "rank" ) {

                    rank = $2

                } else if( typ == "recent" ) {

                    rank = $3 - t

                } else rank = frecent($2, $3)

                if( $1 ~ q ) {

                    matches[$1] = rank

                } else if( tolower($1) ~ tolower(q) ) imatches[$1] = rank

                if( matches[$1] && matches[$1] > hi_rank ) {

                    best_match = $1

                    hi_rank = matches[$1]

                } else if( imatches[$1] && imatches[$1] > ihi_rank ) {

                    ibest_match = $1

                    ihi_rank = imatches[$1]

                }

            }

            END {

                # prefer case sensitive

                if( best_match ) {

                    output(matches, best_match, common(matches))

                } else if( ibest_match ) {

                    output(imatches, ibest_match, common(imatches))

                }

            }

        ')"

        [ $? -gt 0 ] && return

        [ "$cd" ] && cd "$cd"

    fi

}



alias ${_Z_CMD:-z}='_z 2>&1'



[ "$_Z_NO_RESOLVE_SYMLINKS" ] || _Z_RESOLVE_SYMLINKS="-P"



if type compctl >/dev/null 2>&1; then

    # zsh

    [ "$_Z_NO_PROMPT_COMMAND" ] || {

        # populate directory list, avoid clobbering any other precmds.

        if [ "$_Z_NO_RESOLVE_SYMLINKS" ]; then

            _z_precmd() {

                _z --add "${PWD:a}"

            }

        else

            _z_precmd() {

                _z --add "${PWD:A}"

            }

        fi

        [[ -n "${precmd_functions[(r)_z_precmd]}" ]] || {

            precmd_functions[$(($#precmd_functions+1))]=_z_precmd

        }

    }

    _z_zsh_tab_completion() {

        # tab completion

        local compl

        read -l compl

        reply=(${(f)"$(_z --complete "$compl")"})

    }

    compctl -U -K _z_zsh_tab_completion _z

elif type complete >/dev/null 2>&1; then

    # bash

    # tab completion

    complete -o filenames -C '_z --complete "$COMP_LINE"' ${_Z_CMD:-z}

    [ "$_Z_NO_PROMPT_COMMAND" ] || {

        # populate directory list. avoid clobbering other PROMPT_COMMANDs.

        grep "_z --add" <<< "$PROMPT_COMMAND" >/dev/null || {

            PROMPT_COMMAND="$PROMPT_COMMAND"$'\n''_z --add "$(command pwd '$_Z_RESOLVE_SYMLINKS' 2>/dev/null)" 2>/dev/null;'

        }

    }

fi

p = subprocess.Popen(cmd, stdout = subprocess.PIPE, stdin = subprocess.PIPE, shell = True)

p.wait()

// hang here

print "finished"

p = subprocess.Popen(cmd, stdout = subprocess.PIPE, stdin = subprocess.PIPE, shell = True)

stdout, stderr = p.communicate()

p.wait()

print "Finsih"

# -*- coding=utf-8 -*-

__author__ = 'rocky @ www.30daydo.com'

import urllib2, re, os, codecs,sys,datetime

from bs4 import BeautifulSoup

# example https://zhhrb.sinaapp.com/index.php?date=20160610

from mail_template import MailAtt

reload(sys)

sys.setdefaultencoding('utf-8')



def save2file(filename, content):

    filename = filename + ".txt"

    f = codecs.open(filename, 'a', encoding='utf-8')

    f.write(content)

    f.close()





def getPost(date_time, filter_p):

    url = 'https://zhhrb.sinaapp.com/index.php?date=' + date_time

    user_agent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)"

    header = {"User-Agent": user_agent}

    req = urllib2.Request(url, headers=header)

    resp = urllib2.urlopen(req)

    content = resp.read()

    p = re.compile('<h2 class="question-title">(.*)</h2></br></a>')

    result = re.findall(p, content)

    count = -1

    row = -1

    for i in result:

        #print i

        return_content = re.findall(filter_p, i)



        if return_content:

            row = count

            break

            #print return_content[0]

        count = count + 1

    #print row

    if row == -1:

        return 0

    link_p = re.compile('<a href="(.*)" target="_blank" rel="nofollow">')

    link_result = re.findall(link_p, content)[row + 1]

    print link_result

    result_req = urllib2.Request(link_result, headers=header)

    result_resp = urllib2.urlopen(result_req)

    #result_content= result_resp.read()

    #print result_content



    bs = BeautifulSoup(result_resp, "html.parser")

    title = bs.title.string.strip()

    #print title

    filename = re.sub('[\/:*?"<>|]', '-', title)

    print filename

    print date_time

    save2file(filename, title)

    save2file(filename, "\n\n\n\n--------------------%s Detail----------------------\n\n" %date_time)



    detail_content = bs.find_all('div', class_='content')



    for i in detail_content:

        #print i

        save2file(filename,"\n\n-------------------------answer  -------------------------\n\n")

        for j in i.strings:



            save2file(filename, j)



    smtp_server = 'smtp.126.com'

    from_mail = sys.argv[1]

    password = sys.argv[2]

    to_mail = 'xxxxx@kindle.cn'

    send_kindle = MailAtt(smtp_server, from_mail, password, to_mail)

    send_kindle.send_txt(filename)





def main():

    sub_folder = os.path.join(os.getcwd(), "content")

    if not os.path.exists(sub_folder):

        os.mkdir(sub_folder)

    os.chdir(sub_folder)





    date_time = '20160611'

    filter_p = re.compile('大误.*')

    ori_day=datetime.date(datetime.date.today().year,01,01)

    t=datetime.date(datetime.date.today().year,datetime.date.today().month,datetime.date.today().day)

    delta=(t-ori_day).days

    print delta

    for i in range(delta):

        day=datetime.date(datetime.date.today().year,01,01)+datetime.timedelta(i)

        getPost(day.strftime("%Y%m%d"),filter_p)

    #getPost(date_time, filter_p)



if __name__ == "__main__":

    main()

import os



def isContain(des_str,ori_str):

	for i in des_str:

		if ori_str == i:

			return True

	return False





path=os.getcwd()

print path

des_str=['img','cfg','bct','bin','sh','dtb','txt','mk','pem','mk','pk8','xml','lib','pl','blob','dat']

for fpath,dirs,fname in os.walk(path):

	#print fname

	

	if fname:

		for i in fname:

			#print i

			name=i.split('.')

			if len(name)>=2:

				#print name[1]

				if isContain(des_str,name[1]):

					filepath=os.path.join(fpath,i)

					print "delete file %s" %filepath

					os.remove(filepath)