__author__ = 'rocchen'

# working v1.0

from bs4 import BeautifulSoup

import urllib2, datetime, time, codecs, cookielib, random, threading

import os,sys





def getInfo(max_index_user=5):

    stock_news_site =

"http://ggjd.cnstock.com/gglist/search/ggkx/"

 

    my_userAgent = [

        'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',

        'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',

        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0',

        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)',

        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)',

        'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',

        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',

        'Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',

        'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11',

        'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11',

        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)',

        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',

        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)',

        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)']

    index = 0

    max_index = max_index_user

    num = 1

    temp_time = time.strftime("[%Y-%m-%d]-[%H-%M]", time.localtime())



    store_filename = "StockNews-%s.log" % temp_time

    fOpen = codecs.open(store_filename, 'w', 'utf-8')



    while index < max_index:

        user_agent = random.choice(my_userAgent)

        # print user_agent

        company_news_site = stock_news_site + str(index)

        # content = urllib2.urlopen(company_news_site)

        headers = {'User-Agent': user_agent, 'Host': "ggjd.cnstock.com", 'DNT': '1',

                   'Accept': 'text/html, application/xhtml+xml, */*', }

        req = urllib2.Request(url=company_news_site, headers=headers)

        resp = None

        raw_content = ""

        try:

            resp = urllib2.urlopen(req, timeout=30)



        except urllib2.HTTPError as e:

            e.fp.read()

        except urllib2.URLError as e:

            if hasattr(e, 'code'):

                print "error code %d" % e.code

            elif hasattr(e, 'reason'):

                print "error reason %s " % e.reason



        finally:

            if resp:

                raw_content = resp.read()

                time.sleep(2)

                resp.close()



        soup = BeautifulSoup(raw_content, "html.parser")

        all_content = soup.find_all("span", "time")



        for i in all_content:

            news_time = i.string

            node = i.next_sibling

            str_temp = "No.%s \n%s\t%s\n---> %s \n\n" % (str(num), news_time, node['title'], node['href'])

            #print "inside %d" %num

            #print str_temp

            fOpen.write(str_temp)

            num = num + 1



        #print "index %d" %index

        index = index + 1



    fOpen.close()





def execute_task(n=60):

    period = int(n)

    while True:

        print datetime.datetime.now()

        getInfo(3)

        

        time.sleep(60 * period)

        





if __name__ == "__main__":



    sub_folder = os.path.join(os.getcwd(), "stock")

    if not os.path.exists(sub_folder):

        os.mkdir(sub_folder)

    os.chdir(sub_folder)

    start_time = time.time()  # user can change the max index number getInfo(10), by default is getInfo(5)

    if len(sys.argv) <2:

        n = raw_input("Input Period : ? mins to download every cycle")

    else:

        n=int(sys.argv[1])

    execute_task(n)

    end_time = time.time()

    print "Total time: %s s." % str(round((end_time - start_time), 4))

Don't use BeautifulSoup, use lxml.soupparser then you're sitting on top of the power of lxml and can use the good bits of BeautifulSoup which is to deal with really broken and crappy HTML.
 
 
 
9down vote
In summary, 

lxml

 is positioned as a lightning-fast production-quality html and xml parser that, by the way, also includes a 

soupparser

 module to fall back on BeautifulSoup's functionality. 

BeautifulSoup

is a one-person project, designed to save you time to quickly extract data out of poorly-formed html or xml.
lxml documentation says that both parsers have advantages and disadvantages. For this reason, 

lxml

 provides a 

soupparser

 so you can switch back and forth. Quoting,
[quote]
BeautifulSoup uses a different parsing approach. It is not a real HTML parser but uses regular expressions to dive through tag soup. It is therefore more forgiving in some cases and less good in others. It is not uncommon that lxml/libxml2 parses and fixes broken HTML better, but BeautifulSoup has superiour support for encoding detection. It very much depends on the input which parser works better.

The downside of using this parser is that it is much slower than the HTML parser of lxml. So if performance matters, you might want to consider using soupparser only as a fallback for certain cases.

lxml

BeautifulSoup

soupparser

lxml

BeautifulSoup

lxml

[code]>>> from BeautifulSoup import UnicodeDammit



>>> def decode_html(html_string):

...     converted = UnicodeDammit(html_string, isHTML=True)

...     if not converted.unicode:

...         raise UnicodeDecodeError(

...             "Failed to detect encoding, tried [%s]",

...             ', '.join(converted.triedEncodings))

...     # print converted.originalEncoding

...     return converted.unicode



>>> root = lxml.html.fromstring(decode_html(tag_soup))

BeautifulSoup

That's it! Have fun! I wrote Beautiful Soup to save everybody time. Once you get used to it, you should be able to wrangle data out of poorly-designed websites in just a few minutes. Send me email if you have any comments, run into problems, or want me to know about your project that uses Beautiful Soup.

[code] --Leonard

[/code]

lxml has been downloaded from the Python Package Index more than two million times and is also available directly in many package distributions, e.g. for Linux or MacOS-X.

The C libraries libxml2 and libxslt have huge benefits:... Standards-compliant... Full-featured... fast. fast! FAST! ... lxml is a new Python binding for libxml2 and libxslt...

#-*-coding=utf-8-*-

__author__ = 'rocky chen'

from bs4 import BeautifulSoup

import urllib2,sys,StringIO,gzip,time,random,re,urllib,os

reload(sys)

sys.setdefaultencoding('utf-8')

class Xitek():

    def __init__(self):

        self.url="http://photo.xitek.com/"

        user_agent="Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)"

        self.headers={"User-Agent":user_agent}

        self.last_page=self.__get_last_page()





    def __get_last_page(self):

        html=self.__getContentAuto(self.url)

        bs=BeautifulSoup(html,"html.parser")

        page=bs.find_all('a',class_="blast")

        last_page=page[0]['href'].split('/')[-1]

        return int(last_page)





    def __getContentAuto(self,url):

        req=urllib2.Request(url,headers=self.headers)

        resp=urllib2.urlopen(req)

        #time.sleep(2*random.random())

        content=resp.read()

        info=resp.info().get("Content-Encoding")

        if info==None:

            return content

        else:

            t=StringIO.StringIO(content)

            gziper=gzip.GzipFile(fileobj=t)

            html = gziper.read()

            return html



    #def __getFileName(self,stream):





    def __download(self,url):

        p=re.compile(r'href="(/photoid/\d+)"')

        #html=self.__getContentNoZip(url)



        html=self.__getContentAuto(url)



        content = p.findall(html)

        for i in content:

            print i



            photoid=self.__getContentAuto(self.url+i)

            bs=BeautifulSoup(photoid,"html.parser")

            final_link=bs.find('img',class_="mimg")['src']

            print final_link

            #pic_stream=self.__getContentAuto(final_link)

            title=bs.title.string.strip()

            filename = re.sub('[\/:*?"<>|]', '-', title)

            filename=filename+'.jpg'

            urllib.urlretrieve(final_link,filename)

            #f=open(filename,'w')

            #f.write(pic_stream)

            #f.close()

        #print html

        #bs=BeautifulSoup(html,"html.parser")

        #content=bs.find_all(p)

        #for i in content:

        #    print i

        '''

        print bs.title

        element_link=bs.find_all('div',class_="element")

        print len(element_link)

        k=1

        for href in element_link:



            #print type(href)

            #print href.tag

        '''

        '''

            if href.children[0]:

                print href.children[0]

        '''

        '''

            t=0



            for i in href.children:

                #if i.a:

                if t==0:

                    #print k

                    if i['href']

                    print link



                        if p.findall(link):

                            full_path=self.url[0:len(self.url)-1]+link

                            sub_html=self.__getContent(full_path)

                            bs=BeautifulSoup(sub_html,"html.parser")

                            final_link=bs.find('img',class_="mimg")['src']

                            #time.sleep(2*random.random())

                            print final_link

                    #k=k+1

                #print type(i)

                #print i.tag

                #if hasattr(i,"href"):

                    #print i['href']

                #print i.tag

                t=t+1

                #print "*"



        '''



        '''

            if href:

                if href.children:

                    print href.children[0]

        '''

            #print "one element link"







    def getPhoto(self):



        start=0

        #use style/0

        photo_url="http://photo.xitek.com/style/0/p/"

        for i in range(start,self.last_page+1):

            url=photo_url+str(i)

            print url

            #time.sleep(1)

            self.__download(url)



        '''

        url="http://photo.xitek.com/style/0/p/10"

        self.__download(url)

        '''

        #url="http://photo.xitek.com/style/0/p/0"

        #html=self.__getContent(url)

        #url="http://photo.xitek.com/"

        #html=self.__getContentNoZip(url)

        #print html

        #'''

def main():

    sub_folder = os.path.join(os.getcwd(), "content")

    if not os.path.exists(sub_folder):

        os.mkdir(sub_folder)

    os.chdir(sub_folder)

    obj=Xitek()

    obj.getPhoto()





if __name__=="__main__":

    main()

    list=[1,2,3,5,4,6,434,2323,333,99999]

    print "max of list is ",

    print max(list)

#-*-coding=utf-8-*-

__author__ = 'rocchen'

from lxml import html

from lxml import etree

import urllib2



def lxml_test():

    url="http://www.caixunzz.com"

    req=urllib2.Request(url=url)

    resp=urllib2.urlopen(req)

    #print resp.read()



    tree=etree.HTML(resp.read())

    href=tree.xpath('//a[@class="label"]/@href')

    #print href.tag

    for i in href:

        #print html.tostring(i)

        #print type(i)

        print i



    print type(href)



lxml_test()

可以在终端里敲入以下命令



alec@PC$ git config --global color.status auto  

alec@PC$ git config --global color.diff auto  

alec@PC$ git config --global color.branch auto  

alec@PC$ git config --global color.interactive auto 

>>> x = { }

>>> y = x

>>> x = { 'a' : 'b' }

>>> y

>>> { }

>>> x = { }

>>> y = x

>>> x['a'] = 'b'

>>> y

>>> { 'a' : 'b' }

# Copyright (c) 2009 rupa deadwyler under the WTFPL license



# maintains a jump-list of the directories you actually use

#

# INSTALL:

#     * put something like this in your .bashrc/.zshrc:

#         . /path/to/z.sh

#     * cd around for a while to build up the db

#     * PROFIT!!

#     * optionally:

#         set $_Z_CMD in .bashrc/.zshrc to change the command (default z).

#         set $_Z_DATA in .bashrc/.zshrc to change the datafile (default ~/.z).

#         set $_Z_NO_RESOLVE_SYMLINKS to prevent symlink resolution.

#         set $_Z_NO_PROMPT_COMMAND if you're handling PROMPT_COMMAND yourself.

#         set $_Z_EXCLUDE_DIRS to an array of directories to exclude.

#         set $_Z_OWNER to your username if you want use z while sudo with $HOME kept

#

# USE:

#     * z foo     # cd to most frecent dir matching foo

#     * z foo bar # cd to most frecent dir matching foo and bar

#     * z -r foo  # cd to highest ranked dir matching foo

#     * z -t foo  # cd to most recently accessed dir matching foo

#     * z -l foo  # list matches instead of cd

#     * z -c foo  # restrict matches to subdirs of $PWD



[ -d "${_Z_DATA:-$HOME/.z}" ] && {

    echo "ERROR: z.sh's datafile (${_Z_DATA:-$HOME/.z}) is a directory."

}



_z() {



    local datafile="${_Z_DATA:-$HOME/.z}"



    # bail if we don't own ~/.z and $_Z_OWNER not set

    [ -z "$_Z_OWNER" -a -f "$datafile" -a ! -O "$datafile" ] && return



    # add entries

    if [ "$1" = "--add" ]; then

        shift



        # $HOME isn't worth matching

        [ "$*" = "$HOME" ] && return



        # don't track excluded directory trees

        local exclude

        for exclude in "${_Z_EXCLUDE_DIRS[@]}"; do

            case "$*" in "$exclude*") return;; esac

        done



        # maintain the data file

        local tempfile="$datafile.$RANDOM"

        while read line; do

            # only count directories

            [ -d "${line%%\|*}" ] && echo $line

        done < "$datafile" | awk -v path="$*" -v now="$(date +%s)" -F"|" '

            BEGIN {

                rank[path] = 1

                time[path] = now

            }

            $2 >= 1 {

                # drop ranks below 1

                if( $1 == path ) {

                    rank[$1] = $2 + 1

                    time[$1] = now

                } else {

                    rank[$1] = $2

                    time[$1] = $3

                }

                count += $2

            }

            END {

                if( count > 9000 ) {

                    # aging

                    for( x in rank ) print x "|" 0.99*rank[x] "|" time[x]

                } else for( x in rank ) print x "|" rank[x] "|" time[x]

            }

        ' 2>/dev/null >| "$tempfile"

        # do our best to avoid clobbering the datafile in a race condition

        if [ $? -ne 0 -a -f "$datafile" ]; then

            env rm -f "$tempfile"

        else

            [ "$_Z_OWNER" ] && chown $_Z_OWNER:$(id -ng $_Z_OWNER) "$tempfile"

            env mv -f "$tempfile" "$datafile" || env rm -f "$tempfile"

        fi



    # tab completion

    elif [ "$1" = "--complete" -a -s "$datafile" ]; then

        while read line; do

            [ -d "${line%%\|*}" ] && echo $line

        done < "$datafile" | awk -v q="$2" -F"|" '

            BEGIN {

                if( q == tolower(q) ) imatch = 1

                q = substr(q, 3)

                gsub(" ", ".*", q)

            }

            {

                if( imatch ) {

                    if( tolower($1) ~ tolower(q) ) print $1

                } else if( $1 ~ q ) print $1

            }

        ' 2>/dev/null



    else

        # list/go

        while [ "$1" ]; do case "$1" in

            --) while [ "$1" ]; do shift; local fnd="$fnd${fnd:+ }$1";done;;

            -*) local opt=${1:1}; while [ "$opt" ]; do case ${opt:0:1} in

                    c) local fnd="^$PWD $fnd";;

                    h) echo "${_Z_CMD:-z} [-chlrtx] args" >&2; return;;

                    x) sed -i -e "\:^${PWD}|.*:d" "$datafile";;

                    l) local list=1;;

                    r) local typ="rank";;

                    t) local typ="recent";;

                esac; opt=${opt:1}; done;;

             *) local fnd="$fnd${fnd:+ }$1";;

        esac; local last=$1; [ "$#" -gt 0 ] && shift; done

        [ "$fnd" -a "$fnd" != "^$PWD " ] || local list=1



        # if we hit enter on a completion just go there

        case "$last" in

            # completions will always start with /

            /*) [ -z "$list" -a -d "$last" ] && cd "$last" && return;;

        esac



        # no file yet

        [ -f "$datafile" ] || return



        local cd

        cd="$(while read line; do

            [ -d "${line%%\|*}" ] && echo $line

        done < "$datafile" | awk -v t="$(date +%s)" -v list="$list" -v typ="$typ" -v q="$fnd" -F"|" '

            function frecent(rank, time) {

                # relate frequency and time

                dx = t - time

                if( dx < 3600 ) return rank * 4

                if( dx < 86400 ) return rank * 2

                if( dx < 604800 ) return rank / 2

                return rank / 4

            }

            function output(files, out, common) {

                # list or return the desired directory

                if( list ) {

                    cmd = "sort -n >&2"

                    for( x in files ) {

                        if( files[x] ) printf "%-10s %s\n", files[x], x | cmd

                    }

                    if( common ) {

                        printf "%-10s %s\n", "common:", common > "/dev/stderr"

                    }

                } else {

                    if( common ) out = common

                    print out

                }

            }

            function common(matches) {

                # find the common root of a list of matches, if it exists

                for( x in matches ) {

                    if( matches[x] && (!short || length(x) < length(short)) ) {

                        short = x

                    }

                }

                if( short == "/" ) return

                # use a copy to escape special characters, as we want to return

                # the original. yeah, this escaping is awful.

                clean_short = short

                gsub(/\[\(\)\[\]\|\]/, "\\\\&", clean_short)

                for( x in matches ) if( matches[x] && x !~ clean_short ) return

                return short

            }

            BEGIN {

                gsub(" ", ".*", q)

                hi_rank = ihi_rank = -9999999999

            }

            {

                if( typ == "rank" ) {

                    rank = $2

                } else if( typ == "recent" ) {

                    rank = $3 - t

                } else rank = frecent($2, $3)

                if( $1 ~ q ) {

                    matches[$1] = rank

                } else if( tolower($1) ~ tolower(q) ) imatches[$1] = rank

                if( matches[$1] && matches[$1] > hi_rank ) {

                    best_match = $1

                    hi_rank = matches[$1]

                } else if( imatches[$1] && imatches[$1] > ihi_rank ) {

                    ibest_match = $1

                    ihi_rank = imatches[$1]

                }

            }

            END {

                # prefer case sensitive

                if( best_match ) {

                    output(matches, best_match, common(matches))

                } else if( ibest_match ) {

                    output(imatches, ibest_match, common(imatches))

                }

            }

        ')"

        [ $? -gt 0 ] && return

        [ "$cd" ] && cd "$cd"

    fi

}



alias ${_Z_CMD:-z}='_z 2>&1'



[ "$_Z_NO_RESOLVE_SYMLINKS" ] || _Z_RESOLVE_SYMLINKS="-P"



if type compctl >/dev/null 2>&1; then

    # zsh

    [ "$_Z_NO_PROMPT_COMMAND" ] || {

        # populate directory list, avoid clobbering any other precmds.

        if [ "$_Z_NO_RESOLVE_SYMLINKS" ]; then

            _z_precmd() {

                _z --add "${PWD:a}"

            }

        else

            _z_precmd() {

                _z --add "${PWD:A}"

            }

        fi

        [[ -n "${precmd_functions[(r)_z_precmd]}" ]] || {

            precmd_functions[$(($#precmd_functions+1))]=_z_precmd

        }

    }

    _z_zsh_tab_completion() {

        # tab completion

        local compl

        read -l compl

        reply=(${(f)"$(_z --complete "$compl")"})

    }

    compctl -U -K _z_zsh_tab_completion _z

elif type complete >/dev/null 2>&1; then

    # bash

    # tab completion

    complete -o filenames -C '_z --complete "$COMP_LINE"' ${_Z_CMD:-z}

    [ "$_Z_NO_PROMPT_COMMAND" ] || {

        # populate directory list. avoid clobbering other PROMPT_COMMANDs.

        grep "_z --add" <<< "$PROMPT_COMMAND" >/dev/null || {

            PROMPT_COMMAND="$PROMPT_COMMAND"$'\n''_z --add "$(command pwd '$_Z_RESOLVE_SYMLINKS' 2>/dev/null)" 2>/dev/null;'

        }

    }

fi

p = subprocess.Popen(cmd, stdout = subprocess.PIPE, stdin = subprocess.PIPE, shell = True)

p.wait()

// hang here

print "finished"

p = subprocess.Popen(cmd, stdout = subprocess.PIPE, stdin = subprocess.PIPE, shell = True)

stdout, stderr = p.communicate()

p.wait()

print "Finsih"

# -*- coding=utf-8 -*-

__author__ = 'rocky @ www.30daydo.com'

import urllib2, re, os, codecs,sys,datetime

from bs4 import BeautifulSoup

# example https://zhhrb.sinaapp.com/index.php?date=20160610

from mail_template import MailAtt

reload(sys)

sys.setdefaultencoding('utf-8')



def save2file(filename, content):

    filename = filename + ".txt"

    f = codecs.open(filename, 'a', encoding='utf-8')

    f.write(content)

    f.close()





def getPost(date_time, filter_p):

    url = 'https://zhhrb.sinaapp.com/index.php?date=' + date_time

    user_agent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)"

    header = {"User-Agent": user_agent}

    req = urllib2.Request(url, headers=header)

    resp = urllib2.urlopen(req)

    content = resp.read()

    p = re.compile('<h2 class="question-title">(.*)</h2></br></a>')

    result = re.findall(p, content)

    count = -1

    row = -1

    for i in result:

        #print i

        return_content = re.findall(filter_p, i)



        if return_content:

            row = count

            break

            #print return_content[0]

        count = count + 1

    #print row

    if row == -1:

        return 0

    link_p = re.compile('<a href="(.*)" target="_blank" rel="nofollow">')

    link_result = re.findall(link_p, content)[row + 1]

    print link_result

    result_req = urllib2.Request(link_result, headers=header)

    result_resp = urllib2.urlopen(result_req)

    #result_content= result_resp.read()

    #print result_content



    bs = BeautifulSoup(result_resp, "html.parser")

    title = bs.title.string.strip()

    #print title

    filename = re.sub('[\/:*?"<>|]', '-', title)

    print filename

    print date_time

    save2file(filename, title)

    save2file(filename, "\n\n\n\n--------------------%s Detail----------------------\n\n" %date_time)



    detail_content = bs.find_all('div', class_='content')



    for i in detail_content:

        #print i

        save2file(filename,"\n\n-------------------------answer  -------------------------\n\n")

        for j in i.strings:



            save2file(filename, j)



    smtp_server = 'smtp.126.com'

    from_mail = sys.argv[1]

    password = sys.argv[2]

    to_mail = 'xxxxx@kindle.cn'

    send_kindle = MailAtt(smtp_server, from_mail, password, to_mail)

    send_kindle.send_txt(filename)





def main():

    sub_folder = os.path.join(os.getcwd(), "content")

    if not os.path.exists(sub_folder):

        os.mkdir(sub_folder)

    os.chdir(sub_folder)





    date_time = '20160611'

    filter_p = re.compile('大误.*')

    ori_day=datetime.date(datetime.date.today().year,01,01)

    t=datetime.date(datetime.date.today().year,datetime.date.today().month,datetime.date.today().day)

    delta=(t-ori_day).days

    print delta

    for i in range(delta):

        day=datetime.date(datetime.date.today().year,01,01)+datetime.timedelta(i)

        getPost(day.strftime("%Y%m%d"),filter_p)

    #getPost(date_time, filter_p)



if __name__ == "__main__":

    main()