import tushare as ts

df=ts.get_stock_basics()

def get_stock_basics():

    """

        获取沪深上市公司基本情况

    Return

    --------

    DataFrame

               code,代码

               name,名称

               industry,细分行业

               area,地区

               pe,市盈率

               outstanding,流通股本

               totals,总股本(万)

               totalAssets,总资产(万)

               liquidAssets,流动资产

               fixedAssets,固定资产

               reserved,公积金

               reservedPerShare,每股公积金

               eps,每股收益

               bvps,每股净资

               pb,市净率

               timeToMarket,上市日期

    """

    request = Request(ct.ALL_STOCK_BASICS_FILE)

    text = urlopen(request, timeout=10).read()

    text = text.decode('GBK')

    text = text.replace('--', '')

    df = pd.read_csv(StringIO(text), dtype={'code':'object'})

    df = df.set_index('code')

    return df

P_TYPE = {'http': 'http://', 'ftp': 'ftp://'}



DOMAINS = {'sina': 'sina.com.cn', 'sinahq': 'sinajs.cn',

           'ifeng': 'ifeng.com', 'sf': 'finance.sina.com.cn',

           'vsf': 'vip.stock.finance.sina.com.cn', 

           'idx': 'www.csindex.com.cn', '163': 'money.163.com',

           'em': 'eastmoney.com', 'sseq': 'query.sse.com.cn',

           'sse': 'www.sse.com.cn', 'szse': 'www.szse.cn',

           'oss': '218.244.146.57', 'idxip':'115.29.204.48',

           'shibor': 'www.shibor.org', 'mbox':'www.cbooo.cn'}

request = Request(ct.REPORT_URL%(ct.P_TYPE['http'], ct.DOMAINS['vsf'], ct.PAGES['fd'],year, quarter, pageNo, ct.PAGE_NUM[1]))

text = urlopen(request, timeout=10).read()

        df=ts.get_report_data(2016,4)

        #第四季度就是年报

# -*-coding=utf-8-*-

#数据库的操作

'''

http://30daydo.com

weigesysu@qq.com

'''

import sqlite3, time, datetime



__author__ = 'rocky'





class SqliteDb():





    def __init__(self,dbtable):

        '''

        self.today = time.strftime("%Y-%m-%d")

        self.DBname = self.today + '.db'

        self.conn = sqlite3.connect(self.DBname)

        '''

        today = time.strftime("%Y-%m-%d")

        DBname = today + '.db'

        self.conn = sqlite3.connect(DBname)

        self.dbtable=dbtable

        create_tb = "CREATE TABLE %s (date varchar(10),id varchar(6), name varchar(30), p_change REAL,turnover REAL);" %self.dbtable

        self.conn.execute(create_tb)

        self.conn.commit()



    def store_break_high(self,price_high_data):



        #data 是创新高的个股信息  dataframe

        #print today

        #create_tb = 'CREATE TABLE STOCK (date TEXT,id text PRIMARY KEY, p_change REAL,turnover REAL);'



        #conn.commit()

        #print "(%s,%s,%f,%f)" %(price_high_data[0], price_high_data[1], price_high_data[2], price_high_data[3])

        insert_data_cmd = "INSERT INTO %s(date,id,name,p_change,turnover) VALUES(\"%s\",\"%s\",\"%s\",%f,%f);" %(self.dbtable,price_high_data[0], price_high_data[1], price_high_data[2], price_high_data[3],price_high_data[4])

        self.conn.execute(insert_data_cmd)

        #self.conn.execute('INSERT INTO STOCK(date,id,name,p_change,turnover) VALUES(?,?,?,?,?)',(price_high_data[0], price_high_data[1], price_high_data[2], price_high_data[3],price_high_data[4]))

        self.conn.commit()





    def close(self):

        self.conn.close()

    if df is None:

        print "None len==0"

        return False

#-*-coding=utf-8-*-

__author__ = 'rocky'

'''

http://30daydo.com

weigesysu@qq.com

'''

#获取破指定天数内的新高 比如破60日新高

import tushare as ts

import datetime

info=ts.get_stock_basics()



def loop_all_stocks():

    for EachStockID in info.index:

         if is_break_high(EachStockID,60):

             print "High price on",

             print EachStockID,

             print info.ix[EachStockID]['name'].decode('utf-8')







def is_break_high(stockID,days):

    end_day=datetime.date(datetime.date.today().year,datetime.date.today().month,datetime.date.today().day)

    days=days*7/5

    #考虑到周六日非交易

    start_day=end_day-datetime.timedelta(days)



    start_day=start_day.strftime("%Y-%m-%d")

    end_day=end_day.strftime("%Y-%m-%d")

    df=ts.get_h_data(stockID,start=start_day,end=end_day)



    period_high=df['high'].max()

    #print period_high

    today_high=df.iloc[0]['high']

    #这里不能直接用 .values

    #如果用的df【：1】 就需要用.values

    #print today_high

    if today_high>=period_high:

        return True

    else:

        return False



loop_all_stocks()

__author__ = 'rocky'

import urllib,urllib2,StringIO,gzip

url="http://image.xitek.com/photo/2 ... ot%3B

filname=url.split("/")[-1]

req=urllib2.Request(url)

resp=urllib2.urlopen(req)

content=resp.read()

#data = StringIO.StringIO(content)

#gzipper = gzip.GzipFile(fileobj=data)

#html = gzipper.read()

f=open(filname,'w')

f.write()

f.close()

sudo gedit /etc/updatedb.conf

PRUNEPATHS="/tmp /var/spool /media /home/.ecryptfs"

1 day's raise is 10.00

2 day's raise is 21.00

3 day's raise is 33.10

4 day's raise is 46.41

5 day's raise is 61.05

__author__ = 'rocchen'

# working v1.0

from bs4 import BeautifulSoup

import urllib2, datetime, time, codecs, cookielib, random, threading

import os,sys





def getInfo(max_index_user=5):

    stock_news_site =

"http://ggjd.cnstock.com/gglist/search/ggkx/"

 

    my_userAgent = [

        'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',

        'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',

        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0',

        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)',

        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)',

        'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',

        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',

        'Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',

        'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11',

        'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11',

        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)',

        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',

        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)',

        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)']

    index = 0

    max_index = max_index_user

    num = 1

    temp_time = time.strftime("[%Y-%m-%d]-[%H-%M]", time.localtime())



    store_filename = "StockNews-%s.log" % temp_time

    fOpen = codecs.open(store_filename, 'w', 'utf-8')



    while index < max_index:

        user_agent = random.choice(my_userAgent)

        # print user_agent

        company_news_site = stock_news_site + str(index)

        # content = urllib2.urlopen(company_news_site)

        headers = {'User-Agent': user_agent, 'Host': "ggjd.cnstock.com", 'DNT': '1',

                   'Accept': 'text/html, application/xhtml+xml, */*', }

        req = urllib2.Request(url=company_news_site, headers=headers)

        resp = None

        raw_content = ""

        try:

            resp = urllib2.urlopen(req, timeout=30)



        except urllib2.HTTPError as e:

            e.fp.read()

        except urllib2.URLError as e:

            if hasattr(e, 'code'):

                print "error code %d" % e.code

            elif hasattr(e, 'reason'):

                print "error reason %s " % e.reason



        finally:

            if resp:

                raw_content = resp.read()

                time.sleep(2)

                resp.close()



        soup = BeautifulSoup(raw_content, "html.parser")

        all_content = soup.find_all("span", "time")



        for i in all_content:

            news_time = i.string

            node = i.next_sibling

            str_temp = "No.%s \n%s\t%s\n---> %s \n\n" % (str(num), news_time, node['title'], node['href'])

            #print "inside %d" %num

            #print str_temp

            fOpen.write(str_temp)

            num = num + 1



        #print "index %d" %index

        index = index + 1



    fOpen.close()





def execute_task(n=60):

    period = int(n)

    while True:

        print datetime.datetime.now()

        getInfo(3)

        

        time.sleep(60 * period)

        





if __name__ == "__main__":



    sub_folder = os.path.join(os.getcwd(), "stock")

    if not os.path.exists(sub_folder):

        os.mkdir(sub_folder)

    os.chdir(sub_folder)

    start_time = time.time()  # user can change the max index number getInfo(10), by default is getInfo(5)

    if len(sys.argv) <2:

        n = raw_input("Input Period : ? mins to download every cycle")

    else:

        n=int(sys.argv[1])

    execute_task(n)

    end_time = time.time()

    print "Total time: %s s." % str(round((end_time - start_time), 4))

Don't use BeautifulSoup, use lxml.soupparser then you're sitting on top of the power of lxml and can use the good bits of BeautifulSoup which is to deal with really broken and crappy HTML.
 
 
 
9down vote
In summary, 

lxml

 is positioned as a lightning-fast production-quality html and xml parser that, by the way, also includes a 

soupparser

 module to fall back on BeautifulSoup's functionality. 

BeautifulSoup

is a one-person project, designed to save you time to quickly extract data out of poorly-formed html or xml.
lxml documentation says that both parsers have advantages and disadvantages. For this reason, 

lxml

 provides a 

soupparser

 so you can switch back and forth. Quoting,
[quote]
BeautifulSoup uses a different parsing approach. It is not a real HTML parser but uses regular expressions to dive through tag soup. It is therefore more forgiving in some cases and less good in others. It is not uncommon that lxml/libxml2 parses and fixes broken HTML better, but BeautifulSoup has superiour support for encoding detection. It very much depends on the input which parser works better.

The downside of using this parser is that it is much slower than the HTML parser of lxml. So if performance matters, you might want to consider using soupparser only as a fallback for certain cases.

lxml

BeautifulSoup

soupparser

lxml

BeautifulSoup

lxml

[code]>>> from BeautifulSoup import UnicodeDammit



>>> def decode_html(html_string):

...     converted = UnicodeDammit(html_string, isHTML=True)

...     if not converted.unicode:

...         raise UnicodeDecodeError(

...             "Failed to detect encoding, tried [%s]",

...             ', '.join(converted.triedEncodings))

...     # print converted.originalEncoding

...     return converted.unicode



>>> root = lxml.html.fromstring(decode_html(tag_soup))

BeautifulSoup

That's it! Have fun! I wrote Beautiful Soup to save everybody time. Once you get used to it, you should be able to wrangle data out of poorly-designed websites in just a few minutes. Send me email if you have any comments, run into problems, or want me to know about your project that uses Beautiful Soup.

[code] --Leonard

[/code]

lxml has been downloaded from the Python Package Index more than two million times and is also available directly in many package distributions, e.g. for Linux or MacOS-X.

The C libraries libxml2 and libxslt have huge benefits:... Standards-compliant... Full-featured... fast. fast! FAST! ... lxml is a new Python binding for libxml2 and libxslt...

#-*-coding=utf-8-*-

__author__ = 'rocky chen'

from bs4 import BeautifulSoup

import urllib2,sys,StringIO,gzip,time,random,re,urllib,os

reload(sys)

sys.setdefaultencoding('utf-8')

class Xitek():

    def __init__(self):

        self.url="http://photo.xitek.com/"

        user_agent="Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)"

        self.headers={"User-Agent":user_agent}

        self.last_page=self.__get_last_page()





    def __get_last_page(self):

        html=self.__getContentAuto(self.url)

        bs=BeautifulSoup(html,"html.parser")

        page=bs.find_all('a',class_="blast")

        last_page=page[0]['href'].split('/')[-1]

        return int(last_page)





    def __getContentAuto(self,url):

        req=urllib2.Request(url,headers=self.headers)

        resp=urllib2.urlopen(req)

        #time.sleep(2*random.random())

        content=resp.read()

        info=resp.info().get("Content-Encoding")

        if info==None:

            return content

        else:

            t=StringIO.StringIO(content)

            gziper=gzip.GzipFile(fileobj=t)

            html = gziper.read()

            return html



    #def __getFileName(self,stream):





    def __download(self,url):

        p=re.compile(r'href="(/photoid/\d+)"')

        #html=self.__getContentNoZip(url)



        html=self.__getContentAuto(url)



        content = p.findall(html)

        for i in content:

            print i



            photoid=self.__getContentAuto(self.url+i)

            bs=BeautifulSoup(photoid,"html.parser")

            final_link=bs.find('img',class_="mimg")['src']

            print final_link

            #pic_stream=self.__getContentAuto(final_link)

            title=bs.title.string.strip()

            filename = re.sub('[\/:*?"<>|]', '-', title)

            filename=filename+'.jpg'

            urllib.urlretrieve(final_link,filename)

            #f=open(filename,'w')

            #f.write(pic_stream)

            #f.close()

        #print html

        #bs=BeautifulSoup(html,"html.parser")

        #content=bs.find_all(p)

        #for i in content:

        #    print i

        '''

        print bs.title

        element_link=bs.find_all('div',class_="element")

        print len(element_link)

        k=1

        for href in element_link:



            #print type(href)

            #print href.tag

        '''

        '''

            if href.children[0]:

                print href.children[0]

        '''

        '''

            t=0



            for i in href.children:

                #if i.a:

                if t==0:

                    #print k

                    if i['href']

                    print link



                        if p.findall(link):

                            full_path=self.url[0:len(self.url)-1]+link

                            sub_html=self.__getContent(full_path)

                            bs=BeautifulSoup(sub_html,"html.parser")

                            final_link=bs.find('img',class_="mimg")['src']

                            #time.sleep(2*random.random())

                            print final_link

                    #k=k+1

                #print type(i)

                #print i.tag

                #if hasattr(i,"href"):

                    #print i['href']

                #print i.tag

                t=t+1

                #print "*"



        '''



        '''

            if href:

                if href.children:

                    print href.children[0]

        '''

            #print "one element link"







    def getPhoto(self):



        start=0

        #use style/0

        photo_url="http://photo.xitek.com/style/0/p/"

        for i in range(start,self.last_page+1):

            url=photo_url+str(i)

            print url

            #time.sleep(1)

            self.__download(url)



        '''

        url="http://photo.xitek.com/style/0/p/10"

        self.__download(url)

        '''

        #url="http://photo.xitek.com/style/0/p/0"

        #html=self.__getContent(url)

        #url="http://photo.xitek.com/"

        #html=self.__getContentNoZip(url)

        #print html

        #'''

def main():

    sub_folder = os.path.join(os.getcwd(), "content")

    if not os.path.exists(sub_folder):

        os.mkdir(sub_folder)

    os.chdir(sub_folder)

    obj=Xitek()

    obj.getPhoto()





if __name__=="__main__":

    main()

    list=[1,2,3,5,4,6,434,2323,333,99999]

    print "max of list is ",

    print max(list)

#-*-coding=utf-8-*-

__author__ = 'rocchen'

from lxml import html

from lxml import etree

import urllib2



def lxml_test():

    url="http://www.caixunzz.com"

    req=urllib2.Request(url=url)

    resp=urllib2.urlopen(req)

    #print resp.read()



    tree=etree.HTML(resp.read())

    href=tree.xpath('//a[@class="label"]/@href')

    #print href.tag

    for i in href:

        #print html.tostring(i)

        #print type(i)

        print i



    print type(href)



lxml_test()