Don't use BeautifulSoup, use lxml.soupparser then you're sitting on top of the power of lxml and can use the good bits of BeautifulSoup which is to deal with really broken and crappy HTML.
 
 
 
9down vote
In summary, 

lxml

 is positioned as a lightning-fast production-quality html and xml parser that, by the way, also includes a 

soupparser

 module to fall back on BeautifulSoup's functionality. 

BeautifulSoup

is a one-person project, designed to save you time to quickly extract data out of poorly-formed html or xml.
lxml documentation says that both parsers have advantages and disadvantages. For this reason, 

lxml

 provides a 

soupparser

 so you can switch back and forth. Quoting,
[quote]
BeautifulSoup uses a different parsing approach. It is not a real HTML parser but uses regular expressions to dive through tag soup. It is therefore more forgiving in some cases and less good in others. It is not uncommon that lxml/libxml2 parses and fixes broken HTML better, but BeautifulSoup has superiour support for encoding detection. It very much depends on the input which parser works better.

The downside of using this parser is that it is much slower than the HTML parser of lxml. So if performance matters, you might want to consider using soupparser only as a fallback for certain cases.

lxml

BeautifulSoup

soupparser

lxml

BeautifulSoup

lxml

[code]>>> from BeautifulSoup import UnicodeDammit



>>> def decode_html(html_string):

...     converted = UnicodeDammit(html_string, isHTML=True)

...     if not converted.unicode:

...         raise UnicodeDecodeError(

...             "Failed to detect encoding, tried [%s]",

...             ', '.join(converted.triedEncodings))

...     # print converted.originalEncoding

...     return converted.unicode



>>> root = lxml.html.fromstring(decode_html(tag_soup))

BeautifulSoup

That's it! Have fun! I wrote Beautiful Soup to save everybody time. Once you get used to it, you should be able to wrangle data out of poorly-designed websites in just a few minutes. Send me email if you have any comments, run into problems, or want me to know about your project that uses Beautiful Soup.

[code] --Leonard

[/code]

lxml has been downloaded from the Python Package Index more than two million times and is also available directly in many package distributions, e.g. for Linux or MacOS-X.

The C libraries libxml2 and libxslt have huge benefits:... Standards-compliant... Full-featured... fast. fast! FAST! ... lxml is a new Python binding for libxml2 and libxslt...

    list=[1,2,3,5,4,6,434,2323,333,99999]

    print "max of list is ",

    print max(list)

#-*-coding=utf-8-*-

__author__ = 'rocchen'

from lxml import html

from lxml import etree

import urllib2



def lxml_test():

    url="http://www.caixunzz.com"

    req=urllib2.Request(url=url)

    resp=urllib2.urlopen(req)

    #print resp.read()



    tree=etree.HTML(resp.read())

    href=tree.xpath('//a[@class="label"]/@href')

    #print href.tag

    for i in href:

        #print html.tostring(i)

        #print type(i)

        print i



    print type(href)



lxml_test()

>>> x = { }

>>> y = x

>>> x = { 'a' : 'b' }

>>> y

>>> { }

>>> x = { }

>>> y = x

>>> x['a'] = 'b'

>>> y

>>> { 'a' : 'b' }

p = subprocess.Popen(cmd, stdout = subprocess.PIPE, stdin = subprocess.PIPE, shell = True)

p.wait()

// hang here

print "finished"

p = subprocess.Popen(cmd, stdout = subprocess.PIPE, stdin = subprocess.PIPE, shell = True)

stdout, stderr = p.communicate()

p.wait()

print "Finsih"

import os



def isContain(des_str,ori_str):

	for i in des_str:

		if ori_str == i:

			return True

	return False





path=os.getcwd()

print path

des_str=['img','cfg','bct','bin','sh','dtb','txt','mk','pem','mk','pk8','xml','lib','pl','blob','dat']

for fpath,dirs,fname in os.walk(path):

	#print fname

	

	if fname:

		for i in fname:

			#print i

			name=i.split('.')

			if len(name)>=2:

				#print name[1]

				if isContain(des_str,name[1]):

					filepath=os.path.join(fpath,i)

					print "delete file %s" %filepath

					os.remove(filepath)

import socket,sys

import time

from thread_test import MyThread



socket.setdefaulttimeout(1)

#设置每个线程socket的timeou时间，超过1秒没有反应就认为端口不开放

thread_num=4

#线程数目 

ip_end=256

ip_start=0

scope=ip_end/thread_num



def scan(ip_head,ip_low, port):

    try:

        # Alert !!! below statement should be inside scan function. Else each it is one s

        ip=ip_head+str(ip_low)

	print ip

	s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

	s.connect((ip, port))

	#通过这一句判断 是否连通

        s.close()

	print "ip %s port %d open\n" %(ip,port)

        return True

    except:

        return False





def scan_range(ip_head,ip_range,port):

	start,end=ip_range

	for i in range(start,end):

		scan(ip_head,i,port)



if len(sys.argv)<3:

	print "input ip and port"

	exit()



ip_head=sys.argv[1]

port=int(sys.argv[2])





ip_range=

for i in range(thread_num):

	x_range=[i*scope,(i+1)*scope-1]

	ip_range.append(x_range)



threads=

for i in range(thread_num):

	t=MyThread(scan_range,(ip_head,ip_range,port))

	threads.append(t)

for i in range(thread_num):

	threads.start()

for i in range(thread_num):

	threads.join()

	#设置进程阻塞，防止主线程退出了，其他的多线程还在运行



print "*****end*****"

import thread,threading,time,datetime

from time import sleep,ctime

def loop1():

	print "start %s " %ctime()

	print "start in loop1"

	sleep(3)

	print "end %s " %ctime()



def loop2():

	print "sart %s " %ctime()

	print "start in loop2"

	sleep(6)

	print "end %s " %ctime()





class MyThread(threading.Thread):

	def __init__(self,fun,arg,name=""):

		threading.Thread.__init__(self)

		self.fun=fun

		self.arg=arg

		self.name=name

		#self.result



	def run(self):

		self.result=apply(self.fun,self.arg)

	

	def getResult(self):

		return self.result



def fib(n):

	if n<2:

		return 1

	else:

		return fib(n-1)+fib(n-2)





def sum(n):

	if n<2:

		return 1

	else:

		return n+sum(n-1)	



def fab(n):

	if n<2:

		return 1

	else:

		return n*fab(n-1)



def single_thread():		

	print fib(12)		

	print sum(12)

	print fab(12)





def multi_thread():

	print "in multithread"

	fun_list=[fib,sum,fab]

	n=len(fun_list)

	threads=

	count=12

	for i in range(n):

		t=MyThread(fun_list,(count,),fun_list.__name__)

		threads.append(t)

	for i in range(n):

		threads.start()



	for i in range(n):

		threads.join()

		result= threads.getResult()

		print result

def main():

	'''

	print "start at main"

	thread.start_new_thread(loop1,())

	thread.start_new_thread(loop2,())

	sleep(10)

	print "end at main"

	'''

	start=ctime()

	#print "Used %f" %(end-start).seconds

	print start	

	single_thread()

	end=ctime()

	print end

	multi_thread()

	#print "used %s" %(end-start).seconds 

if __name__=="__main__":

	main()