1. Beautiful Soup的簡介
Beautiful Soup 是一個(gè)可以從HTML或XML文件中提取數(shù)據(jù)的Python庫.它能夠通過你喜歡的轉(zhuǎn)換器實(shí)現(xiàn)慣用的文檔導(dǎo)航,查找,修改文檔的方式.Beautiful Soup會(huì)幫你節(jié)省數(shù)小時(shí)甚至數(shù)天的工作時(shí)間.Beautiful Soup 3 目前已經(jīng)停止開發(fā),我們推薦在現(xiàn)在的項(xiàng)目中使用Beautiful Soup 4, 移植到BS4，出現(xiàn)的例子在Python2.7和Python3.2中的執(zhí)行結(jié)果相同
2.使用的示例代碼
安裝代碼
sudo easy_install -U beautifulsoup4
下面的一段HTML代碼將作為例子被多次用到.這是 愛麗絲夢游仙境的 的一段內(nèi)容
html_doc = """
<html><head><title>The Dormouse's story</title></head>
<body>
<p class="title"><b>The Dormouse's story</b></p>
<p class="story">Once upon a time there were three little sisters; and their names were
<a class="sister" id="link1">Elsie</a>,
<a class="sister" id="link2">Lacie</a> and
<a class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
<p class="story">...</p>
"""
常見的元素輸出
#第一個(gè)參數(shù)是html對象
soup=BeautifulSoup(html,"lxml")
#輸出<title>的元素
print soup.title
#輸出<head>的元素
print soup.head
#輸出<a>的元素
print soup.a
#輸出<p>的元素
print soup.p
#輸出name的元素
print soup.name
#輸出head的name屬性
print soup.head.name
#輸出p元素的屬性和對印值得到字典
print soup.p.attrs
#輸出p元素的里面的class屬性
print soup.p['class']
print soup.p.get('class')
#修改class屬性內(nèi)容
soup.p['class']="newClass"
print soup.p
#獲取標(biāo)簽內(nèi)容
print soup.p.string
#輸出所有head屬性的一個(gè)數(shù)組
print soup.head.contents
#遍歷所有body的子節(jié)點(diǎn)
for child in soup.body.children:
print child
for child in soup.descendants:
print child
string 、strings和stripped_strings的效果
print soup.head.string
#The Dormouse's story
print soup.title.string
#The Dormouse's story
#獲取多個(gè)內(nèi)容，遍歷獲取所有元素
for string in soup.strings:
print(repr(string))
# u"The Dormouse's story"
# u'\n\n'
# u"The Dormouse's story"
# u'\n\n'
# u'Once upon a time there were three little sisters; and their names were\n'
# u'Elsie'
# u',\n'
# u'Lacie'
# u' and\n'
# u'Tillie'
# u';\nand they lived at the bottom of a well.'
# u'\n\n'
# u'...'
# u'\n
#輸出的字符串中可能包含了很多空格或空行,使用 .stripped_strings 可以去除多余空白內(nèi)容
for string in soup.stripped_strings:
print(repr(string))
#u'Once upon a time there were three little sisters; and their names were'
#u','
#u'Lacie'
#u'and'
#u'Tillie'
#u';\nand they lived at the bottom of a well.'
輸出父節(jié)點(diǎn)parent、parents
content=soup.head.title.string
print content.parent.name
#title
for parent in content.parents:
print parent.name
#title
#head
#html
#[document]
兄弟節(jié)點(diǎn)next_sibling和prev_sibling
print soup.p.next_sibling
#
#
#
print soup.p.prev_sibling
#None
print soup.p.next_sibling.next_sibling
#<p class="story">Once upon a time there were three little sisters; and their names were
#<a class="sister" id="link1"><!-- Elsie --></a>,
#<a class="sister" id="link2">Lacie</a> and
#<a class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
搜索文檔樹
函數(shù)原型
find_all( name , attrs , recursive , string , **kwargs )
find_parents( name , attrs , recursive , string , **kwargs )
find_parent( name , attrs , recursive , string , **kwargs )
。。。
1.name查找所有名為name的tag
soup=BeautifulSoup(html,"lxml")
print soup.find_all('p')
#[<p class="title" name="dromouse"><b>The Dormouse's story</b></p>, <p class="story">Once upon a time there were three little si
sters; and their names were\n<a class="sister" id="link1"><!-- Elsie --></a>,\n<a class="sister
" id="link2">Lacie</a> and\n<a class="sister" id="link3">Tilli
e</a>;\nand they lived at the bottom of a well.</p>]
2.傳入正則表達(dá)
soup=BeautifulSoup(html,"lxml")
for tag in soup.find_all(re.compile("b.*?")):
print(tag.name)
#body
#b
3.傳入數(shù)組
soup=BeautifulSoup(html,"lxml")
#包含a,b標(biāo)簽
print soup.find_all(["a","b"])
#[<b>The Dormouse's story</b>, <a class="sister" id="link1"><!-- Elsie --></a>, <a class="sister
" id="link2">Lacie</a>, <a class="sister" id="link3">Tillie</a
>]
4.傳入True
#所有節(jié)點(diǎn)
soup=BeautifulSoup(html,"lxml")
for tag in soup.find_all(True):
print tag.name
#html
#head
#title
#body
#p
#b
#p
#a
#a
#a
5.傳入方法
#含有class而沒有href的tag
def has_class_but_no_id(tag):
return tag.has_attr('class') and not tag.has_attr('href')
print soup.find_all(has_class_but_no_id)
#[<p class="title" name="dromouse"><b>The Dormouse's story</b></p>, <p class="story">Once upon a time there were three little si
sters; and their names were\n<a class="sister" id="link1"><!-- Elsie --></a>,\n<a class="sister
" id="link2">Lacie</a> and\n<a class="sister" id="link3">Tilli
e</a>;\nand they lived at the bottom of a well.</p>]
6.傳入字典
soup=BeautifulSoup(html,"lxml")
#id含有l(wèi)ink2項(xiàng)的
print soup.find_all(id='link2')
#[<a class="sister" id="link2">Lacie</a>]
#href中含有elsie項(xiàng)
print soup.find_all(href=re.compile("elsie"))
#[<a class="sister" id="link1"><!-- Elsie --></a>]
#href含有elsie項(xiàng)并且id為link1項(xiàng)
print soup.find_all(href=re.compile("elsie"),id='link1')
#[<a class="sister" id="link1"><!-- Elsie --></a>]
中文文檔
