最近在學(xué)習(xí)Python相關(guān)，學(xué)習(xí)了基本的語法后想搞點(diǎn)事情試試，所以來爬取下古詩文網(wǎng)中的相關(guān)作者信息
準(zhǔn)備資料：
爬取目標(biāo)：爬取古詩文網(wǎng)的唐代作者的信息
目標(biāo)分析：
作者列表頁面
一級頁面是所有唐代作者的列表，點(diǎn)擊名字后會(huì)跳轉(zhuǎn)到作者詳情頁，所以這一頁我們要做的就是解析出每一項(xiàng)作者詳情頁url，并且自動(dòng)翻頁
作者詳情頁
上述是作者詳情頁，其中主要包含作者名字，作者簡述和作者生平故事，這一頁我要做的事情是解析這些數(shù)據(jù)并保存到本地文件；
詳細(xì)代碼
#!/usr/bin/env python2.7
import requests
import codecs
import json
from bs4 import BeautifulSoup
DOWNLOAD_URL = 'https://so.gushiwen.org/authors/Default.aspx?p=1&c=%E5%94%90%E4%BB%A3'
def download_page(url):
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36'
}
data = requests.get(url, headers=headers).content
return data
def parseHtml(html):
soup = BeautifulSoup(html, features="html.parser")
nextPage = 'https://so.gushiwen.org' + soup.find('a', attrs={'class': 'amore'}).get('href')
leftDiv = soup.find('div', attrs={'class': 'main3'}).find('div', attrs={'class': 'left'})
data = []
for item in leftDiv.find_all('div', attrs={'class': 'sonspic'}):
data.append('https://so.gushiwen.org' + item.find('a')['href'])
return data, nextPage
def buildJson(keys, values):
dictionary = dict(zip(keys, values))
return json.dumps(dictionary)
def parseAuthorHtml(html):
soup = BeautifulSoup(html, features="html.parser")
all = soup.find('div', attrs={'class': 'main3'}).find('div', attrs={'class': 'left'})
author = all.find('div', attrs={'class': 'sonspic'}).find('div', attrs={'class': 'cont'}).find('b').getText()
desc = all.find('div', attrs={'class': 'sonspic'}).find('div', attrs={'class': 'cont'}).find('p').getText()
sons = all.find('div',attrs={'class': 'sons'})
yishiUrl = None
if sons:
yishiUrl = sons.get('id')
if yishiUrl:
yishiUrl = 'https://so.gushiwen.org/authors/ajaxziliao.aspx?id=' + all.find('div',attrs={'class': 'sons'})['id'].replace('fanyi','')
else:
yishiUrl = None
return author, desc, yishiUrl
def parseAuthorMore(html):
soup = BeautifulSoup(html, features='html.parser')
yishi = []
if soup.find('div', attrs={'clase', 'contyishang'}):
for p in soup.find('div', attrs={'clase', 'contyishang'}).find_all('p'):
yishi.append(p.getText())
return ''.join(yishi)
def main():
url = DOWNLOAD_URL
keys = ['author', 'desc', 'story']
with codecs.open('author.json', 'w', encoding='utf-8') as fp:
fp.write('[')
while url:
html = download_page(url)
data, url = parseHtml(html)
for item in data:
authorHtml = download_page(item)
author, desc, yishiUrl = parseAuthorHtml(authorHtml)
if yishiUrl:
yishiHtml = download_page(yishiUrl)
yishi = parseAuthorMore(yishiHtml)
else:
yishi = ""
fp.write(buildJson(keys, [author, desc,yishi]))
fp.write(',')
fp.write(']')
if __name__ == '__main__':
main()
整體思路：請求-->解析數(shù)據(jù)-->存儲(chǔ)
請求使用了requests庫，佯裝正常的瀏覽器請求，獲取到數(shù)據(jù)之后，將數(shù)據(jù)交給BeautifulSoup來做數(shù)據(jù)解析，主要是找到html中的相關(guān)標(biāo)簽并獲取其相關(guān)屬性值和標(biāo)簽內(nèi)容，然后組裝成自己要的目標(biāo)作者的詳情頁url，之后訪問詳情頁，再獲取數(shù)據(jù)并存儲(chǔ)再本地
