爬蟲--大數(shù)據(jù)-- 使用Xpath語法進行解析--使用lxml中的xpath
路徑表達式
- 提取本地html中的數(shù)據(jù)
# 使用lxml提取 h1 標簽中的內(nèi)容
from lxml import html
# 讀取html文件
with open('./index.html', 'r', encoding='utf-8') as f:
htm_data = f.read()
# print(htm_data)
# 解析html 文件，獲取selector對象
selector = html.fromstring(htm_data)
# selector中調(diào)用xpath方法
# 要獲取標簽中的內(nèi)容，末尾要添加text()
# /從根節(jié)點選取
h1 = selector.xpath('/html/body/h1/text()')
print(h1[0])
# //代表可以從任意位置出發(fā)
# //標簽1[@屬性=屬性值】/標簽2[@屬性=屬性值]
a = selector.xpath('//div[@id="container"]/a/text()')
print(a)
# 獲取p標簽的內(nèi)容
p = selector.xpath('//div[@id="container"]/p/text()')
print(p[0])
# 獲取屬性
h = selector.xpath('//div[@id="container"]/a/@href')
print(h[0])
運行截圖
- 本地html文件
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Title</title>
</head>
<body>
<h1>歡迎來到王者榮耀</h1>
<ul><!--無序列表 布局-->
<li><a ><img src="https://game.gtimg.cn/images/yxzj/img201606/heroimg/131/131.jpg" alt=""> 李白</a></li>
<li><a ><img src="https://game.gtimg.cn/images/yxzj/img201606/heroimg/106/106.jpg" alt=""> 小喬</a></li>
<li><a ><img src="https://game.gtimg.cn/images/yxzj/img201606/heroimg/193/193.jpg" alt=""> 戰(zhàn)士</a></li>
</ul>
<ol><!--有序列表 布局-->
<li>刺客</li>
<li>法師</li>
<li>凱</li>
</ol>
<!--div + css 布局-->
<div>第一個div標簽</div>
<div id="container">
<P>被動：李白使用普通攻擊攻擊敵人時，會積累1道劍氣，持續(xù)3秒；積累4道劍氣后進入俠客行狀態(tài)，增加30點物理攻擊力并解除青蓮劍歌的封印，持續(xù)5秒；攻擊建筑不會積累劍氣</P>
<a >歡迎來到百度</a>
</div>
<div>第三個div標簽</div>
</body>
</html>
Requests
- 導(dǎo)入
import requests
- 方法
# Requests
# 導(dǎo)入
import requests
url = 'http://www.baidu.com/'
response = requests.get(url)
print(response)
# 獲取str類型的響應(yīng)
print(response.text)
# 獲取bytes類型的響應(yīng)
print(response.content)
# 獲取響應(yīng)頭
print(response.headers)
# 獲取狀態(tài)碼
print(response.status_code)
# 獲取編碼方式
print(response.encoding)
print(response.apparent_encoding)
# 200 Ok 404 500
# 沒有添加請求頭的知乎網(wǎng)站
resp = requests.get('https://www.zhihu.com/',)
print(resp.status_code)
#使用字典定義請求頭
header = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36"}
resp = requests.get('https://www.zhihu.com/',headers=header)
print(resp.status_code)
運行截圖
提取當當網(wǎng)的信息 -圖書名稱、圖書購買鏈接、圖書價格、圖書賣家名稱—顯示價格最低的前10家 柱狀圖
import requests
from lxml import html
import pandas as pd
from matplotlib import pyplot as plt
plt.rcParams["font.sans-serif"] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False
def spider_dangdang(isbn):
# 目標站點地址
url = 'http://search.dangdang.com/?key={}&act=input'.format(isbn)
# print(url)
header = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36"}
resp = requests.get(url, headers=header)
html_data = resp.text
# 將html頁面寫入本地
# with open('dangdang.html', 'w', encoding='utf-8') as f:
# f.write(html_data)
# 提取目標站的信息
selector = html.fromstring(html_data)
ul_list = selector.xpath('//div[@id="search_nature_rg"]/ul/li')
print('您好，一共有{}家店鋪售賣次數(shù)此書'.format(len(ul_list)))
book_list = []
for li in ul_list:
# 圖書名稱
title = li.xpath('./a/@title')[0].strip()
print(title)
# 圖書購買鏈接
href = li.xpath('./a/@href')[0]
print(href)
# 圖書價格
price = li.xpath('./p[@class="price"]/span[@class="search_now_price"]/text()')[0]
price = float(price.replace("￥"," "))
print(price)
# 圖書賣家名稱
store = li.xpath('./p[@class="search_shangjia"]/a/text()')
store = '當當自營' if len(store) == 0 else store[0]
print(store)
# 添加每一個商家的圖書信息
book_list.append({
'name': title,
'link': href,
'price': price,
'store': store
})
# 按照價格進行排序
book_list.sort(key=lambda x: x['price'])
# 遍歷book_list
for book in book_list:
print(book)
# 顯示價格最低的前10家 柱狀圖
top10_store = [book_list[i] for i in range(10)]
# x = []
# for store in top10:
# x.append(store['store'])
x = [x['store'] for x in top10_store]
print(x)
y = [x['price'] for x in top10_store]
print(y)
# plt.bar(x,y)
plt.barh(x,y)
plt.show()
# 存儲成csv文件
df = pd.DataFrame(book_list)
df.to_csv('dangdang.csv')
spider_dangdang('9787115428028')
柱狀圖
提取豆瓣電影網(wǎng)站信息—電影名，上映日期，類型，上映國家，想看人數(shù)、根據(jù)想看人數(shù)進行排序、繪制即將上映電影國家的占比圖、繪制top5最想看的電影
import requests
from lxml import html
import pandas as pd
import jieba
from matplotlib import pyplot as plt
plt.rcParams["font.sans-serif"] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False
def Film():
# 目標站點地址
url = 'https://movie.douban.com/cinema/later/chongqing/'
header = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36"}
resp = requests.get(url, headers=header)
html_data = resp.text
# 提取目標站的信息
selector = html.fromstring(html_data)
film = selector.xpath('//div[@id="showing-soon"]/div')
print(film)
div_list = []
for film_list in film:
# 電影名
title_list = film_list.xpath('./div/h3/a/text()')[0]
print(title_list)
# 上映時間
time_list = film_list.xpath('./div/ul/li[1]/text()')[0]
print(time_list)
# 電影類型
type_list = film_list.xpath('./div/ul/li[2]/text()')[0]
print(type_list)
# 上映國家
con_list = film_list.xpath('./div/ul/li[3]/text()')[0]
print(con_list)
# 想看人數(shù)
number_list = film_list.xpath('./div/ul/li[4]/span/text()')[0]
print(number_list)
# 替換
number_list = int(number_list.replace('人想看',''))
# 添加電影信息
div_list.append({
'title': title_list,
'time': time_list,
'type': type_list,
'con': con_list,
'number': number_list
})
# 按照想看人數(shù)排序
div_list.sort(key=lambda x:x['number'], reverse=True )
print(div_list)
# 遍歷
for items_list in div_list:
print(items_list)
# 繪制top5最想看的電影占比圖
# 提取前五部電影信息
top5_store = [div_list[i] for i in range(5)]
# 提取電影名
x = [x['title'] for x in top5_store]
print(x)
# 提取想看人數(shù)
y = [x['number'] for x in top5_store]
print(y)
explode = [0.1, 0, 0, 0, 0]
plt.pie(y, explode=explode, labels=x, shadow=True, autopct='%1.1f%%')
plt.axis('equal')
plt.legend(loc=2)
plt.show()
# 繪制即將上映電影國家的占比圖
counts = {}
# 提取所有上映國家
s = [x['con'] for x in div_list]
print(s)
# 統(tǒng)計上映國家與數(shù)量
for word in s:
counts[word] = counts.get(word, 0) + 1
print(counts)
# 提取上映國家
name = counts.keys()
print(name)
# 提取數(shù)量
number = counts.values()
print(number)
explode1 = [0.1, 0, 0, 0]
plt.pie(number, explode=explode1, labels=name, shadow=True, autopct='%1.1f%%')
plt.axis('equal')
plt.legend(loc=2)
plt.show()
Film()
top5最想看的電影占比圖
上映電影國家的占比圖
