爬蟲

• 在Gogle瀏覽器上安裝Xpath Helper插件
• 實(shí)例：爬圖書的價格，排序等

import requests
from lxml import html
import pandas as pd
from matplotlib import pyplot as plt
plt.rcParams["font.sans-serif"] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False
def spider_dangdang(isbn):
    book_list = []
    # 目標(biāo)站點(diǎn)地址
    url = 'http://search.dangdang.com/?key={}&act=input'.format(isbn)
    # print(url)
    # 獲取站點(diǎn)str類型的響應(yīng)
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36"}

    resp = requests.get(url, headers=headers)
    html_data = resp.text
    #  將html頁面寫入本地
    # with open('dangdang.html', 'w', encoding='utf-8') as f:
    #     f.write(html_data)

    # 提取目標(biāo)站的信息
    selector = html.fromstring(html_data)
    ul_list = selector.xpath('//div[@id="search_nature_rg"]/ul/li')
    print('您好，共有{}家店鋪售賣此圖書'.format(len(ul_list)))
    # print('qwertyui', ul_list)

    # 遍歷 ul_list
    for li in ul_list:
        #  圖書名稱
        title = li.xpath('./a/@title')[0].strip()
        print('title---', title)
        #  圖書購買鏈接
        link = li.xpath('a/@href')[0]
        # print(link)
        #  圖書價格
        price = li.xpath('./p[@class="price"]/span[@class="search_now_price"]/text()')[0]
        #print(price)
        price = float(price.replace('￥',''))
        print(price)
        # 圖書賣家名稱
        store = li.xpath('./p[@class="search_shangjia"]/a/text()')
        store = '當(dāng)當(dāng)自營' if len(store) == 0 else store[0]
        # print(store)

        # 添加每一個商家的圖書信息
        book_list.append({
            'title':title,
            'price':price,
            'link':link,
            'store':store
        })


    # 按照價格進(jìn)行排序
    book_list.sort(key=lambda x:x['price'])

    # 遍歷booklist
    for book in book_list:
        print(book)

    # 展示價格最低的前10家 柱狀圖
    # 店鋪的名稱
    top10_store = [book_list[i] for i in range(10)]
    # x = []
    # for store in top10_store:
    #     x.append(store['store'])
    x = [x['store'] for x in top10_store]
    print(x)
    # 圖書的價格
    y = [x['price'] for x in top10_store]
    print(y)
    # plt.bar(x, y)
    plt.barh(x, y)
    plt.show()

    # 存儲成csv文件
    df = pd.DataFrame(book_list)
    df.to_csv('dangdang.csv')

spider_dangdang('9787115428028')

• 作業(yè)：爬電影網(wǎng)站，得到電影名、想看人數(shù)等信息，繪制分析圖

import requests
from lxml import html
import pandas as pd
from matplotlib import pyplot as plt
plt.rcParams["font.sans-serif"] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False
def spider_douban():
    movie_list = []
    # 目標(biāo)站點(diǎn)地址
    url = 'https://movie.douban.com/cinema/later/chongqing/?qq-pf-to=pcqq.group'
    print(url)
    # 添加頭文件，偽裝成瀏覽器，防止被發(fā)現(xiàn)
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36"}
    resp = requests.get(url, headers=headers)
    html_data = resp.text
    selector = html.fromstring(html_data)
    ul_list = selector.xpath("http://div[@id='showing-soon']/div")
    print('重慶即將上映的電影有',len(ul_list),'部') #打印有多少部電影
    # 遍歷 ul_list
    for li in ul_list:
        i = 0
        # i = 0
        # 電影名
        name_movie = li.xpath("./div/h3/a/text()")[0]
        # print('電影名', name_movie)
        # 上映日期
        line_data = li.xpath("./div/ul/li[1]/text()")[i]
        # print('上映日期', line_data)
        # 類型
        movie_type = li.xpath("./div/ul/li[2]/text()")[i]
        # print('電影類型', movie_type)
        # 上映國家
        movie_city = li.xpath("./div/ul/li[3]/text()")[i]
        # print('上映國家', movie_city)
        # 想看人數(shù)
        movie_wantnum = li.xpath("./div/ul/li/span/text()")[i]
        # print('想看人數(shù)', movie_wantnum)
        movie_wantnum = int(movie_wantnum.replace('人想看', ''))
        movie_list.append({
            '電影名': name_movie,
            '上映日期': line_data,
            '類型': movie_type,
            '上映國家': movie_city,
            '想看人數(shù)': movie_wantnum,
        })
        i += 1
    movie_list.sort(key=lambda a: a['想看人數(shù)'], reverse=True)
    print(movie_list)
    movie_city = []
    for i in movie_list:
        movie_city.append(i['上映國家'])
    print('上映國家', movie_city)
    ################# 繪制上映國家云詞 
    text = ' '.join(movie_city)
    print(text)
    from wordcloud import WordCloud
    import imageio
    mask = imageio.imread('./image/china.jpg')
    WordCloud(
        font_path='msyh.ttc',
        background_color='black',
        width=800,
        height=600,
        collocations=False,  # 相鄰兩個重復(fù)詞之間的匹配
        mask=mask
    ).generate(text).to_file('上映國家.png')

    # 繪制 上映國家占比
    from random import randint
    from matplotlib import pyplot as plt
    # 解決亂碼
    plt.rcParams["font.sans-serif"] = ['SimHei']
    plt.rcParams['axes.unicode_minus'] = False
    import numpy as np
    counts = {} # 上映國家和人數(shù)
    lli = [] # 上映國家
    lii = [] # 每個國家出現(xiàn)的次數(shù)
    for i in movie_city:
        counts[i] = counts.get(i, 0)+1
    print(counts)
    items = list(counts.items())  # 將字典轉(zhuǎn)換為列表
    print(len(items))
    items.sort(key=lambda x: x[1], reverse=True)  # 使用函數(shù)從大到小排序
    for i in range(len(items)): # 有多少個國家循環(huán)多少次，解包 出國家和國家的次數(shù)
        role, count = items[i]  # 序列解包
        lli.append(count)
        lii.append(role)
    #     for _ in range(count):
    #         li.append(role)
    # print(li)
    print(lii)
    print(lli)
    explode = [0.1, 0, 0, 0]
    plt.pie(lli, explode=explode, shadow=True, labels=lii, autopct='%1.1f%%')

    # top5_store = [movie_list[i] for i in range(5)]
    name_movie = [] #電影名
    for i in movie_list:
        name_movie.append(i['電影名'])
        if len(name_movie) >= 5:
            break
    print('電影名', name_movie)
    # 想看人數(shù)
    movie_wantnum = []
    for i in movie_list:
        movie_wantnum.append(i['想看人數(shù)'])
        if len(movie_wantnum) >= 5:
            break
    print('想看人數(shù)', movie_wantnum)
    plt.barh(name_movie, movie_wantnum)

    plt.show()


spider_douban()


# # # 獲取str類型的響應(yīng)
# # print(response.text)
# # # 獲取bytes類型的響應(yīng)
# # print(response.content)
# # # 獲取響應(yīng)頭
# # print(response.headers)
# # # 獲取狀態(tài)碼
# # print(response.status_code)

上映國家云圖.PNG

上映國家占比.PNG

想看人數(shù)TOP5電影.PNG