以爬取各城市每天的空氣質(zhì)量數(shù)據(jù)為例

• 安裝好scrapy后（可用conda安裝）在cmd中輸入以下代碼完成爬蟲準(zhǔn)備工作

cd Desktop
:: 創(chuàng)建projiect
scrapy startproject wheather
cd wheather
:: 需要爬取的網(wǎng)址，以真氣網(wǎng)為例
scrapy genspider Wheather www.aqistudy.cn
:: 查看需要爬取的具體網(wǎng)址
scrapy shell https://www.aqistudy.cn/historydata/
view(response)

1. 編輯wheather文件夾中的items.py文件，確定爬取的變量

• 修改（或新增）WheatherItem的內(nèi)容，該案例只爬取3個(gè)變量，結(jié)果如下

class WheatherItem(scrapy.Item):
    city = scrapy.Field(serializer=lambda x: x[8:])  # 城市
    date = scrapy.Field()  # 日期
    quality = scrapy.Field()  # 空氣質(zhì)量
    # location = scrapy.Field()  # 地點(diǎn)

2. 編輯Pipelines.py文件，增加輸出格式（可忽略該部分）

• 該案例添加json輸出格式

# 輸出json.............................................
import json
class WheatherPipeline(object):
    def __init__(self):
        self.f = open('spdr.json', 'w+', encoding='utf-8')
    def process_item(self, item, spider):
        content = json.dumps(dict(item), ensure_ascii=False) + '\n'
        self.f.write(content)
        return item
    def close_spider(self, spider):
        self.f.close()

3. 編輯settings.py文件

#..............................................
# 改為False
ROBOTSTXT_OBEY = False
# 加代理
USER_AGENT ='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36' 
# log級(jí)別設(shè)為warning
LOG_LEVEL='WARNING' 
# 導(dǎo)出json
ITEM_PIPELINES = {
   'wheather.pipelines.WheatherPipeline': 300,
}
# 涉及到動(dòng)態(tài)加載，該案例用scrapy-selenium包來(lái)嫁接scrapy和selenium。
# 網(wǎng)上有不少教程自己寫中間件來(lái)引入selenium，可以但是沒(méi)有必要?。?！
DOWNLOADER_MIDDLEWARES  = {
     'scrapy_selenium.SeleniumMiddleware':800
}
SELENIUM_DRIVER_NAME= 'chrome'  # 用chromedriver
SELENIUM_DRIVER_EXECUTABLE_PATH = r"C:/Anaconda3/chromedriver.exe"
SELENIUM_DRIVER_ARGUMENTS = ['--headless',]  # '--start-maximized'
# 原包不提供無(wú)圖模式，該案例對(duì)包的代碼進(jìn)行了修改
# 若不需要無(wú)圖模式，可忽略余下部分
SELENIUM_DRIVER_EXP_ARGUMENTS=[{"profile.managed_default_content_settings.images": 2}] 
# 修改部分如下
'''
browser_executable_path,driver_exp_arguments):  #.........wdy
or exp_argument in driver_exp_arguments: #..................... wdy
     driver_options.add_experimental_option("prefs", exp_argument) # ...wdy
driver_exp_arguments = crawler.settings.get('SELENIUM_DRIVER_EXP_ARGUMENTS')  #...............wdy
driver_exp_arguments=driver_exp_arguments #............wdy
'''

4. 編輯爬蟲文件Wheather.py

# -*- coding: utf-8 -*-
# 爬取中國(guó)各個(gè)城市的歷史天氣數(shù)據(jù)
import scrapy
import re
from scrapy.linkextractors import LinkExtractor
from ..items import WheatherItem
from scrapy_selenium import SeleniumRequest
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC

class WheatherSpider(scrapy.Spider):
    '''
    start_requests，用selenium代替request
    parse，獲取城市鏈接
    parse_month，獲取各城市的各日期鏈接
    parse_final，爬
    '''
    name = 'Wheather'
    allowed_domains = ['www.aqistudy.cn']

    def start_requests(self): # 注意名字不要隨意改！?。。。。?        start_urls = 'https://www.aqistudy.cn/historydata/'
        yield SeleniumRequest(url=start_urls, callback=self.parse)
    
    def parse(self, response):
        i=0
        citylink = LinkExtractor(restrict_xpaths='/html/body/div[3]/div/div[1]/div[2]/div[2]')
        citylinks = citylink.extract_links(response)  # 各個(gè)城市的天氣數(shù)據(jù)鏈接   
        for cityurl in citylinks: 
            i+=1
            yield scrapy.Request(url=cityurl.url,meta={'i':i}, callback=self.parse_month)
    
    def parse_month(self, response):
        print(response.meta['i'])  # meta傳遞數(shù)值（對(duì)整體沒(méi)有影響，作者自己看的）
        monthlink = LinkExtractor(restrict_xpaths='/html/body/div[3]/div[1]/div[2]/div[2]/div[2]')
        monthlinks = monthlink.extract_links(response)  # 每個(gè)城市各個(gè)月的天氣數(shù)據(jù)鏈接
        for monthurl in monthlinks:
            if int(str(monthurl.url)[-6:]) == 201712:  # 簡(jiǎn)單起見(jiàn)，只選取了2017年12月的數(shù)據(jù)
                yield SeleniumRequest(url=monthurl.url,wait_time=4,wait_until=EC.presence_of_element_located((By.XPATH, '/html/body/div[3]/div[1]/div[1]/table/tbody/tr[29]/td[2]')),callback=self.parse_final)
    
    def parse_final(self,response):
        long = len(response.selector.xpath('/html/body/div[3]/div[1]/div[1]/table/tbody/tr'))  # 計(jì)算每頁(yè)一共這么多條
        dit = WheatherItem()
        # response.request.meta['driver'].current_url)[0]
        for line in range(2, long+1):
            dit['city'] = re.findall('(.+?)空氣質(zhì)量',response.selector.xpath('//*[@id="title"]/text()').extract_first())[0]
            dit['date'] = response.selector.xpath('/html/body/div[3]/div[1]/div[1]/table/tbody/tr[%s]/td[1]/text()' % str(line)).extract_first()
            dit['quality'] = response.selector.xpath('/html/body/div[3]/div[1]/div[1]/table/tbody/tr[%s]/td[3]/span/text()' % str(line)).extract_first()
            yield dit
# scrapy crawl somespider -s JOBDIR=crawls/somespider-1 （作者自己看的）

5. 運(yùn)行和輸出

• 在cmd中輸入以下代碼可運(yùn)行代碼和輸出csv格式的文件

cd wheather
scrapy crawl Wheather -o books.csv