1. items.py

    class DouyuspiderItem(scrapy.Item):
        name = scrapy.Field()# 存儲(chǔ)照片的名字
        imagesUrls = scrapy.Field()# 照片的url路徑
        imagesPath = scrapy.Field()# 照片保存在本地的路徑

2. spiders/douyu.py

import scrapy
import json
from douyuSpider.items import DouyuspiderItem

class DouyuSpider(scrapy.Spider):
    name = "douyu"
    allowd_domains = ["http://capi.douyucdn.cn"]

    offset = 0
    url = "http://capi.douyucdn.cn/api/v1/getVerticalRoom?limit=20&offset="
    start_urls = [url + str(offset)]

  def parse(self, response):
      # 返回從json里獲取 data段數(shù)據(jù)集合
      data = json.loads(response.text)["data"]

      for each in data:
          item = DouyuspiderItem()
          item["name"] = each["nickname"]
          item["imagesUrls"] = each["vertical_src"]

          yield item

      self.offset += 20
      yield scrapy.Request(self.url + str(self.offset), callback = self.parse)

3. 設(shè)置setting.py

ITEM_PIPELINES = {'douyuSpider.pipelines.ImagesPipeline': 1}

# Images 的存放位置，之后會(huì)在pipelines.py里調(diào)用
IMAGES_STORE = "/Users/Power/lesson_python/douyuSpider/Images"

# user-agent
USER_AGENT = 'DYZB/2.290 (iPhone; iOS 9.3.4; Scale/2.00)'

4. pipelines.py

    import scrapy
    import os
    from scrapy.pipelines.images import ImagesPipeline
    from scrapy.utils.project import get_project_settings

    class ImagesPipeline(ImagesPipeline):
        IMAGES_STORE = get_project_settings().get("IMAGES_STORE")

        def get_media_requests(self, item, info):
            image_url = item["imagesUrls"]
            yield scrapy.Request(image_url)

        def item_completed(self, results, item, info):
            # 固定寫(xiě)法，獲取圖片路徑，同時(shí)判斷這個(gè)路徑是否正確，如果正確，就放到 image_path里，ImagesPipeline源碼剖析可見(jiàn)
            image_path = [x["path"] for ok, x in results if ok]

            os.rename(self.IMAGES_STORE + "/" + image_path[0], self.IMAGES_STORE + "/" + item["name"] + ".jpg")
            item["imagesPath"] = self.IMAGES_STORE + "/" + item["name"]

            return item

    #get_media_requests的作用就是為每一個(gè)圖片鏈接生成一個(gè)Request對(duì)象，這個(gè)方法的輸出將作為item_completed的輸入中的results，results是一個(gè)元組，每個(gè)元組包括(success, imageinfoorfailure)。如果success=true，imageinfoor_failure是一個(gè)字典，包括url/path/checksum三個(gè)key。

在項(xiàng)目根目錄下新建main.py文件,用于調(diào)試

from scrapy import cmdline
  cmdline.execute('scrapy crawl douyu'.split())

執(zhí)行程序

 py2 main.py