修改了豆瓣電影的名字個(gè)數(shù)以及利用正則表達(dá)式只留下評(píng)分人數(shù)中的數(shù)字

# -*- coding: utf-8 -*-
# import scrapy
from scrapy.spiders import CrawlSpider
from scrapy.http import Request
from douban.items import DoubanItem
import urllib
import re


class DoubanMovieTop250Spider(CrawlSpider):
    name = 'douban'  # 此處name必須與項(xiàng)目名字一致
    allowed_domians = ['douban.com']

    def start_requests(self):
        # , headers={'User_Agent': 'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'})
        headers = {
            # 'USER_AGENT': 'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36',
            'referer': 'https://www.baidu.com/link?url=7u5fi9zxZ7UhcghOj540fjWpmi_nT2hX2EEpXKItLyv78gYcl3zlgHfVo15ZkzTY&wd=&eqid=d2ac1cbe001b4b8c0000000358fb62bf'
        }
        yield Request("https://movie.douban.com/tag/", headers=headers, callback=self.tag_page)

    def tag_page(self, response):
        alltag_url = response.xpath(
            '//*[contains(@class,"tagCol")]//@href').extract()
        for url in alltag_url:
            yield Request(urllib.parse.urljoin(response.url, url), callback=self.index_page)

    def index_page(self, response):
        selector_nextpage = response.xpath(
            '//*[contains(@class,"next")]/a/@href').extract()
        # //*[contains(@class,"next")]/a/@href與//*[contains(@class,"next")]//@href效果一樣
        for url in selector_nextpage:
            yield Request(urllib.parse.urljoin(response.url, url), callback=self.index_page)

        selector_items = response.xpath(
            '//tr[@class="item"]')
        for item in selector_items:
            yield self.parse_item(item, response)

    def parse_item(self, selector, response):
        movie = DoubanItem()
        # title1 = selector.xpath(
        #     './/*[@class="pl2"]/a/text()').extract()
        # title2 = selector.xpath(
        #     './/*[@class="pl2"]/a/span/text()').extract()
        # if title2 == []:
        #     for i in range(len(title1)):
        #         t1 = title1[i]
        #     t1 = t1.strip()
        #     movie['title'] = [t1]
        # if title2 == []:
        #     t1 = title1[0].strip()
        #     t1 = t1.strip()
        #     movie['title'] = [t1]
        # else:
        #     movie['title'] = title2
        t = selector.xpath(
            './/*[@class="pl2"]/a/text()').extract()[0].replace('/', '').strip()
        # 將xpath獲取的標(biāo)題有tuple轉(zhuǎn)為字符串，再將字符串末尾中的'/'替換為空格，最后使用strip函數(shù)去掉字符串首尾的空格和換行符
        movie['title'] = [t]  # 將處理好的字符轉(zhuǎn)換成tuple賦值給movie['title']
        # movie['title'] = selector.xpath(
        #     './/*[@class="pl2"]/a/text()').extract()[0]  # .replace('/', '').strip()
        movie['star'] = selector.xpath(
            './/*[@class="rating_nums"]/text()').extract()
        num1 = selector.xpath(
            './/span[@class="pl"]/text()').extract()[0]
        num2 = re.findall(r"\d+", num1)  # 正則xpath提取的評(píng)分人數(shù)，只獲取數(shù)值
        movie['score']=num2
      #  for _, num in enumerate(num2):
       #   num = num.group()
        #  movie['score'] = [str(num)]
        # movie['score']= selector.xpath(
        #     './/span[@class="pl"]/text()').extract()
        return movie