用Pymongo保存數(shù)據(jù)

爬取豆瓣電影top250movie.douban.com/top250的電影數(shù)據(jù)，并保存在MongoDB中。

items.py

class DoubanspiderItem(scrapy.Item):
    # 電影標(biāo)題
    title = scrapy.Field()
    # 電影評分
    score = scrapy.Field()
    # 電影信息
    content = scrapy.Field()
    # 簡介
    info = scrapy.Field()

spiders/douban.py

import scrapy
from doubanSpider.items import DoubanspiderItem

class DoubanSpider(scrapy.Spider):
    name = "douban"
    allowed_domains = ["movie.douban.com"]
    start = 0
    url = 'https://movie.douban.com/top250?start='
    end = '&filter='
    start_urls = [url + str(start) + end]

    def parse(self, response):

        item = DoubanspiderItem()

        movies = response.xpath("http://div[@class=\'info\']")

        for each in movies:
            title = each.xpath('div[@class="hd"]/a/span[@class="title"]/text()').extract()
            content = each.xpath('div[@class="bd"]/p/text()').extract()
            score = each.xpath('div[@class="bd"]/div[@class="star"]/span[@class="rating_num"]/text()').extract()
            info = each.xpath('div[@class="bd"]/p[@class="quote"]/span/text()').extract()

            item['title'] = title[0]
            # 以;作為分隔，將content列表里所有元素合并成一個新的字符串
            item['content'] = ';'.join(content)
            item['score'] = score[0]
            item['info'] = info[0]
            # 提交item

            yield item

        if self.start <= 225:
            self.start += 25
            yield scrapy.Request(self.url + str(self.start) + self.end, callback=self.parse)

pipelines.py

from scrapy.conf import settings
import pymongo

class DoubanspiderPipeline(object):
    def __init__(self):
        # 獲取setting主機名、端口號和數(shù)據(jù)庫名
        host = settings['MONGODB_HOST']
        port = settings['MONGODB_PORT']
        dbname = settings['MONGODB_DBNAME']

        # pymongo.MongoClient(host, port) 創(chuàng)建MongoDB鏈接
        client = pymongo.MongoClient(host=host,port=port)

        # 指向指定的數(shù)據(jù)庫
        mdb = client[dbname]
        # 獲取數(shù)據(jù)庫里存放數(shù)據(jù)的表名
        self.post = mdb[settings['MONGODB_DOCNAME']]

    def process_item(self, item, spider):
        data = dict(item)
        # 向指定的表里添加數(shù)據(jù)
        self.post.insert(data)
        return item

settings.py

BOT_NAME = 'doubanSpider'

SPIDER_MODULES = ['doubanSpider.spiders']
NEWSPIDER_MODULE = 'doubanSpider.spiders'

ITEM_PIPELINES = {
        'doubanSpider.pipelines.DoubanspiderPipeline' : 300
        }

# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36'

# MONGODB 主機環(huán)回地址127.0.0.1
MONGODB_HOST = '127.0.0.1'
# 端口號，默認是27017
MONGODB_PORT = 27017
# 設(shè)置數(shù)據(jù)庫名稱
MONGODB_DBNAME = 'DouBan'
# 存放本次數(shù)據(jù)的表名稱
MONGODB_DOCNAME = 'DouBanMovies'

運行

啟動MongoDB數(shù)據(jù)庫需要兩個命令：

mongod：是mongoDB數(shù)據(jù)庫進程本身
mongo：是命令行shell客戶端

sudo mongod # 首先啟動數(shù)據(jù)庫服務(wù)，再執(zhí)行Scrapy
sudo mongo # 啟動數(shù)據(jù)庫shell

在mongo shell下使用命令:

# 查看當(dāng)前數(shù)據(jù)庫
> db

# 列出所有的數(shù)據(jù)庫
> show dbs

# 連接DouBan數(shù)據(jù)庫
> use DouBan

# 列出所有表
> show collections

# 查看表里的數(shù)據(jù)
> db.DouBanMoives.find()

day55_爬蟲-scrapy實戰(zhàn)項目-01.png