評(píng)論鏈接

放入excel(negkey)中的內(nèi)容

# -*- coding: utf-8 -*-
"""
Created on Thu Aug 16 14:52:09 2018

@author: Shirley
"""
#好評(píng)鏈接：http://www.dianping.com/shop/2044996/review_all/p2?queryType=reviewGrade&queryVal=good
#差評(píng)鏈接：http://www.dianping.com/shop/2044996/review_all/p2?queryType=reviewGrade&queryVal=bad


import requests
from lxml import etree
import xlrd
import random
import time

data = []
def Comments(url):
    
    headers = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
               'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36',
               'Cookie': '瀏覽器獲取的cookie'
               }
    #url = 'http://www.dianping.com/shop/19110508/review_all?queryType=reviewGrade&queryVal=bad'
    resp = requests.get(url,headers=headers)

    #print (resp.content.decode('utf-8'))#查看爬取網(wǎng)頁(yè)代碼，代碼中文內(nèi)容用二進(jìn)制表示，故print時(shí)要decode

    comments = etree.HTML(resp.text,parser=etree.HTMLParser(encoding='utf-8'))
    commentlong = comments.xpath('//div[@class="review-words Hide"]')#長(zhǎng)評(píng)
    commentshort = comments.xpath('//div[@class="review-words"]')#短評(píng)
    for l in commentlong:
        data.append(l.xpath('string(.)').replace("\t","").replace("\n","").replace(" ","").replace("收起評(píng)論","").encode('gbk','ignore').decode('gbk'))
    #把不需要的字符清除，可以用split()去掉\xa0，\t，\n，但是句子會(huì)被拆分成一個(gè)列表，不方便后續(xù)使用
    for s in commentshort:
        data.append(s.xpath('string(.)').replace("\t","").replace("\n","").replace(" ","").encode('gbk','ignore').decode('gbk'))

def getComments():
    path = 'D:/anaconda/shirleylearn/dazhongdianping/negkey.xlsx'#抓取關(guān)鍵字
    excelfile = xlrd.open_workbook(path)
    keys = excelfile.sheet_by_name('Sheet1')
    n = keys.nrows
    for i in range(0,n):
        key = int(keys.row(i)[0].value)
        for page in range(1,int(keys.row(i)[1].value)+1):
            url = 'http://www.dianping.com/shop/%d/review_all/p%d?queryType=reviewGrade&queryVal=bad'%(key,page)#拼鏈接，修改bad,good即為差評(píng)和好評(píng)
            #print(url)
            Comments(url)
            time.sleep(random.random())

if __name__ == "__main__":
    getComments()
    print (len(data))
    with open('neg.txt','w') as f:#把評(píng)論放入txt,好評(píng)改為pos.txt
        for k in data:
            f.write(k)

抓取結(jié)果展示

爬一次IP就被封了 o(╥﹏╥)o