import requests
import time
from lxml import etree
import xlwt
headers = {
? ? 'User-Aegnt':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'
}
all_info_lists = []
def get_info(url,headers=headers):
? ? res = requests.get(url)
? ? html = etree.HTML(res.text)
? ? infos = html.xpath('//div[@class="col1"]/div')
? ? for info in infos:
? ? ? ? try:
? ? ? ? ? ? id = info.xpath('div[1]/a[2]/h2/text()')[0]
? ? ? ? except IndexError:
? ? ? ? ? ? pass
? ? ? ? ? ? id = info.xpath('div[1]/span[2]/h2/text()')[0]
? ? ? ? content1 = info.xpath('a[1]/div/span[1]')[0]
? ? ? ? content = content1.xpath('string(.)').strip()
? ? ? ? laugh = info.xpath('div[2]/span[1]/i/text()')[0]
? ? ? ? comment = info.xpath('div[2]/span[2]/a/i/text()')[0]
? ? ? ? info_list = [id,content,laugh,comment]
? ? ? ? all_info_lists.append(info_list)
if __name__ == '__main__':
? ? book = xlwt.Workbook(encoding='utf-8')
? ? sheet = book.add_sheet('sheet1')
? ? header =['id', 'content', 'laugh', 'comment']
? ? for t in range(len(header)):
? ? ? ? sheet.write(0, t, header[t])
? ? urls = ['https://www.qiushibaike.com/text/page/{}/'.format(str(i)) for i in range(1,14)]
? ? for url in urls:
? ? ? ? get_info(url)
? ? ? ? time.sleep(2)
? ? i = 1
? ? for list in all_info_lists:
? ? ? ? j = 0
? ? ? ? for data in list:
? ? ? ? ? ? sheet.write(i, j, data)
? ? ? ? ? ? j += 1
? ? ? ? i += 1
? ? book.save('C:/Users/madin/Desktop/test.xls')
