Python網(wǎng)絡(luò)爬蟲05——實(shí)例一

代碼如下:

import json
import re
import requests
import time
from requests.exceptions import RequestException
from multiprocessing import Pool

author = 'damao'

"""爬取貓眼電影TOP100"""


def get_one_page(url):
    # header = {"User-Agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 7_1_2 like Mac OS X) App leWebKit/537.51.2 (KHTML, like Gecko) Version/7.0 Mobile/11D257 Safari/9537.53"}
    header = {"User-Agent":"Mozilla/5.0 (Windows; U; Windows NT 5.1) Gecko/20070309 Firefox/2.0.0.3"}
    try:
        response = requests.get(url=url,headers=header)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        return None

def parse_one_page(html):
    pattern = re.compile('<dd>.*?board-index.*?>(\d+)</i>.*?data-src="(.*?)".*?name"><a'
                         +'.*?>(.*?)</a>.*?star">(.*?)</p>.*?releasetime">(.*?)</p>'
                          +'.*?integer">(.*?)</i>.*?fraction">(.*?)</i>.*?</dd>',re.S)
    items = re.findall(pattern=pattern,string=html)
    for item in items:
        yield {
                "index": item[0],
                "image": item[1],
                "name": item[2],
                "acyor": item[3].strip()[3:],
                "time" : item[4].strip()[5:],
                "score": item[5]+item[6]
        }

def write_to_file(content):
    with open(file="test.txt",mode='a',encoding="utf-8") as f:
        f.write(json.dumps(content) + "\n")


def main(offset):
    url = 'http://maoyan.com/board/4?offset=' + str(offset)
    html = get_one_page(url=url)
    # print(html)
    for item in parse_one_page(html):
        print(item)
        write_to_file(item)
        # time.sleep(1)


if __name__ == '__main__':
    # for i in range(10):
    #     main(i*10)
    pool = Pool()
    pool.map(main,[i*10 for i in range(10)])
最后編輯于
?著作權(quán)歸作者所有,轉(zhuǎn)載或內(nèi)容合作請(qǐng)聯(lián)系作者
【社區(qū)內(nèi)容提示】社區(qū)部分內(nèi)容疑似由AI輔助生成,瀏覽時(shí)請(qǐng)結(jié)合常識(shí)與多方信息審慎甄別。
平臺(tái)聲明:文章內(nèi)容(如有圖片或視頻亦包括在內(nèi))由作者上傳并發(fā)布,文章內(nèi)容僅代表作者本人觀點(diǎn),簡書系信息發(fā)布平臺(tái),僅提供信息存儲(chǔ)服務(wù)。

友情鏈接更多精彩內(nèi)容