很多人可能都沒想過用selenium能做多線程爬蟲
其實實現(xiàn)起來都是一樣的簡單

導入相關庫

from selenium import webdriver
from multiprocessing.dummy import Pool

訪問

def spider(url):
    driver = webdriver.Chrome('chromedriver')
    driver.get(url)
    html = driver.page_source
    print(html)
    driver.close()
    driver.quit()

控制

def run(x):
    url = 'https://www.baidu.com/s?wd=今天有哪些美女&pn={}'
    pages = []
    for i in range(0, x * 10, 10):
        page = url.format(i)
        pages.append(page)
    print(pages)
    pool = Pool(5)
    result = pool.map(spider, pages)
    pool.close()
    pool.join()
    return result

試一下

if __name__ == '__main__':
    run(10)