拾玖-Python 異步 爬蟲 MongoDB 等事宜實(shí)施

1. 綜述

根據(jù)前期的 NodeJS 代理池開展的爬蟲終于有所進(jìn)展，技術(shù)實(shí)現(xiàn)已初步實(shí)現(xiàn)，相關(guān)引用資料如下所示：

2.代碼及注釋

'''
@Descripttion: 爬蟲測試頁面
@Author: BerryBC
@Date: 2020-01-30 15:00:34
@LastEditors  : BerryBC
@LastEditTime : 2020-01-31 13:53:19
'''
# 超時(shí)
# https://www.cnblogs.com/gl1573/p/10129382.html

# 爬蟲
# http://c.biancheng.net/view/2011.html

# 鏈接MongoDB
# https://blog.51cto.com/1767340368/2092960

# 異步處理
# https://blog.csdn.net/tinyzhao/article/details/52684473             //這不懂，做不了異步
# https://github.com/Blackyukun/IPProxyPool                           //這是有些代碼
# https://segmentfault.com/a/1190000008814676?utm_source=tag-newest   //這可以了
# https://www.cnblogs.com/shenh/p/9090586.html                        //最大并發(fā)數(shù)
# https://docs.aiohttp.org/en/stable/client_advanced.html             //沒啥事還是得去看官方文檔


# 引用庫
import requests
import time
import pymongo
from bs4 import BeautifulSoup
from target import arrTarget
import random
import time
import asyncio
import aiohttp

# 鏈接 MongoDB 的鏈接信息以及引用相關(guān)的表
myclient = pymongo.MongoClient("mongodb://localhost:27017/")
mydb = myclient["dbProxy"]
mydb.authenticate('Berry', 'Berry')
mycol = mydb["tbProxy"]

# 讀取 MongoDB 中的代理信息
arrProxy = []
# 其中從數(shù)據(jù)庫中讀取 fail 次數(shù)小于等于8次的，只讀取 u 跟 p 字段，按 fail 倒序排序
for x in mycol.find({"fail": {"$lte": 8}}, {"_id": 0, "u": 1, "p": 1}).sort("fail", -1):
    arrProxy.append(x)

# 傳入給 aiohttp 用的信息
url = 'http://***.******.***/'
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36'}
intProxyLen = len(arrProxy)

# 聲明拼死干活的函數(shù)
async def funSpyWeb(eleWeb, inSemaphore):
    # 限制并發(fā)數(shù)
    async with inSemaphore:
        bolRetry = True
        intTryTime = 0
        url = eleWeb
        try:
            async with aiohttp.ClientSession() as session:
                # 判斷是否需要重試以及是否所有代理均已用完
                while (bolRetry and (intTryTime < intProxyLen)):
                    try:
                        # 導(dǎo)入代理
                        real_proxy = "http://" + \
                            arrProxy[intTryTime]["u"] + \
                            ":"+arrProxy[intTryTime]["p"]
                        # 異步請(qǐng)求網(wǎng)頁內(nèi)容
                        async with session.get(url, proxy=real_proxy, timeout=5,headers=headers) as response:
                            # 如果成功了，這是測試，所以就暫時(shí)不管是否成功，只要不超時(shí)就行了
                            # if response.status == 200:
                            #     self._conn.put(proxy)
                            #     print('Valid proxy', proxy)
                            strhtml = await response.text()
                            # 列出所有鏈接，方便以后處理
                            soup = BeautifulSoup(strhtml, 'lxml')
                            data = soup.select('a')
                            # for item in data:
                            #     result = {
                            #         'title': item.get_text(),
                            #         'link': item.get('href')
                            #     }
                            #     print(result)
                            # 如果鏈接不超時(shí)即完成。
                            bolRetry = False
                            print("  After " + str(intTryTime) +
                                    " time, success reach " + url)
                            # print(data[0].get_text())
                    except Exception as e:
                        intTryTime += 1
                        # print('    Fail ' + str(intTryTime) + ' time')
        except Exception as e:
            print(e)

# 這里就只是調(diào)用而已，并且記錄開始結(jié)束時(shí)間
print('Begin : '+time.strftime('%Y-%m-%d %H:%M:%S'))
loop = asyncio.get_event_loop()
# 定義并發(fā)線程數(shù)
semaphore = asyncio.Semaphore(20)
# 把待爬 URL 放進(jìn)其中
waittask = asyncio.gather(
    *([funSpyWeb(strWebSite, semaphore) for strWebSite in arrTarget]))
loop.run_until_complete(waittask)
loop.close()
print('Program End : '+time.strftime('%Y-%m-%d %H:%M:%S'))


# --------- 以下不懂，失敗嘗試-------------
# class classSpy():
#     def __init__(self, arrInProxy):
#         self.arrProxy = iter(arrInProxy)

#     def __aiter__(self):
#         return self

#     async def __anext__(self):
#         try:
#             eleProxy = next(self.arrProxy)
#         except StopIteration:
#             raise StopAsyncIteration
#         return eleProxy
# arrTmp = []
# arrTmp.append(1)
# arrTmp.append(2)
# arrTmp.append(3)
# arrTmp.append(4)
# arrTmp.append(5)
# arrTmp.append(6)
# async def run():
#     print('Begin : '+time.strftime('%Y-%m-%d %H:%M:%S'))
#     async for eleBe in classSpy(arrTmp):
#         await asyncio.sleep(random.randint(1, 3))
#         print('  Now : ' + str(eleBe) + ' , time: ' +
#             time.strftime('%Y-%m-%d %H:%M:%S'))
#     print('End : '+time.strftime('%Y-%m-%d %H:%M:%S'))
# loop = asyncio.get_event_loop()
# loop.run_until_complete(run())
# ------------------------------------------


# --------- 以下為異步嘗試成功-------------
# arrTmp = []
# arrTmp.append(1)
# arrTmp.append(2)
# arrTmp.append(3)
# arrTmp.append(4)
# arrTmp.append(5)
# arrTmp.append(6)
# async def run(eleBe,inSemaphore):
#     async with inSemaphore:
#         await asyncio.sleep(random.randint(1, 3))
#         print('  Now : ' + str(eleBe) + ' , time: ' +
#             time.strftime('%Y-%m-%d %H:%M:%S'))
# def funDone(waittask):
#     print('Callback End : '+time.strftime('%Y-%m-%d %H:%M:%S'))
# print('Begin : '+time.strftime('%Y-%m-%d %H:%M:%S'))

# # -------------調(diào)用方式1----------------
# async def main():
#     semaphore=asyncio.Semaphore(2)
#     waittask =asyncio.gather (*( [run(proxy,semaphore) for proxy in arrTmp]))
#     waittask.add_done_callback(funDone)
#     await asyncio.gather(waittask)
# asyncio.run(main())
# # -------------------------------------


# # -------------調(diào)用方式2----------------
# loop = asyncio.get_event_loop()
# semaphore=asyncio.Semaphore(2)
# waittask =asyncio.gather (*( [run(proxy,semaphore) for proxy in arrTmp]))
# waittask.add_done_callback(funDone)
# loop.run_until_complete(waittask)
# loop.close()
# # -------------------------------------


# print('Program End : '+time.strftime('%Y-%m-%d %H:%M:%S'))
# ------------------------------------------

3.輸出結(jié)果

  ...
  After 2 time, success reach http://nx.cntour.cn/
  After 48 time, success reach http://www.cntour.cn/usercenter/default.aspx
  After 48 time, success reach http://bj.cntour.cn/
  After 48 time, success reach http://nmg.cntour.cn/
  After 48 time, success reach http://cntour.cn
  After 97 time, success reach http://sd.cntour.cn/
  After 97 time, success reach http://sh.cntour.cn/
  After 58 time, success reach http://sc.cntour.cn/
  After 58 time, success reach http://us.cntour.cn/
  After 21 time, success reach http://jp.cntour.cn/
  After 21 time, success reach http://www.cntour.cn/domestic/237/
  After 87 time, success reach http://mc.cntour.cn/
  ...

4.后續(xù)思考

其實(shí)主要做的當(dāng)然是把爬出來的 URL 放進(jìn)數(shù)據(jù)庫，然后標(biāo)識(shí)時(shí)間，定期情理。
每次爬的都是少量，可以嘗試爬門戶網(wǎng)站或者社交網(wǎng)站等。
之后當(dāng)然要考慮爬相關(guān)網(wǎng)站時(shí)，如何區(qū)分已爬信息跟未爬信息，這個(gè)可能需要重點(diǎn)考慮或者考慮個(gè)性化。

?著作權(quán)歸作者所有,轉(zhuǎn)載或內(nèi)容合作請(qǐng)聯(lián)系作者
【社區(qū)內(nèi)容提示】社區(qū)部分內(nèi)容疑似由AI輔助生成，瀏覽時(shí)請(qǐng)結(jié)合常識(shí)與多方信息審慎甄別。
平臺(tái)聲明：文章內(nèi)容（如有圖片或視頻亦包括在內(nèi)）由作者上傳并發(fā)布，文章內(nèi)容僅代表作者本人觀點(diǎn)，簡書系信息發(fā)布平臺(tái)，僅提供信息存儲(chǔ)服務(wù)。

相關(guān)閱讀更多精彩內(nèi)容

友情鏈接更多精彩內(nèi)容