爬蟲學(xué)習(xí)（四）多線程

1.多線程

簡單說就是在一個任務(wù)進(jìn)程中，采用多個線程來分別完成子任務(wù)，從而提高運(yùn)行速度及效率，需要用到的模塊為threading模塊。
實(shí)例：

import threading
import time

def coding():
    for x in range(1,4):
        print('正在寫%s' %x)
        time.sleep(1)

def drawing():
    for x in  range(1,4):
        print("正在畫%s" %x)
        time.sleep(1)

def main():
    t1 = threading.Thread(target=coding)
    t2 = threading.Thread(target=drawing)
    t1.start()
    t2.start()

if __name__ == '__main__':
    main()

常用方法

threading.enumerate() 查看線程數(shù)
threading.current_thread() 查看線程的名字

繼承自threading.Thread類
實(shí)例：

import threading
import time

class coding(threading.Thread):
    def run(self):
        for x in range(1,4):
            print('正在寫%s' %x)
            time.sleep(1)

class drawing(threading.Thread):
    def run(self):
        for x in  range(1,4):
            print("正在畫%s" %x)
            time.sleep(1)

def main():
    t1 = coding()
    t2 = drawing()
    t1.start()
    t2.start()

if __name__ == '__main__':
    main()

2.多線程全局變量沖突的解決

import threading
value = 0
glock = threading.Lock()
def add_value():
    global value
    glock.acquire() #上鎖,其余線程等待中
    for x in range(1000000):
        value += 1
    glock.release()#釋放
    print(value)
def main():
    for x in range(2):
        t = threading.Thread(target=add_value)
        t.start()
if __name__ == '__main__':
    main()

lock版生產(chǎn)者與消費(fèi)者

import threading
import time
import random

gmoney = 1000
glock = threading.Lock()
totaltimes = 10
times = 0

class producer(threading.Thread):
    def run(self):
        global gmoney,totaltimes,times
        while True:
            money = random.randint(100,1000)
            glock.acquire()
            if times>=totaltimes:
                glock.release()
                break
            gmoney = gmoney+money
            print('%s生產(chǎn)者生產(chǎn)了%d元，剩余%d元' %(threading.current_thread(),money,gmoney))
            times = times+1
            glock.release()
            time.sleep(0.5)
class customer(threading.Thread):
    def run(self):
        global gmoney
        while True:
            money = random.randint(100,1000)
            glock.acquire()
            if gmoney>=money:
                gmoney -= money
                print("%s消費(fèi)者消費(fèi)了%d元，還剩%d元" %(threading.current_thread(),money,gmoney))
            else:
                if times>=totaltimes:
                    glock.release()
                    break
                print("%s消費(fèi)者準(zhǔn)備消費(fèi)%d元，還剩%d元，不足" %(threading.current_thread(),money,gmoney))
            glock.release()
            time.sleep(0.5)

def main():
    for x in range(3):
        t = customer(name="消費(fèi)者線程%d" %x)
        t.start()
    for x in range(5):
        t = producer(name='生產(chǎn)者線程%d' %x)
        t.start()

if __name__ == '__main__':
    main()

condition版生產(chǎn)者與消費(fèi)者(節(jié)省CPU資源)

import threading
import time
import random

gmoney = 1000
gcondition = threading.Condition()
totaltimes = 10
times = 0

class producer(threading.Thread):
    def run(self):
        global gmoney,totaltimes,times
        while True:
            money = random.randint(100,1000)
            gcondition.acquire()
            if times>=totaltimes:
                gcondition.release()
                break
            gmoney = gmoney+money
            print('%s生產(chǎn)者生產(chǎn)了%d元，剩余%d元' %(threading.current_thread(),money,gmoney))
            times = times+1
            gcondition.notify_all() #喚醒所有等待的線程
            gcondition.release()
            time.sleep(0.5)

class customer(threading.Thread):
    def run(self):
        global gmoney
        while True:
            money = random.randint(100,1000)
            gcondition.acquire()
            while gmoney<money: #判斷錢是否足夠
                if times>=totaltimes: #判斷生產(chǎn)者是否生產(chǎn)完
                    gcondition.release()
                    return
                print("%s消費(fèi)者準(zhǔn)備消費(fèi)%d元，還剩%d元，不足" % (threading.current_thread(), money, gmoney))
                gcondition.wait() #錢不足時將線程處于等待狀態(tài)，生產(chǎn)后喚醒
            gmoney = gmoney-money
            print("%s消費(fèi)者消費(fèi)了%d元，還剩%d元" % (threading.current_thread(), money, gmoney))
            gcondition.release()
            time.sleep(0.5)

def main():
    for x in range(3):
        t = customer(name="消費(fèi)者線程%d" %x)
        t.start()
    for x in range(5):
        t = producer(name='生產(chǎn)者線程%d' %x)
        t.start()

if __name__ == '__main__':
    main()

3.Queue線程安全隊(duì)列

先進(jìn)先出隊(duì)列（Queue），后進(jìn)先出隊(duì)列（LifoQueue）
常用操作

from queue import Queue
#創(chuàng)建一個先進(jìn)先出隊(duì)列
Queue(maxsize)
#判斷隊(duì)列是否為空
Queue.empty()
#判斷隊(duì)列是否滿了
Queue.full()
#獲取隊(duì)列最后一個數(shù)據(jù)，即最先進(jìn)入隊(duì)列的數(shù)據(jù)
Queue.get()
#將一個數(shù)據(jù)放到隊(duì)列中
Queue.Put()

實(shí)例：Queue多線程爬取斗圖吧表情

import requests
from lxml import etree
from urllib import request
from queue import Queue
import os
import re
import threading

class Producer(threading.Thread):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'
    }
    def __init__(self,page_queue,img_queue,*args,**kwargs): #初始化父類屬性并新建屬性page_queue,img_queue
        super(Producer,self).__init__(*args,**kwargs) #使producer包含父類所有屬性
        self.page_queue = page_queue
        self.img_queue = img_queue
    def run(self):
        while True:
            if self.page_queue.empty():
                break
            url = self.page_queue.get()
            self.page_parse(url)
    def page_parse(self,url):
        proxy = {
            'http':'113.195.18.53:9999',
            'http':'114.99.23.137:1133',
            'http':'163.204.244.247:9999',
            'http':'123.207.57.145:1080'

        }
        resp = requests.get(url,headers=self.headers,proxies=proxy)
        text = resp.text
        html = etree.HTML(text)
        imgs = html.xpath("http://div[@class='page-content text-center']//img[@class!='gif']") #過濾GIF類型
        for img in imgs:
            img_url = img.get("data-original")
            alt = img.get("alt") #獲取圖片名
            alt = re.sub(r'[\?？\.,，。!！“”\*]','',alt) #去除文字中的特殊符號
            houzhui = os.path.splitext(img_url)[1]#獲取后綴
            filename = alt + houzhui #獲取完整文件名
            self.img_queue.put((img_url,filename)) #添加到下載隊(duì)列

class Customer(threading.Thread):
    def __init__(self,page_queue,img_queue,*args,**kwargs):
        super(Customer,self).__init__(*args,**kwargs)
        self.page_queue = page_queue
        self.img_queue = img_queue
    def run(self):
        while True:
            if self.img_queue.empty() and self.page_queue.empty():
                break
            img_url,filename = self.img_queue.get()
            request.urlretrieve(img_url,'imgs/'+filename) #下載圖片
            print(filename+"打印成功")

def main():
    page_queue = Queue(100)
    img_queue = Queue(100)
    base_url = 'https://www.doutula.com/photo/list/?page={}'
    for x in range(1,3):
        url = base_url.format(x)
        page_queue.put(url) #添加到解析隊(duì)列
    for x in range(5): #創(chuàng)建5個生產(chǎn)者，即5個線程
        t = Producer(page_queue,img_queue)
        t.start()
    for x in range(5):
        t = Customer(page_queue,img_queue)
        t.start()

if __name__ == '__main__':
    main()
?著作權(quán)歸作者所有,轉(zhuǎn)載或內(nèi)容合作請聯(lián)系作者
【社區(qū)內(nèi)容提示】社區(qū)部分內(nèi)容疑似由AI輔助生成，瀏覽時請結(jié)合常識與多方信息審慎甄別。
平臺聲明：文章內(nèi)容（如有圖片或視頻亦包括在內(nèi)）由作者上傳并發(fā)布，文章內(nèi)容僅代表作者本人觀點(diǎn)，簡書系信息發(fā)布平臺，僅提供信息存儲服務(wù)。

友情鏈接更多精彩內(nèi)容