新手學(xué)python爬蟲中,用這個網(wǎng)站練了一下手,代碼還是有點(diǎn)亂,還需要優(yōu)化,這是第一個版本,沒有加超線程和錯誤調(diào)試

在win10下測試通過,不知道為啥放在在linux執(zhí)行到filename那行出錯,碎覺了不改了.

import requests
from bs4 import BeautifulSoup
import os
import re

headers={'Referer': 'http://www.kuaikanmanhua.com/',
         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36',}

s=requests.session()
s.headers=headers

#基本設(shè)置
#保存根路徑,默認(rèn)在當(dāng)前目錄下
basepath='./'
#漫畫集的首頁,示例見下面的網(wǎng)址
url='http://www.kuaikanmanhua.com/web/topic/2047/'

def savejpg(url,path):
    global s
    filename=re.search('.*/(.*\.jpg)',url)[1]
    res=s.get(url)
    #res=requests(url,headers=headers)
    if res.status_code==200:
        print('保存圖片'+filename+'到'+path)
        with open(path+filename,'wb') as f:
            f.write(res.content)
            f.close()

def get_imgs(url,path):
    global s
    html=s.get(url).text
    soup=BeautifulSoup(html,'html.parser')
    img_links=soup.select('.kklazy')
    for img_link in img_links:
        savejpg(img_link['data-kksrc'],path)


def parser_index(url):
    comic_img_info={}
    soup=BeautifulSoup(s.get(url).text,'html.parser')
    comic_name=soup.select('.comic-name')[0].text
    comic_titles=soup.findAll('a',attrs={'class':' article-img'})
    for titles in comic_titles:
        comic_img_info['name']=comic_name
        comic_img_info['title']=titles['title']
        comic_img_info['url']='http://www.kuaikanmanhua.com/'+titles['href']
        yield comic_img_info

def main(url=url,basepath=basepath):
    for item in parser_index(url):
        path=basepath+item['name']+'/'+item['title']+'/'
        if not os.path.exists(path):os.makedirs(path)
        get_imgs(item['url'],path)

if __name__=='__main()__':
    main()
  

捕獲.PNG