python腳本先貼上

# -*- coding: utf-8 -*-

import requests
import re
from urllib.parse import urljoin
import csv
import chardet

header = {
   'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
                 'Chrome/70.0.3538.77 Safari/537.36',
}

def web_link_collection(page_url):
   lis = []
   try:
       resp = requests.get(page_url, headers=header, timeout=3)
   except Exception as e:
       print('error:', e)
   else:
       encoding = chardet.detect(resp.content)["encoding"]
       resp.encoding = encoding
       link_list = re.findall(r'<a.*?href=["\'](.*?)["\'].*?>(.*?)</a>', resp.text, re.S | re.I)
       for link in link_list:
           url = urljoin(page_url, link[0])
           sc = url_status(url.strip())
           text = link[1]
           print([url, sc, text])
           lis.append([url, sc, text])
   return lis

def url_status(url):
   try:
       res = requests.get(url, headers=header, allow_redirects=False)
   except Exception as e:
       print('error:', e)
   else:
       sc = res.status_code
       if sc == 301 or sc == 302:
           loc_url = res.headers['Location']
           sc = '%s#%s' % (sc, loc_url)
       return sc

if __name__ == '__main__':
   url = ''  # 需要查詢的頁(yè)面
   lis = web_link_collection(url)

   file = open('result.csv', 'w', encoding='utf-8', newline='')
   cw = csv.writer(file)
   cw.writerow(['URL', 'Status_code', '鏈接文本'])
   for line in lis:
       cw.writerow(line)
   file.close()

腳本說(shuō)明

1. 功能：采集網(wǎng)頁(yè)上所有鏈接，存入csv。

• 格式：url - 狀態(tài)碼（如果是301或302，則獲取跳轉(zhuǎn)后的鏈接） - 鏈接文本

2. 作用

• 是否存在500、404、301、302等頁(yè)面。
• 文字鏈接：鏈接文本是否設(shè)置正確；圖片鏈接：查看是否有alt屬性。
• 部分鏈接是否需要添加rel="nofollow"，或添加title=""屬性，或使用JS代替。

擴(kuò)展閱讀：https://searchengineland.com/heres-what-happened-when-i-followed-googlebot-for-3-months-308674