抓取基本解釋信息 及其他字典地址

# --------------------------------------
#   抓取字典詳細(xì)信息，保存
#   保存字典其他信息網(wǎng)址
# --------------------------------------

import string
import urllib.request
import re
from bs4 import BeautifulSoup
import codecs
# 頁面地址


# 開網(wǎng)頁地址文件并抓取
def scrapt(zurl):
    print(zurl)
    req = urllib.request.Request(zurl)
    req.add_header('Referer', 'http://www.zdic.net/z/jbs/')
    req.add_header('User-Agent', 'None')
    responseb = urllib.request.urlopen(req)
    index_z = responseb.read()
# 理數(shù)據(jù)得到字典解釋及相關(guān)字典
    index_z = index_z.decode('utf8')
# 個真是無語了
# 取頁面中的信息及字頁面地址
    soup = BeautifulSoup(index_z)
    tab_raw = soup.find_all(attrs={'class' : 'tab-row'})
# 取自他字典地址
    for itemtab in tab_raw:
        zurllist = re.findall(r'/./.*?htm', str(itemtab))
        for line in zurllist:
            zurllistfile.write('http://www.zdic.net'+line+'\n')
# 數(shù)不能為疊加器

    tab_page = soup.find_all(attrs={'class':'tab-page'})
# 取字典解釋獲取解釋內(nèi)容
# 取url中的16進(jìn)制代碼
    keyq = re.split(r'[/.]',zurl)[-2]
    print(keyq)
    if len(keyq)>4:
        keyq = keyq[1:]
    print(keyq)
    key = (b'\u' + keyq.encode()).decode('unicode-escape')
    print(key)
# 存字典內(nèi)容
    for tab_page_item in tab_page:
        tab_page_item['key'] = key
        type(str(tab_page_item))
        hdjbjs.write(str(tab_page_item)+'\n')
# 數(shù)不能為疊加器


hdjbjs = codecs.open("hdjibenjieshi_file", "w",'utf-8')
# 開字典數(shù)據(jù)文件
hdjbjs.write("<xml name='漢典基本解釋'>")
# 數(shù)不能為疊加器
zurllistfile = codecs.open("otherzurllist", "w",'utf-8')
# 開字典地址文件
jsurlfile = open('zdurlfile_jibenjieshizdurl','r')
# 取網(wǎng)址
a = jsurlfile.read()
b = a.split('\n')
for zurl in b:
    if len(zurl) !=0:
        scrapt(zurl)

# 出前關(guān)閉文件
zurllistfile.close()
hdjbjs.write("</xml>")
# 數(shù)不能為疊加器
hdjbjs.close()