# 一、div標(biāo)簽文本提取
# 將學(xué)習(xí)視頻中xpath.html文件中div標(biāo)簽下文本值
from lxml import etree
file=open('xpath.html','r',encoding='utf-8')
html=file.read()
file.close()
# print(html)
selector=etree.HTML(html)
# print(selector,type(selector))
# div_1=selector.xpath('//ul[@class="title"]/text()')[0].strip()
# div_2=selector.xpath('//ul[@class="xpath_learn"]/text()')[0]
div=selector.xpath('//div/text()')
print(div)
# div1=selector.xpath('//div/text()')[0].strip()? #請(qǐng)問(wèn)老師此處是否應(yīng)該用到循環(huán)？而不是直接找位置？
# div2=selector.xpath('//div/text()')[3].strip()
# print(div1,div2)
# “第一個(gè)div” ，“第二個(gè)div” 使用xpath結(jié)構(gòu)化提取并打印輸出
div_1=div[0].strip()
div_2=div[3].strip()
print(div_1, div_2)
# 二、ul標(biāo)簽文本提取
# 將xpath.html文件中ul標(biāo)簽下“流程” ，“xpath學(xué)習(xí)”，“流程2”文本值
# 使用xpath結(jié)構(gòu)化提取并打印輸出
ul1=selector.xpath('//ul/text()')[0].strip()
ul2=selector.xpath('//ul/text()')[6].strip()
ul3=selector.xpath('//ul/text()')[8].strip()
print(ul1,ul2,ul3)
# 三、過(guò)濾標(biāo)簽
# 將xpath.html文件中的第一個(gè)div下的前3個(gè)a標(biāo)簽的文本及超鏈接
# 使用xpath結(jié)構(gòu)化提取，打印輸出
infos=selector.xpath('//div[@class="works"][1]/ul[@class="title"][1]/li[position()<4]/a')
for info in infos:
a_text=info.xpath('text()')[0]
a_href=info.xpath('@href')[0]
print(a_text, a_href)
# 四、requests模塊和lxml&xpath結(jié)合提取數(shù)據(jù)
# 結(jié)合上節(jié)課requests模塊知識(shí)，將陽(yáng)光電影網(wǎng)導(dǎo)航欄的文本及超鏈接結(jié)構(gòu)化提取
import requests
url = 'http://www.ygdy8.com/'
headers = {
'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'Accept-Encoding':'gzip, deflate',
'Accept-Language':'en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4',
'Cache-Control':'max-age=0',
'Connection':'keep-alive',
'Cookie':'37cs_pidx=1; 37cs_user=37cs78691413268; 37cs_show=69; UM_distinctid=15e36b74d82561-042fa83480e0e9-8383667-1fa400-15e36b74d836c6; CNZZDATA5783118=cnzz_eid%3D1135589022-1504151834-%26ntime%3D1504151834; cscpvrich4016_fidx=3',
'Host':'www.ygdy8.com',
'If-Modified-Since':'Wed, 30 Aug 2017 14:38:20 GMT',
'If-None-Match':'"056e49f9d21d31:530"',
'Upgrade-Insecure-Requests':'1',
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'
}
req = requests.get(url)
status_code = req.status_code
# print(status_code)
req.encoding = 'gb2312'
html = req.text
#請(qǐng)問(wèn)此處我們需要從網(wǎng)上request建立到一個(gè)html文檔，然后再讀取嗎？？
# fp=open(r'C:\Python\Learn\20170813-list\txt\#14 ygdd.html','w',encoding='utf-8')
# fp.write(html)
# fp.close
#
# file=open('ygdd.html','r',encoding='utf-8')
# html=file.read()
# file.close()
# from lxml import etree
selector=etree.HTML(html)
infos=selector.xpath('//li/a')
for info in infos:
??? title=info.xpath('text()')
??? u1=info.xpath('@href')[0]
??? if len(title)>=1:
??????? print(title, u1)
??? else:
??????? break
