爬蟲第一彈之情人節(jié)前夕

最近學了點python，想寫個爬蟲玩玩，剛好遇到情人節(jié)

時間：2018.5.19
地點：208教室
工具：Chrome、阿里云服務器

先來列舉下要爬取的內容

（1）爬取天氣
（2）爬取文章
（3）爬取句子

爬取天氣

# -*-coding:utf-8 -*-
import requests
from pyquery import PyQuery as pq

def get_response(url):
    headers = {
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecho) Chrome/67.0.3396.10 Safari/537.36'
    }
    res = requests.get(url,headers=headers)
    res.encoding = 'utf-8'
    return res

def get_weather(doc):
    wea = doc('li.sky.skyid.lv3.on').text()
    one = doc('.hide.show > .clearfix > li.li1').text()
    two = doc('.hide.show > .clearfix > li.li2').text()
    three = doc('.hide.show > .clearfix > li.li3').text()
    four = doc('.hide.show > .clearfix > li.li6').text()
    print(wea,'\n')
    print(one,'\n')
    print(two,'\n')
    print(three,'\n')
    print(four,'\n')

if __name__ == '__main__':
    url = 'http://www.weather.com.cn/weather/101120206.shtml'
    res = get_response(url)
    doc = pq(res.text)
    get_weather(doc)

運行結果如下：

19日（今天）
陣雨
14℃
4-5級 

弱 紫外線指數(shù)
輻射較弱，涂擦SPF12-15、PA+護膚品。 

減肥指數(shù)
風雨相伴，堅持室內運動吧。 

較冷 穿衣指數(shù)
建議著厚外套加毛衣等服裝。 

良 空氣污染擴散指數(shù)
氣象條件有利于空氣污染物擴散。

獲取文章鏈接

# -*-coding:utf-8 -*-
import requests
from pyquery import PyQuery as pq   

def get_response(url):
    headers = {
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecho) Chrome/67.0.3396.10 Safari/537.36'
    }
    res = requests.get(url,headers=headers)
    res.encoding = 'utf-8'
    return res

def get_article(doc):
    items = doc('.list-base-article a').items()             #---①
    for item in items:
        href = item.attr('href')                        #---②
        string = 'https://www.duanwenxue.com' + href    #---③
        with open('article.txt','a') as f:              #---④
            f.write(string + '\n')
        print(string)

if __name__ == '__main__':
    page_num = 1
    url = 'https://www.duanwenxue.com/jingdian/lizhi/list_{}.html'
    while page_num < 8:
        res = get_response(url.format(page_num))   #---⑤
        doc = pq(res.text)
        get_article(doc)
        page_num = page_num + 1
知識點總結：
①：pyquery的選擇結果可能是多個節(jié)點，類型都是PyQuery類型，它不會返回一個列表，調用items()方法會得到一個生成器，遍歷一下，就可以得到每個節(jié)點了。
②：用attr()方法獲取屬性值
③：string為拼接后的完整URL，可進一步獲取文章內容
③：以追加的方式（'a'）寫入文件,不然會覆蓋掉原來的內容
④：手動翻頁發(fā)現(xiàn)URL中最后的數(shù)字代表頁數(shù)。使用format()格式化字符串，用來替換掉字符串中的占位符‘{}’，以達到翻頁的效果。

爬取句子

import requests
from pyquery import PyQuery as pq

def get_response(url):
    headers = {
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecho) Chrome/67.0.3396.10 Safari/537.36'
    }
    res = requests.get(url,headers=headers)
    return res

def get_article(doc):
    items = doc('.list-short-article p a').items()
    for item in items:
        content = item.text()
        with open('letters.txt','a') as f:
            f.write(content + '\n')
        print(content)
if __name__ == '__main__':
    page_num = 1
    url = 'https://www.duanwenxue.com/duanwen/lizhi/list_{}.html'
    while page_num < 8:
        res = get_response(url.format(page_num))
        doc = pq(res.text)
        get_article(doc)
        page_num = page_num + 1

隨機選取一篇文章

# -*- coding:utf-8-*-
import linecache
import random

a = random.randrange(1,99)
print(a)
theline = linecache.getline(r'letters_text.txt',a)
print(theline)

python發(fā)送QQ郵件

# -*- coding:utf-8 -*-
from email.mime.text import MIMEText
from email.header import Header
from smtplib import SMTP_SSL
import linecache
import random

def send_masseger():
    host_server = 'smtp.qq.com'
    sender_qq = '1394604132'
    pwd = '***********'
    sender_qq_mail = '**********@qq.com'
    receiver = '**********@qq.com'
    mail_content = "Hello World!"
    mail_title = 'The Mail from ***'
    smtp = SMTP_SSL(host_server)
    smtp.set_debuglevel(1)
    smtp.ehlo(host_server)
    smtp.login(sender_qq,pwd)
    msg = MIMEText(mail_content,'plain','utf-8')
    msg['Subject'] = Header(mail_title,'utf-8')
    msg['From'] = sender_qq_mail
    msg['To'] = receiver
    smtp.sendmail(sender_qq_mail,receiver,msg.as_string())
    smtp.quit()
    
if __name__ == '__main__':
    send_masseger()
python發(fā)送郵件代碼參照：https://zhuanlan.zhihu.com/p/25565454
終于搞完了，先發(fā)給自己試一下~
mail.jpg

其實現(xiàn)在已經(jīng)是5.20了，那個天氣網(wǎng)上的天氣還沒更新，所以顯示的還是昨天的天氣 ~ 文章鏈接和句子是事先爬好存儲起來，隨機抽取的

代碼如下：
# -*- coding:utf-8 -*-
from email.mime.text import MIMEText
from email.header import Header
from smtplib import SMTP_SSL
import linecache
import random
import datetime
import requests
from pyquery import PyQuery as pq
import sys
reload(sys)
sys.setdefaultencoding('utf-8')

def get_time():
    start = datetime.datetime(2017,5,20)
    now = datetime.datetime.now()
    time = now - start
    return str(time)

def send_masseger(content):
    host_server = 'smtp.qq.com'
    sender_qq = '1394604132'
    pwd = 'jzqwgplfatpggcic'
    sender_qq_mail = 'zhanghe0309@qq.com'
    receiver = '1270090178@qq.com'
    mail_content = content
    mail_title = 'The Mail from lover'
    smtp = SMTP_SSL(host_server)
    smtp.set_debuglevel(1)
    smtp.ehlo(host_server)
    smtp.login(sender_qq,pwd)
    msg = MIMEText(mail_content,'plain','utf-8')
    msg['Subject'] = Header(mail_title,'utf-8')
    msg['From'] = sender_qq_mail
    msg['To'] = receiver
    smtp.sendmail(sender_qq_mail,receiver,msg.as_string())
    smtp.quit()
    
if __name__ == '__main__':
    res = requests.get('http://www.weather.com.cn/weather/101120206.shtml')   
    res.encoding = 'utf-8'
    doc = pq(res.text)
    wea = doc('li.sky.skyid.lv3.on').text()
    one = doc('.hide.show > .clearfix > li.li1').text()
    two = doc('.hide.show > .clearfix > li.li2').text()
    three = doc('.hide.show > .clearfix >li.li3').text()
    four = doc('.hide.show > .clearfix >li.li6').text()

    a = random.randrange(1,350)
    b = random.randrange(1,100)
    theline1 = linecache.getline(r'article.txt',a)
    theline2 = linecache.getline(r'letters_text.txt',b)
    article = wea + '\n\n' + one + '\n\n'+ two + '\n\n' + three + '\n\n' + four + '\n\n' + '來自小哥哥的文章推薦:' + '\n' + theline1 + '\n' + theline2 + '\n' + '愛你的日子：' +  get_time()
    send_masseger(article)
PS:

地點：7#409
時間：2018.5.20 0:53
好困，睡了睡了
YZ，等我的郵件呦~

?著作權歸作者所有,轉載或內容合作請聯(lián)系作者
【社區(qū)內容提示】社區(qū)部分內容疑似由AI輔助生成，瀏覽時請結合常識與多方信息審慎甄別。
平臺聲明：文章內容（如有圖片或視頻亦包括在內）由作者上傳并發(fā)布，文章內容僅代表作者本人觀點，簡書系信息發(fā)布平臺，僅提供信息存儲服務。

相關閱讀更多精彩內容

友情鏈接更多精彩內容