知乎推薦文章頁面的爬取是比較復(fù)雜的，應(yīng)為其涉及大量ajax/js解析和知乎的驗證碼登錄，這給爬蟲爬取造成了好多麻煩，不過有一款非常強大的工具可以輕松解決這些問題——selenium，它是一款測試工具，不過他也提供了許多爬蟲用途，通過它與chrome配合使用，可以模擬瀏覽器進行登錄，同時獲取到j(luò)s渲染后的網(wǎng)站源代碼，非常方便。什么都不說了，上代碼：

#知乎推薦文章爬取  
#2017/8/6  
# -*- encoding = utf-8 -*-  
  
from selenium import webdriver  
from selenium.webdriver.common.by import By  
from selenium.webdriver.support.ui import WebDriverWait  
from selenium.webdriver.support import expected_conditions as EC  
from selenium.common.exceptions import TimeoutException  
from selenium.common.exceptions import NoSuchElementException  
from bs4 import BeautifulSoup  
import csv  
import os  
import time  
import re  
  
driver = webdriver.Chrome()  
  
#登錄知乎  
def putcookies(account,password):  
    try:  
        driver.get('https://www.zhihu.com/#signin')  
        WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, "body > div.index-main > div > div.desk-front.sign-flow.clearfix.sign-flow-simple > div.index-tab-navs > div > a.active")))  
        botton = driver.find_element_by_css_selector('body > div.index-main > div > div.desk-front.sign-flow.clearfix.sign-flow-simple > div.view.view-signin > div.qrcode-signin-container > div.qrcode-signin-step1 > div.qrcode-signin-cut-button > span')  
        botton.click()  
        form = driver.find_element_by_css_selector('body > div.index-main > div > div.desk-front.sign-flow.clearfix.sign-flow-simple > div.view.view-signin > form > div.group-inputs > div.account.input-wrapper > input[type="text"]')  
        pas = driver.find_element_by_css_selector('body > div.index-main > div > div.desk-front.sign-flow.clearfix.sign-flow-simple > div.view.view-signin > form > div.group-inputs > div.verification.input-wrapper > input[type="password"]')  
        sub = driver.find_element_by_css_selector('body > div.index-main > div > div.desk-front.sign-flow.clearfix.sign-flow-simple > div.view.view-signin > form > div.button-wrapper.command > button')  
        form.send_keys(account)  
        pas.send_keys(password)  
        sub.click()  
        try:  
            print('請手動輸入驗證碼')  
            driver.implicitly_wait(10)  
            driver.find_element_by_css_selector('#root > div > div:nth-child(2) > header > div > div.SearchBar > button')  
        except NoSuchElementException:    
            sub.click()  
    except:  
        putcookies(account,password)  
  
#滑動頁面  
def change_page(num):  
    WebDriverWait(driver,10).until(EC.presence_of_element_located((By.CSS_SELECTOR,'#root > div > div:nth-child(2) > header > div > div.SearchBar > button')))  
    for i in range(num):  
        driver.execute_script('window.scrollTo(0, document.body.scrollHeight)')    
        time.sleep(3)  
  
#解析頁面  
def findinf(html):  
    soup = BeautifulSoup(html,'lxml')  
    r = re.compile('(\d+)')  
    links = soup.find_all('div',class_='Card TopstoryItem')  
    for link in links:  
        try:  
            maininf = link.find(class_='Feed-meta-item').get_text()[-3:]#話題  
            writer = link.find(class_='AuthorInfo-head').get_text()#作者  
        except:  
            continue  
        try:  
            intd = link.find('div',class_='RichText AuthorInfo-badgeText').string#作者個人介紹  
        except:  
            intd = ''  
        title = link.find('h2',class_='ContentItem-title').get_text()#標(biāo)題  
         + link.find('h2',class_='ContentItem-title').a['href']#文章鏈接  
        try:  
            support = link.find(class_='Button VoteButton VoteButton--up').get_text()#點贊  
        except:  
            support = link.find(class_='Button LikeButton ContentItem-action').get_text()#點贊  
        try:  
            talking = r.match(link.find('button',class_='Button ContentItem-action Button--plain').get_text()[:-3]).group()#評論數(shù)  
        except:  
            talking = ''  
        content = link.find('span',class_='RichText CopyrightRichText-richText').get_text()#摘要  
        yield  {  
                'maininf': maininf,  
                'writer':writer,  
                'intd':intd,  
                'title':title,  
                'support':support,  
                'talking':talking,  
                'content':content,  
                'href':href,  
            }  
  
#創(chuàng)建一個文件夾  
def make(path):  
    if not os.path.exists(path):  
        os.makedirs(path)  
#保存數(shù)據(jù)  
def save_to_csv(inf,path):  
    with open(path + '知乎文章信息概要采集.csv','a') as f:  
        writer = csv.writer(f)  
        writer.writerow(['標(biāo)題','作者','話題','作者個人介紹','點贊數(shù)','評論數(shù)','文章鏈接','摘要'])  
        try:  
            for i in inf:  
                writer.writerow([i['title'],i['writer'],i['maininf'],i['intd'],i['support'],i['talking'],i['href'],i['content']])  
        except:  
            pass  
#主函數(shù)  
def main(account,password,num):  
    path = 'D:/數(shù)據(jù)/知乎文章/'  
    putcookies(account,password)  
    change_page(num)     
    inf = findinf(driver.page_source)  
    make(path)  
    print('---'*43)  
    print('{:^60}'.format('知乎文章概要'))  
    print("***"*43)  
    for i in findinf(driver.page_source):  
        print('標(biāo)題:{:<10s}'.format(i['title']))  
        print('作者:{:>3s}'.format(i['writer']),end = ' '*5)  
        print("話題:{:>3s}".format(i['maininf']))  
        print('作者個人介紹:')  
        print('{:<5s}'.format(i['intd']))  
        print('點贊數(shù):{:<2s}'.format(i['support']),end = ' '*5)  
        print("評論數(shù):{:3s}".format(i['talking']))  
        print("文章鏈接:" + i['href'])  
        print("摘要：")  
        print('{:<5s}'.format(i['content']))  
        print('---' *43)  
    save_to_csv(inf,path)  
  
  
  
#執(zhí)行程序  
if __name__ == '__main__':  
    num = int(input('請輸入要爬取的頁面數(shù)：'))  
    account = input("請輸入知乎賬號：")  
    password = input("請輸入知乎密碼：")  
    time_start = time.time()  
    main(account,password,num)  
    print("^^^"*43)  
    print("共耗時{}秒".format(time.time()-time_start))  
    driver.quit()  

爬取的結(jié)果得到了文章概要信息，可以做一個文章信息的參考。