用到的庫：requests
爬蟲思路
(xxxxxxxxx代表教務(wù)網(wǎng)網(wǎng)址，不便傳播)
- 創(chuàng)建一個Requests會話對象(所有的爬蟲步驟都在該會話下進行)。
s = requests.Session()
- get教務(wù)網(wǎng)的登陸頁面，返回的頭部信息中的cookies信息。
r = s.get(r'http://kdjw.hnust.cn/kdjw/', headers=kdjw_hed)
cookies = r.headers['Set-Cookie'][0:-12]
- 根據(jù)獲取到cookies信息實時獲取當前登錄頁面的驗證碼圖片信息，并把它的二進制文件信息存儲為code.jpg。
r_verifycode = s.get(r'http://xxxxxxxxx/verifycode.servlet', headers=verifycode_hed)
with open('code.jpg','wb') as f:
f.write(r_verifycode.content)
- 請求用戶輸入驗證碼，然后把所收到的驗證碼字符串以及用戶輸入的學號，密碼POST到真正的登錄的頁面，進而實現(xiàn)模擬登陸。
r_login = s.post(r'http://xxxxxxxxx/Logon.do?method=logon', headers=post_hed, data=login_data)
- 模擬登陸后，直接攜帶cookies POST發(fā)送請求給教務(wù)網(wǎng)的成績信息存儲頁面,進而獲得全部成績信息的原始數(shù)據(jù)。
r_grades = s.post(r'http://xxxxxxxxx/xszqcjglAction.do?method=queryxscj', headers=grades_hed, data=form_data)
return r_grades.text
獲取到成績信息的原始數(shù)據(jù)之后，我們就可以進行數(shù)據(jù)的清洗和整理了~
下面給出該思路的源代碼
（大神勿噴，代碼并不能直接運行）
import sys
import io
import requests
from PIL import Image
#改變標準輸出的默認編碼
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8')
def get_cookies():
kdjw_hed = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Cache-Control': 'max-age=0',
'Connection': 'keep-alive',
'Host': 'xxxxxxxxx',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
}
r = s.get(r'http://xxxxxxxxx/',headers=kdjw_hed)
#print(r.headers)
#獲取到cookies
cookies = r.headers['Set-Cookie'][0:-12]
return cookies
def get_verifycode(cookies):
'根據(jù)所得cookies獲取驗證碼'
verifycode_hed = {'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Connection': 'keep-alive',
'Cookie': cookies,
'Host': 'xxxxxxxxx',
'Referer': 'http://xxxxxxxxx/',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
}
#驗證碼獲取
r_verifycode = s.get(r'http://xxxxxxxxx/verifycode.servlet', headers=verifycode_hed)
with open('code.jpg','wb') as f:
f.write(r_verifycode.content)
return True
def show_verifycode():
'通過Image模塊展示驗證碼圖片，并提示用戶輸入驗證碼'
im = Image.open('code.jpg')
im.show()
verifycode = input('請輸入驗證碼: ')
return verifycode
def virtual_login(verifycode, cookies, usr, pwd):
#模擬登陸
login_data = {'useDogCode': '',
'USERNAME': usr,
'PASSWORD': pwd,
'RANDOMCODE': verifycode}
post_hed = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Cache-Control': 'max-age=0',
'Connection': 'keep-alive',
'Content-Length': '86',
'Content-Type': 'application/x-www-form-urlencoded',
'Cookie': cookies,
'Host': 'xxxxxxxxx',
'Origin': 'http://xxxxxxxxx',
'Referer': 'http://xxxxxxxxx/',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
}
r_login = s.post(r'http://xxxxxxxxx/Logon.do?method=logon', headers=post_hed, data=login_data)
r_login.raise_for_status()
return usr
def search_and_get_grades_html(cookies, kksj=''):
'模擬登陸之后，打開成績查詢頁面獲取成績數(shù)據(jù)'
form_data = {'kksj': kksj,
'kcxz': '',
'kcmc': '',
'xsfs': 'qbcj', #全部成績
'ok': '',
}
grades_hed = {'Accept':
'Cookie': cookies,
'Referer': 'http://xxxxxxxxx/jiaowu/cjgl/xszq/query_xscj.jsp',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
}
#成績查詢
r_grades = s.post(r'http://xxxxxxxxx/xszqcjglAction.do?method=queryxscj', headers=grades_hed, data=form_data)
hed_dict = dict(r_grades.headers)
try:
if hed_dict['Content-Length']:
print('學號、密碼或驗證碼輸入錯誤，請重新輸入...')
except KeyError:
print('成功獲取成績數(shù)據(jù)！?。?)
return r_grades.text
def get_code_in_file():
global s, cookies_
s = requests.Session()
#cookies
cookies_ = get_cookies()
#更新驗證碼code.jpg
get_verifycode(cookies_)
return cookies_
def main(verifycode_, cookies_, usr_='', pwd_='', kksj_=''):
#登陸獲取數(shù)據(jù)
if usr_ == '' or pwd_ == '':
username = virtual_login(verifycode_, cookies_)
else:
username = virtual_login(verifycode_, cookies_, usr_, pwd_)
if kksj_ == '':
orignal_html = search_and_get_grades_html(cookies_)
else:
orignal_html = search_and_get_grades_html(cookies_, kksj_)
return orignal_html, username
if __name__ == '__main__':
cookies = get_code_in_file()
show_verifycode()
main(verifycode_=input('驗證碼：'), cookies_=cookies, usr_=input('學號：'), pwd_=input('密碼：'), kksj_='')
公眾號：勝言
個人博客：wangsheng.tech
