python.png
全文簡介
本文用python采集的是拉鉤網(wǎng)上的'python'崗位數(shù)據(jù),然后用python進(jìn)行數(shù)據(jù)的可視化，主要涉及python爬蟲和python數(shù)據(jù)分析的內(nèi)容。
爬蟲部分
首先用瀏覽器打開拉勾網(wǎng)首頁搜索python，然后利用瀏覽器的開發(fā)者工具分析網(wǎng)絡(luò)請求，發(fā)現(xiàn)拉勾網(wǎng)的數(shù)據(jù)交互是動態(tài)網(wǎng)頁，通過對比網(wǎng)頁上的數(shù)據(jù)發(fā)現(xiàn)數(shù)據(jù)提交的真實(shí)網(wǎng)址，再仔細(xì)觀察發(fā)現(xiàn)拉勾網(wǎng)的飯爬措施。提交數(shù)據(jù)是post方式如下圖
圖片.png
仔細(xì)再看一下發(fā)現(xiàn)一個get請求，分析get里面的響應(yīng)內(nèi)容為公司的id，通過對比發(fā)現(xiàn)，和post請求之間有關(guān)聯(lián)，因?yàn)閜ost請求返回的內(nèi)容里面有公司的id，而且剛好是15個。
圖片.png
圖片.png
爬蟲部分代碼實(shí)現(xiàn)
'''
datatime: 2018 03 15
by:北冥神君
內(nèi)容：爬取拉勾網(wǎng)上面的數(shù)據(jù)
'''
#導(dǎo)入模塊
import requests #網(wǎng)絡(luò)請求模塊
import re #正則模塊
import time #時間模塊
import random #隨機(jī)數(shù)模塊
# post的網(wǎng)址
url = 'https://www.lagou.com/jobs/positionAjax.json?needAddtionalResult=false&isSchoolJob=0'
# 反爬措施
header1 = {'Host': 'www.lagou.com',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:58.0) Gecko/20100101 Firefox/58.0',
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
'Accept-Encoding': 'gzip, deflate, br',
'Referer': 'https://www.lagou.com/jobs/list_python?labelWords=&fromSearch=true&suginput=',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'X-Requested-With': 'XMLHttpRequest',
'X-Anit-Forge-Token': 'None',
'X-Anit-Forge-Code': '0',
'Content-Length': '26',
'Cookie': 'Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1519816933,1519816935,1521079570,1521079575; _ga=GA1.2.129319102.1515420746; user_trace_token=20180108221226-f4036578-f47d-11e7-a021-5254005c3644; LGUID=20180108221226-f40369cf-f47d-11e7-a021-5254005c3644; index_location_city=%E5%85%A8%E5%9B%BD; Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1521081597; LGSID=20180315100701-8cabf3af-27f5-11e8-b1fc-525400f775ce; LGRID=20180315103956-2609450b-27fa-11e8-b1ed-5254005c3644; _gid=GA1.2.2023749020.1521079570; JSESSIONID=ABAAABAAAIAACBI02527B187B701F2E661E90B666E236AF; hideSliderBanner20180305WithTopBannerC=1; TG-TRACK-CODE=search_code; SEARCH_ID=c9472cb5ce184e00bf8dcd8989fdc892; _gat=1; X_HTTP_TOKEN=d5fd7e2b382eab92942c6aee48b65dfa',
'Connection': 'keep-alive',
'Pragma': 'no-cache',
'Cache-Control': 'no-cache'}
header2 = {'Host': 'www.lagou.com',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:58.0) Gecko/20100101 Firefox/58.0',
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
'Accept-Encoding': 'gzip, deflate, br',
'Referer': 'https://www.lagou.com/jobs/list_python?labelWords=&fromSearch=true&suginput=',
'X-Requested-With': 'XMLHttpRequest',
'X-Anit-Forge-Token': 'None',
'X-Anit-Forge-Code': '0',
'Cookie': 'Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1519816933,1519816935,1521079570,1521079575; _ga=GA1.2.129319102.1515420746; user_trace_token=20180108221226-f4036578-f47d-11e7-a021-5254005c3644; LGUID=20180108221226-f40369cf-f47d-11e7-a021-5254005c3644; index_location_city=%E5%85%A8%E5%9B%BD; Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1521081597; LGSID=20180315100701-8cabf3af-27f5-11e8-b1fc-525400f775ce; LGRID=20180315103956-2609450b-27fa-11e8-b1ed-5254005c3644; _gid=GA1.2.2023749020.1521079570; JSESSIONID=ABAAABAAAIAACBI02527B187B701F2E661E90B666E236AF; hideSliderBanner20180305WithTopBannerC=1; TG-TRACK-CODE=search_code; SEARCH_ID=c9472cb5ce184e00bf8dcd8989fdc892; _gat=1; X_HTTP_TOKEN=d5fd7e2b382eab92942c6aee48b65dfa',
'Connection': 'keep-alive',
'Pragma': 'no-cache',
'Cache-Control': 'no-cache'}
for n in range(1,31):
# 要提交的數(shù)據(jù)
form = {'first': 'false',
'kd': 'Python',
'pn': str(n)}
time.sleep(random.randint(5, 10))#隨機(jī)暫停5-10秒
# 提交數(shù)據(jù)
html = requests.post(url, data=form, headers=header1)
# 提取數(shù)據(jù)
data = re.findall(
'{"companyId":.*?,"positionName":"(.*?)","workYear":"(.*?)","education":"(.*?)","jobNature":"(.*?)","financeStage":"(.*?)","companyLogo":".*?","industryField":".*?","city":"(.*?)","salary":"(.*?)","positionId":.*?,"positionAdvantage":"(.*?)","companyShortName":"(.*?)","district"',
html.text)
print(data)
#提取公司ID
companyId = re.findall(
'{"companyId":(.*?),.*?,"district"',
html.text)
print(companyId)
companyIds = ','.join(companyId)
print(companyIds)
urlcompanyUrl = 'https://www.lagou.com/c/approve.json?companyIds='+companyIds
print(urlcompanyUrl)
#反爬
get_company = requests.get(url=urlcompanyUrl,headers = header2)
print(get_company.text)
# 轉(zhuǎn)換成數(shù)據(jù)框
data = pd.DataFrame(data)
print(data)
# 保存在本地
data.to_csv(r'LaGouData.csv', header=False, index=False, mode='a+')
數(shù)據(jù)分析
- 學(xué)歷要求
data['學(xué)歷要求'].value_counts().plot(kind='barh',rot=0)
plt.show()
5F4C4359-F6EF-48D6-9D23-146C32E9C5EF.png
- 工作經(jīng)驗(yàn)
data['工作經(jīng)驗(yàn)'].value_counts().plot(kind='bar',rot=0,color='b')
plt.show()
D7666874-4689-491D-BBFE-517EBECBF1C8.png
1080ADD8-883B-4000-820D-D0410B8ADD48.png
- 工作地點(diǎn)
data['工作地點(diǎn)'].value_counts().plot(kind='pie',autopct='%1.2f%%',explode = np.linspace(0,0.4,19))
plt.show()
1080ADD8-883B-4000-820D-D0410B8ADD48.png
- 工資情況
data['工資'].value_counts().plot(kind='pie',autopct='%1.2f%%')
plt.show()
圖片.png
- 詞云分析
final = ''
stopwords = ['PYTHON', 'python', 'Python', '工程師', '（', '）', '/'] # 停止詞
for n in range(data.shape[0]):
seg_list = list(jieba.cut(data['崗位職稱'][n]))
for seg in seg_list:
if seg not in stopwords:
final = final + seg + ' '
python.png
數(shù)據(jù)分析代碼
import pandas as pd # 數(shù)據(jù)框操作
import numpy as np
import matplotlib.pyplot as plt # 繪圖
import jieba # 分詞
import matplotlib as mpl # 配置字體
mpl.rcParams["font.sans-serif"] = ["cmb10"]
mpl.rcParams['axes.unicode_minus'] = False
# 配置繪圖風(fēng)格
plt.rcParams["axes.labelsize"] = 16.
plt.rcParams["xtick.labelsize"] = 14.
plt.rcParams["ytick.labelsize"] = 14.
plt.rcParams["legend.fontsize"] = 12.
plt.rcParams["figure.figsize"] = [15., 15.]
# 導(dǎo)入數(shù)據(jù)
data = pd.read_csv('/Users/tencenting/PycharmProjects/qm/venv/LaGouData.csv',encoding='utf-8') # 導(dǎo)入數(shù)據(jù)
print(data.head())
print(data.tail())
data['學(xué)歷要求'].value_counts().plot(kind='barh',rot=0)
plt.show()
data['工作經(jīng)驗(yàn)'].value_counts().plot(kind='bar',rot=0,color='b')
plt.show()
data['工資'].value_counts().plot(kind='pie',autopct='%1.2f%%')
plt.show()
#data['工作地點(diǎn)'].value_counts().plot(kind='pie',autopct='%1.2f%%',shadow =False)
data['工作地點(diǎn)'].value_counts().plot(kind='pie',autopct='%1.2f%%',explode = np.linspace(0,0.4,19))
x = np.linspace(0,1.5,25)
print(x)
print(len(x))
plt.show()
final = ''
stopwords = ['PYTHON', 'python', 'Python', '工程師', '（', '）', '/'] # 停止詞
for n in range(data.shape[0]):
seg_list = list(jieba.cut(data['崗位職稱'][n]))
for seg in seg_list:
if seg not in stopwords:
final = final + seg + ' '
#final 得到的詞匯
print(final)
數(shù)據(jù)分析總結(jié)
python程序員工作地點(diǎn)大部分集中在北京、深圳、上海、成都、廣州、杭州、武漢，其中北京最多，招聘要求大部分是3-5年和1-3年的工作經(jīng)驗(yàn)，對學(xué)歷的要求為本科，工資大在8k-30k之間，從詞語分析上看從事開發(fā)方向比較多。
