畢業(yè)離校到上海漂泊已經(jīng)有些時日了，每天的生活還算正常，魔都標(biāo)配的房租，日常生活消費(fèi)，告誡來這里的每一位追求夢想的人兒不得不更加努力。加班的周末，寫了一個簡單的腳本，加了些注釋，主要想讓參考的同學(xué)了解些編寫代碼的規(guī)范。我們要像寫情書般優(yōu)雅的寫代碼。周末愉快?。?/p>

# -*- coding: utf-8 -*-
#!/usr/bin/python3
import requests,re
import json
import pymongo
from requests.exceptions import RequestException

from multiprocessing import Pool
#from pathos.multiprocessing import ProcessingPool as Pool

'''
request modle usage:
    1.set headers
    2.verify response.status_code==200
    3.GoodPlan: use RequestException
'''
def getOnePage(url):
    headers={
            'accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
            'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36'
    }
    try:
        response=requests.get(url,headers=headers)
        if response.status_code==200:
            return response.text
        return None
    except ResquestException:
        return None
'''
regular matching item and set a generator
'''
def parseOnePage(html):
    pattern = re.compile('<dd>.*?board-index.*?>(\d+)</i>.*?data-src="(.*?)".*?name"><a'
                        +'.*?>(.*?)</a>.*?star">(.*?)</p>.*?releasetime">(.*?)</p>'
                        +'.*?integer">(.*?)</i>.*?fraction">(.*?)</i>.*?<dd>',re.S)
    items= re.findall(pattern,html)
    for item in items:
        print(item)
        yield{
            'index':item[0],
            'image':item[1],
            'title':item[2],
            'actor':item[3].strip()[3:],
            'time':item[4].strip()[5:],
            'score':item[5]+item[6]
        }
'''
write content to local file
'''
def writeToFile(content):
    with open('result.txt','a',encoding='utf-8') as wf:
        wf.write(json.dumps(content,ensure_ascii=False)+'\n')
        wf.flush()

def main(offset):
    '''
    configure your Database
    '''
    myclient=pymongo.MongoClient('mongodb://192.168.88.114:27017/')
    dblist=myclient.database_names()
    mydb=myclient['home']
    mycol=mydb['maoyan']
    '''
    set uri and to get html
    '''
    url='https://maoyan.com/board/4?offset='+str(offset)
    html =getOnePage(url)
    
    '''
    insert data to database
    '''
    for item in parseOnePage(html):
        try:
            #writeToFile(item)
            mycol.insert_one(item)
        except:
            print("check you datebase configure")

if __name__ == '__main__':
    '''
    create a thread pool 
    '''
    pool =Pool()
    pool.map(main,[i*10 for i in range(10)])

下面則是記錄一些Requests庫使用中的一些需要注意的地方：
1.遇到網(wǎng)站證書校驗(yàn)時，可以視情況使用如下方法解決問題。

import requests
from requests.packages import urllib3
urllib3.disable_warnings()
response =requests.get('https://www.12306.cn',verify=False)
print(response.status_code)

2.加載本地的證書文件

import requests
response = requests.get('https://www.12306.cn',cert=('/path/server.crt','/path/key'))
print(response.status_code)

3.設(shè)置代理

import requests
proxies={'http':"http://127.0.0.1:9743",'https':"https://127.0.0.1:9743"}
response = requests.get('https://taobao.com',proxies=proxies)
print(response.status_code)

4 設(shè)置的代理有密碼

import requests
proxies={'http':"http://user:password@127.0.0.1:9743",'https':"https://user:pw@127.0.0.1:9743"}
response = requests.get('https://taobao.com',proxies=proxies)
print(response.status_code)

5.requests 設(shè)置socks5代理
pip3 install "requests[socks]"

 import requests
proxies={'http':"socks5://127.0.0.1:9742",'https':"socks5://127.0.0.1:9742"}
response = requests.get('https://taobao.com',proxies=proxies)
print(response.status_code)

6.超時設(shè)置,捕獲異常

 import requests
from requests.exception import ReadTimeout
try:
  response = requests.get('https://taobao.com',timeout=3)
  print(response.status_code)
except ReadTimeout:
  print("timeout")

7。認(rèn)證設(shè)置

 import requests  
from requests.auth import HTTPBasicAuth

response = requests.get('https://taobao.com',auth=HTTPBasicAuth('user','123'))
print(response.status_code)