urllib提供了一系列用于操作URL的功能。
urllib的request模塊可以非常方便地抓取URL內(nèi)容，也就是發(fā)送一個GET請求到指定的頁面，然后返回HTTP的響應(yīng)

01 簡單使用

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import urllib.request

request_url = 'http://www.baidu.com'           # 需要請求的URL地址
response = urllib.request.urlopen(request_url) # 發(fā)起請求
print(response.read().decode('utf-8'))         # 打印響應(yīng)的文本，并進(jìn)行UTF-8解碼

• read(), readline(), readlines(), fileno(), close()：對HTTPResponse類型數(shù)據(jù)進(jìn)行操作
• info()：返回HTTPMessage對象，表示遠(yuǎn)程服務(wù)器返回的頭信息
• getcode()：返回Http狀態(tài)碼。如果是http請求，200請求成功完成、404網(wǎng)址未找到等等
• geturl()：返回請求的url

02 GET 方法

 #!/usr/bin/env python
# -*- coding: utf-8 -*-

import urllib.request
import urllib.parse

get_data = {'username': 'aaa', 'password': 'bbb'}          # 此處將GET的數(shù)據(jù)定義為一個字典
get_data_encode = urllib.parse.urlencode(get_data)         # 將GET的數(shù)據(jù)進(jìn)行編碼

request_url = 'http://www.baidu.com'              # 需要請求的URL地址
request_url += '?' + get_data_encode                       # 追加GET參數(shù)到URL后面

# https://www.zhihu.com/#signin?username=aaa&password=bbb
print(request_url)

# 發(fā)起請求
response = urllib.request.urlopen(request_url)
print(response.read().decode('utf-8'))         # 打印響應(yīng)的文本，并進(jìn)行UTF-8解碼

03 GET并獲取header信息

from urllib import request

with request.urlopen('http://www.baidu.com') as f:
    data = f.read()
    print('Status:', f.status, f.reason)
    for k, v in f.getheaders():
        print('%s: %s' % (k, v))
    print('Data:', data.decode('utf-8'))

04 POST 方法

#!/usr/bin/env python
# -*- coding: utf-8 -*-
 

import urllib.request
import urllib.parse

post_data = {'first': 'true', 'pn': 1, 'kd': 'Python'}      # 此處將POST的數(shù)據(jù)定義為一個字典
post_data_encode = urllib.parse.urlencode(post_data)        # 將POST的數(shù)據(jù)進(jìn)行編碼

# UTF-8編碼
# 否則會報錯：POST data should be bytes or an iterable of bytes. It cannot be of type str.
post_data_encode = post_data_encode.encode(encoding='utf-8')
request_url = 'http://www.lagou.com/jobs/positionAjax.json?'               # 需要請求的URL地址

# 發(fā)起請求
# 此處增加了第二個參數(shù)為傳送的POST數(shù)據(jù)（默認(rèn)為None）
# 第三個參數(shù)為請求超時時間，默認(rèn)為socket._GLOBAL_DEFAULT_TIMEOUT
response = urllib.request.urlopen(request_url, post_data_encode, 3)
print(response.read().decode('utf-8'))         # 打印響應(yīng)的文本，并進(jìn)行UTF-8解碼

from urllib import request, parse

print('Login to weibo.cn...')
email = input('Email: ')
passwd = input('Password: ')
login_data = parse.urlencode([
    ('username', email),
    ('password', passwd),
    ('entry', 'mweibo'),
    ('client_id', ''),
    ('savestate', '1'),
    ('ec', ''),
    ('pagerefer', 'https://passport.weibo.cn/signin/welcome?entry=mweibo&r=http%3A%2F%2Fm.weibo.cn%2F')
])

req = request.Request('https://passport.weibo.cn/sso/login')
req.add_header('Origin', 'https://passport.weibo.cn')
req.add_header('User-Agent', 'Mozilla/6.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/8.0 Mobile/10A5376e Safari/8536.25')
req.add_header('Referer', 'https://passport.weibo.cn/signin/login?entry=mweibo&res=wel&wm=3349&r=http%3A%2F%2Fm.weibo.cn%2F')

with request.urlopen(req, data=login_data.encode('utf-8')) as f:
    print('Status:', f.status, f.reason)
    for k, v in f.getheaders():
        print('%s: %s' % (k, v))
    print('Data:', f.read().decode('utf-8'))

04 使用Request 設(shè)置Headers屬性

#!/usr/bin/env python
# -*- coding: utf-8 -*-
 
import urllib.request
import urllib.parse

user_agent = 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87'
referer = 'http://www.lagou.com/jobs/positionAjax.json?'
post_data = {'first': 'true', 'pn': 1, 'kd': 'Python'}                              # 此處將POST的數(shù)據(jù)定義為一個字典
headers = {'User-Agent': user_agent, 'Referer': referer}                            # Headers屬性初始化
post_data_encode = urllib.parse.urlencode(post_data)                                # 將POST的數(shù)據(jù)進(jìn)行編碼

# UTF-8編碼
# 否則會報錯：POST data should be bytes or an iterable of bytes. It cannot be of type str.
post_data_encode = post_data_encode.encode(encoding='utf-8')
request_url = 'http://www.lagou.com/zhaopin/Python/?labelWords=label'               # 需要請求的URL地址

# 使用Request來設(shè)置Headers
request = urllib.request.Request(request_url, post_data_encode, headers)

response = urllib.request.urlopen(request)
print(response.read().decode('utf-8'))         # 打印響應(yīng)的文本，并進(jìn)行UTF-8解碼

06 Proxy（代理）的設(shè)置

#!/usr/bin/env python
# -*- coding: utf-8 -*-

from urllib import request

request_url = 'http://www.xmgc360.com/project/test.php'
proxy = request.ProxyHandler({'http': '119.28.54.102:3389'})   # 設(shè)置代理服務(wù)器
opener = request.build_opener(proxy)                            # 掛載opener
request.install_opener(opener)                                  # 安裝opener
response = request.urlopen(request_url)
print(response.read().decode('utf-8'))         # 打印響應(yīng)的文本，并進(jìn)行UTF-8解碼

07 異常處理

#!/usr/bin/env python
# -*- coding: utf-8 -*-

from urllib import request

request_url = 'http://www.lagou.com/jobs/positionAjax.json?'
proxy = request.ProxyHandler({'http': '127.0.0.1:8989'}) # 設(shè)置代理服務(wù)器
opener = request.build_opener(proxy)                         # 掛載opener
request.install_opener(opener)                               # 安裝opener
try:
    response = request.urlopen(request_url)
except Exception as e:
    print(e)                   # 打印錯誤碼

08 練習(xí)

http://image.baidu.com/channel/listjson?pn=1&rn=30&tag1=%E6%98%8E%E6%98%9F&tag2=%E5%85%A8%E9%83%A8&ie=utf8

抓取信息并保存到數(shù)據(jù)庫