使用Python進行上海市租房信息爬取，通過requests + Beautifulsoup對網(wǎng)頁內(nèi)容進行抓取和數(shù)據(jù)提取。

import requests
from bs4 import BeautifulSoup
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.2669.400 QQBrowser/9.6.10990.400',
'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding':'gzip, deflate, sdch',
'Accept-Language':'zh-CN,zh;q=0.8',
'Connection':'keep-alive',
'Referer':'https://www.baidu.com/link?url=znRkKNPxTXXmN7m4lWy28vWXLWyx_hTEW4bLC5Pi8YR_s5wAdQk4FExi8Fz6HxHc&wd=&eqid=f2ad9a7e00007f1200000004594a3116'         
}
urls = [] # 定義空列表，用于創(chuàng)建所有的爬蟲鏈接

#基于for循環(huán)，構(gòu)造完整的爬蟲鏈接
for i in ['zufang']:
    url = 'http://sh.lianjia.com/%s/' %i       
    for j in list(range(1,2075)): 
        urls.append('http://sh.lianjia.com/%s/d%si1s5' %(i,j)) #拼接所有需要爬蟲的鏈接

#創(chuàng)建csv文件，用于后面的保存數(shù)據(jù)
file = open('D:/jupyter/lianjia.csv','w')

def readpage(url):
    try:
        res = requests.get(url)
    except ConnectionError as e:
        return None
    return res

def viewsubway(res2):
    try:
        view = res2.find_all('span',{'class':'fang-subway-ex'})[0].find_all('span')[0].text
    except IndexError as e:
        return None
    return view

for url in urls: #基于for循環(huán)，抓取出所有滿足條件的標簽和屬性列表，存放在find_all中
    res = readpage(url)
    res = res.text.encode(res.encoding).decode('utf-8')
    soup = BeautifulSoup(res,'html.parser')
    find_all = soup.find_all(name = 'div',attrs = {'class':'info-panel'})  
    for i in list(range(len(find_all))):  #基于for循環(huán)，抓取出所需的各個字段信息
        title = find_all[i].find('a')['title'] #每套租房的標語
        res2 = find_all[i]
        name = res2.find_all('div',{'class':'where'})[0].find_all('span')[0].text #租房所在的小區(qū)名稱
        room_type = res2.find_all('div',{'class':'where'})[0].find_all('span')[1].text #租房的戶型
        size = res2.find_all('div',{'class':'where'})[0].find_all('span')[2].text[:-3]#租房的面積

        #采用列表解析式，刪除字符串的首位空格
        info = [i.strip() for i in res2.find_all('div',{'class':'con'})[0].text.split('\n')]
        region = info[1] #租房所在的區(qū)
        area = info[2] #租房所在的街道地域
        louceng = info[4][:2] #租房所在的樓層
        chaoxiang = info[7][3:] #租房的朝向 

        #每套租房的租金
        price = res2.find_all('div',{'class':'price'})[0].find_all('span')[0].text
        
        #地鐵站及地鐵
        view = viewsubway(res2)
        if view != None:
            subway = view[2:5] #租房附近的地鐵線
            distance = view[-4:-1] #租房到地鐵站的距離
        else:
            subway = '\0'
            distance = '\0'
        
        #將上面的各字段信息值寫入并保存到csv文件中
        file.write(','.join((name.encode('gbk','ignore'),room_type.encode('gbk','ignore'),size.encode('gbk','ignore'),region.encode('gbk','ignore'),area.encode('gbk','ignore'),louceng.encode('gbk','ignore'),chaoxiang.encode('gbk','ignore'),price.encode('gbk','ignore'),subway.encode('gbk','ignore'),distance.encode('gbk','ignore'),title.encode('gbk','ignore'))) + '\n')

關閉文件(否則數(shù)據(jù)不會寫入到csv文件中)
file.close()```