抓取結(jié)果

p{'lorder_gender': '女', 'prices': '438', 'lorder_name': 'gjy1234', 'addr': '上海市徐匯區(qū)高安路69弄', 'house_image': 'http://image.xiaozhustatic1.com/00,800,533/7,0,86,9803,1800,1200,0b2f41ad.jpg', 'title': '衡山路 精致一居 近地鐵1/7/9/10號(hào)線(xiàn)'}
{'lorder_gender': '男', 'prices': '538', 'lorder_name': '王老闆', 'addr': '上海市虹口區(qū)北蘇州路400號(hào)河濱大樓', 'house_image': 'http://image.xiaozhustatic1.com/00,800,533/4,0,68,8498,1798,1200,ca0c88ef.jpg', 'title': '看外灘的超大陽(yáng)臺(tái)房 沙遜1935的河濱大樓'}
{'lorder_gender': '男', 'prices': '998', 'lorder_name': '風(fēng)子上海', 'addr': '上海市徐匯區(qū)余慶路', 'house_image': 'http://image.xiaozhustatic1.com/00,801,533/6,0,59,6286,1136,756,d7287d22.jpg', 'title': '【春上春宿】法租界名人故居/有松鼠飛鳥(niǎo)的院子'}
……

CODE

from bs4 import BeautifulSoup
import requests
import time


urls = ["http://sh.xiaozhu.com/search-duanzufang-p{}-0/".format(str(i)) for i in range(1,3)]
def get_detail_url(url):
    web_data = requests.get(url)
    time.sleep(1)
    soup = BeautifulSoup(web_data.text, 'lxml')
    detail_urls = soup.select('#page_list > ul > li >div.result_btm_con.lodgeunitname')
    for single_detail_url in detail_urls:
        data = single_detail_url.get('detailurl')
        get_page_details(data)#獲取每頁(yè)詳細(xì)內(nèi)容


def get_page_details(url, infos = None):
    web_data = requests.get(url)
    time.sleep(1)
    soup = BeautifulSoup(web_data.text, 'lxml')
    titles =soup.select('body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > h4 > em ')
    addrs = soup.select('body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > p > span.pr5 ')
    prices = soup.select('div.day_l > span ')
    house_images = soup.select('#curBigImage')
    lorder_names = soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > a')
    lorder_genders = soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > span ')

    if infos == None:
        for title, addr, price, house_image, lorder_name, lorder_gender in zip(titles, addrs, prices, house_images, lorder_names, lorder_genders):
            info = {
            'title':title.get_text(),
            'addr':addr.get_text().strip(),
            'prices':price.get_text(),
            'house_image':house_image.get('src'),
            'lorder_name':lorder_name.get('title'),
            'lorder_gender':get_gender(lorder_gender.get('class'))
            }
            print(info)


def get_gender(class_name):
    if class_name ==['member_boy_ico']:
        return '男'
    elif class_name == ['member_girl_ico']:
        return '女'
    else:
        return None

for url in urls:
    get_detail_url(url)

心得體會(huì)：

• 這節(jié)課安裝lxml庫(kù)的時(shí)候被難住了，網(wǎng)上到處搜，最后發(fā)現(xiàn)解決辦法：

Paste_Image.png

• 函數(shù)可以嵌套使用，在本節(jié)課的練習(xí)中使用了三個(gè)函數(shù)，用get_detail_url()函數(shù)嵌套get_page_details()。

• 圖片的抓取是難點(diǎn)，本例中還沒(méi)有熟練掌握。抓取結(jié)果疑似無(wú)法下載。