最近想做一個電影資訊的App，由于豆瓣不對外開發(fā)API了，正愁怎么錄入數(shù)據(jù)呢。最近在學(xué)Python 想想那就用它來抓取電影信息吧。

Python2.7
Mac OS

抓取的是電影天堂里面最新電影的頁面。鏈接地址：http://www.dytt8.net/html/gndy/dyzz/index.html

獲取頁面的中電影詳情頁鏈接

import urllib2
import os
import re
import string

# 電影URL集合
movieUrls = []

# 獲取電影列表
def queryMovieList():

 url = 'http://www.dytt8.net/html/gndy/dyzz/index.html'   
 conent = urllib2.urlopen(url)
 conent =  conent.read()
 conent = conent.decode('gb2312','ignore').encode('utf-8','ignore')  
 pattern = re.compile ('<div class="title_all"><h1><font color=#008800>.*?</a>></font></h1></div>'+
                       '(.*?)<td height="25" align="center" bgcolor="#F4FAE2"> ',re.S)
 items = re.findall(pattern,conent) 
 
 str = ''.join(items)
 pattern = re.compile ('<a href="(.*?)" class="ulink">(.*?)</a>.*?<td colspan.*?>(.*?)</td>',re.S)
 news = re.findall(pattern, str)

 for  j in news:
    
         movieUrls.append('http://www.dytt8.net'+j[0])

抓取詳情頁中的電影數(shù)據(jù)

def queryMovieInfo(movieUrls):

 for index, item in enumerate(movieUrls):

  print('電影URL: ' + item)

  conent = urllib2.urlopen(item)
  conent = conent.read()
  conent = conent.decode('gb2312','ignore').encode('utf-8','ignore')  

  movieName = re.findall(r'<div class="title_all"><h1><font color=#07519a>(.*?)</font></h1></div>', conent, re.S)
  if (len(movieName) > 0):
   movieName = movieName[0] + ""
   # 截取名稱
   movieName = movieName[movieName.find("《") + 3:movieName.find("》")]
  else:
   movieName = ""

  print("電影名稱: " + movieName.strip())

  movieContent = re.findall(r'<div class="co_content8">(.*?)</tbody>',conent , re.S)

  pattern = re.compile('<ul>(.*?)<tr>', re.S)
  movieDate = re.findall(pattern,movieContent[0])

  if (len(movieDate) > 0):
   movieDate = movieDate[0].strip() + ''
  else:
   movieDate = ""

  print("電影發(fā)布時間: " + movieDate[-10:])

  pattern = re.compile('<br /><br />(.*?)<br /><br /><img')
  movieInfo = re.findall(pattern, movieContent[0])

  if (len(movieInfo) > 0):
   movieInfo = movieInfo[0]+''

   # 刪除<br />標(biāo)簽
   movieInfo = movieInfo.replace("<br />","")

   # 根據(jù) ◎ 符號拆分

   movieInfo = movieInfo.split('◎')

  else:
   movieInfo = ""

  print("電影基礎(chǔ)信息: ")

  for item in movieInfo:
   print(item)

  # 電影海報
  pattern = re.compile('<img.*? src="(.*?)".*? />', re.S)  
  movieImg = re.findall(pattern,movieContent[0])

  if (len(movieImg) > 0):
   movieImg = movieImg[0]
  else:
   movieImg = ""
 
  print("電影海報: " + movieImg)

  pattern = re.compile('<td style="WORD-WRAP: break-word" bgcolor="#fdfddf"><a href="(.*?)">.*?</a></td>', re.S)
  movieDownUrl = re.findall(pattern,movieContent[0])

  if (len(movieDownUrl) > 0):
   movieDownUrl = movieDownUrl[0]
  else:
   movieDownUrl = ""

  print("電影下載地址：" + movieDownUrl + "")

  print("------------------------------------------------\n\n\n")
 

執(zhí)行抓取

if __name__=='__main__':

     print("開始抓取電影數(shù)據(jù)");
  
     queryMovieList()

     print(len(movieUrls))

     queryMovieInfo(movieUrls)

     print("結(jié)束抓取電影數(shù)據(jù)")

總結(jié)

學(xué)好正則表達(dá)式很重要,很重要,很重要!!!! Python的語法好有感覺, 對比Java ...