從巨潮網(wǎng)按公告時(shí)間批量下載上市公司年報(bào)

import socket
import time
import csv
import requests
import datetime
from dateutil.relativedelta import relativedelta

socket.setdefaulttimeout(20)#這里對整個(gè)socket層設(shè)置超時(shí)時(shí)間。后續(xù)文件中如果再使用到socket，不必再設(shè)置  

#下載pdf公告的函數(shù)
def downloadpdf(url, file):
    resp = requests.get(url)
    f = open(file, 'wb')
    f.write(resp.content)
    f.close()
    
#新建csv文件，存儲(chǔ)公告詳細(xì)信息
csvf = open('年報(bào).csv','a+', encoding='gbk', newline='')
writer = csv.writer(csvf)
writer.writerow(('公司簡稱','股票代碼', '發(fā)布時(shí)間', '公告標(biāo)題', '公告pdf下載地址'))

#新建txt文件，存儲(chǔ)有問題的網(wǎng)址
csvf2 = open('年報(bào)pdf\\未下載的網(wǎng)址.csv','a+', encoding='gbk', newline='')
writer2 = csv.writer(csvf2)
writer2.writerow(('時(shí)間','網(wǎng)址'))

//以下從網(wǎng)站上看參數(shù),
headers = {'User-Agent':' '}
cookies = {'Cookie': ' '}
url = ' '

//巨潮網(wǎng)貌似有一個(gè)bug，100頁之后無法翻頁，所以把爬取時(shí)間單位設(shè)置為月，以免漏掉公告。
bg = datetime.datetime.strptime('20190201', '%Y%m%d')
while bg>datetime.datetime.strptime('20181201', '%Y%m%d'):
    end = bg+relativedelta(months=1)
    bgstr = bg.strftime('%Y-%m-%d')
    endstr = end.strftime('%Y-%m-%d')
    daterange = bgstr+'~'+endstr

    for page in range(101): 
        try:
            data = {''}//網(wǎng)站上查參數(shù)          
            #發(fā)起請求，采集
            sleep_download_time =10
            time.sleep(sleep_download_time)
            resp = requests.post(url, params=data, headers=headers, cookies=cookies)
            pdfs = resp.json()['announcements']
            resp.close()
            for pdf in pdfs:
                announcementTitle = pdf['announcementTitle']
                if announcementTitle.find(u"摘要") ==-1:
                    secName = pdf['secName']
                    secCode =pdf['secCode']
                    adjunctUrl = 'http://static.cninfo.com.cn/'+pdf['adjunctUrl']
                    pdffile = '\\年報(bào)pdf\\'+'sec'+str(pdf['secCode'])+'_'+announcementTitle+'.pdf'
                    downloadpdf(url=adjunctUrl, file=pdffile)
                    print('sec'+str(pdf['secCode'])+'_'+announcementTitle+'.pdf已下載完成')
                    #announcementTypeName = pdf['announcementTypeName']
                    announcementTime = pdf['announcementTime']
                    #print(secName, secCode, announcementTime, announcementTitle, adjunctUrl, announcementTypeName)
                    writer.writerow((secName, secCode, announcementTime, announcementTitle, adjunctUrl))
        except:
            print('出問題的網(wǎng)址', resp.url)
            writer2.writerow((data['seDate'],resp.url))
    bg=bg-relativedelta(months=1)                              
    print(data['seDate']+'已下載完成')
csvf.close()
csvf2.close()