受人之托，需要youtube視頻下面的評(píng)論數(shù)據(jù)，由于youtube對(duì)于爬蟲有限制，因此第一次嘗試selenium模擬人工點(diǎn)擊爬蟲，但爬取到的評(píng)論數(shù)遠(yuǎn)遠(yuǎn)低于youtube上面顯示的評(píng)論數(shù)。后面經(jīng)過調(diào)查發(fā)現(xiàn)，Google有自己的youtube data api，方便你對(duì)評(píng)論數(shù)據(jù)的需要。

網(wǎng)址：https://developers.google.com/youtube/v3/quickstart/python

第一步：
點(diǎn)擊剛才給的網(wǎng)址，注冊(cè)一個(gè)Google cloud platform，操作就按下面圖片進(jìn)行就好。

image.png

import os
import numpy as np
import google_auth_oauthlib.flow
import googleapiclient.discovery
import googleapiclient.errors
from googleapiclient.errors import HttpError
import pandas as pd
import json
import socket
import socks
import requests
## 科學(xué)上網(wǎng)，你也需要科學(xué)使用代理，不然科學(xué)不了外網(wǎng)，也許你不會(huì)需要。
## headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36'}
## socks.set_default_proxy(socks.SOCKS5, "127.0.0.1", 1080)
## socket.socket = socks.socksocket
## 
scopes = ["https://www.googleapis.com/auth/youtube.force-ssl"]

def main():
    # Disable OAuthlib's HTTPS verification when running locally.
    # *DO NOT* leave this option enabled in production.
    os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1"

    api_service_name = "youtube"
    api_version = "v3"
# 這個(gè)文件需要自己注冊(cè)完，自己下載
    client_secrets_file = "client_secret_961831598513-urdgliumr9j4ab4g68jtocc30dimqb9g.apps.googleusercontent.com.json"
    # Get credentials and create an API client
    flow = google_auth_oauthlib.flow.InstalledAppFlow.from_client_secrets_file(
        client_secrets_file, scopes)
    credentials = flow.run_console()

    youtube = googleapiclient.discovery.build(
        api_service_name, api_version, credentials=credentials)
    videoId = '5YGc4zOqozo'
    request = youtube.commentThreads().list(
        part="snippet,replies",
        videoId=videoId,
        maxResults = 100
    )
    response = request.execute()
    # print(response)

    totalResults = 0
    totalResults = int(response['pageInfo']['totalResults'])

    count = 0
    nextPageToken = ''
    comments = []
    first = True
    further = True
    while further:
        halt = False
        if first == False:
            print('..')
            try:
                response = youtube.commentThreads().list(
                    part="snippet,replies",
                    videoId=videoId,
                    maxResults = 100,
                    textFormat='plainText',
                    pageToken=nextPageToken
                            ).execute()
                totalResults = int(response['pageInfo']['totalResults'])
            except HttpError as e:
                print("An HTTP error %d occurred:\n%s" % (e.resp.status, e.content))
                halt = True

        if halt == False:
            count += totalResults
            for item in response["items"]:
                # 這只是一部分?jǐn)?shù)據(jù)，你需要啥自己選就行，可以先打印下你能拿到那些數(shù)據(jù)信息，按需爬取。
                comment = item["snippet"]["topLevelComment"]
                author = comment["snippet"]["authorDisplayName"]
                text = comment["snippet"]["textDisplay"]
                likeCount = comment["snippet"]['likeCount']
                publishtime = comment['snippet']['publishedAt']
                comments.append([author, publishtime, likeCount, text])
            if totalResults < 100:
                further = False
                first = False
            else:
                further = True
                first = False
                try:
                    nextPageToken = response["nextPageToken"]
                except KeyError as e:
                    print("An KeyError error occurred: %s" % (e))
                    further = False
    print('get data count: ', str(count))
    ### write to csv file
    data = np.array(comments)
    df = pd.DataFrame(data, columns=['author', 'publishtime', 'likeCount', 'comment'])
    df.to_csv('google_comments.csv', index=0, encoding='utf-8')

    ### write to json file
    result = []
    for name, time, vote, comment in comments:
        temp = {}
        temp['author'] = name
        temp['publishtime'] = time
        temp['likeCount'] = vote
        temp['comment'] = comment
        result.append(temp)
    print('result: ', len(result))

    json_str = json.dumps(result, indent=4)
    with open('google_comments.json', 'w', encoding='utf-8') as f:
        f.write(json_str)

    f.close()
if __name__ == "__main__":
    main()