Java抓取公眾號(服務號)所有文章

1.添加Maven依賴

<dependency>
    <groupId>org.seleniumhq.selenium</groupId>
    <artifactId>selenium-server</artifactId>
    <version>3.0.1</version>
</dependency>

2.下載chromedriver

http://npm.taobao.org/mirrors/chromedriver/

注意:chrome和chromedriver版本一定要一致,不然會出現(xiàn)一些小問題

3.小栗子

private static String webDriver = "webdriver.chrome.driver";
private static String webDriverPath = "C:\\MineProjects\\JavaDemo\\chromedriver_win32\\chromedriver.exe";
private static String targetPath = "https://mp.weixin.qq.com";
private static String searchPath = "https://mp.weixin.qq.com/cgi-bin/searchbiz";
private static String appmsgPath = "https://mp.weixin.qq.com/cgi-bin/appmsg";
private static Random random = new Random(1);
private static  Gson gson = new Gson();

private static String sourceName = "漁愉魚";   // 要爬的公眾號名稱(準確名稱)

private static String username = null;
private static String password = null;

static {
    ResourceBundle rb = ResourceBundle.getBundle("reptile");
    username = rb.getString("reptile.username");
    password = rb.getString("reptile.password");
}

public static void main(String[] args) {
    System.setProperty(webDriver, webDriverPath);
    WebDriver driver = null;
    try {
        driver = new ChromeDriver();

        weixinLogin(driver);

        String token = getToken(driver);

        Platform platform = getPlatform(driver, token);
        if (Objects.isNull(platform)) {
            throw new Exception("不存在" + sourceName + "公眾號");
        }
        InfoResult infoResult = getInfoResult(driver, token, platform.getFakeId(), 0, 5);
        System.out.println(infoResult);
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        if (Objects.nonNull(driver)) {
            driver.close();
        }
    }
}

/**
 * 獲取公眾號某一頁數(shù)據(jù)
 */
private static InfoResult getInfoResult(WebDriver driver, String token, String fakeId,
                                        Integer begin, Integer count) throws Exception {
    Map<String, String> queryInfoParams = new HashMap<>();
    queryInfoParams.put("token", token);
    queryInfoParams.put("lang", "zh_CN");
    queryInfoParams.put("f", "json");
    queryInfoParams.put("ajax", "1");
    queryInfoParams.put("random", random.nextDouble() + "");
    queryInfoParams.put("action", "list_ex");
    queryInfoParams.put("query", "");
    queryInfoParams.put("type", "9");
    queryInfoParams.put("fakeid", fakeId);

    queryInfoParams.put("begin", begin + "");
    queryInfoParams.put("count", count + "");

    appmsgPath = HttpUtils.setParams(appmsgPath, queryInfoParams);
    driver.get(appmsgPath);
    Document infoDocument = Jsoup.parse(driver.getPageSource());
    Elements infoList = infoDocument.select("pre");
    if (Objects.isNull(infoList)) {
        throw new Exception("獲取公眾號文章錯誤");
    }
    return gson.fromJson(infoList.text(), InfoResult.class);
}

/**
 * 獲取公眾號信息
 */
private static Platform getPlatform(WebDriver driver, String token) throws Exception {
    Map<String, String> searchNameParams = new HashMap<>();
    searchNameParams.put("action", "search_biz");
    searchNameParams.put("token", token);
    searchNameParams.put("lang", "zh_CN");
    searchNameParams.put("f", "json");
    searchNameParams.put("ajax", "1");
    searchNameParams.put("random", random.nextDouble() + "");
    searchNameParams.put("query", sourceName);
    searchNameParams.put("begin", "0");
    searchNameParams.put("count", "5");

    searchPath = HttpUtils.setParams(searchPath, searchNameParams);
    driver.get(searchPath);

    Document preDocument = Jsoup.parse(driver.getPageSource());
    Elements preList = preDocument.select("pre");
    if (Objects.isNull(preList)) {
        throw new Exception("獲取公眾號錯誤");
    }
    PlatformResult result = gson.fromJson(preList.text(), PlatformResult.class);

    Platform platform = null;
    for (int index = 0; index < result.getList().size(); index ++) {
        Platform item = result.getList().get(index);
        if (sourceName.equals(item.getNickname())) {
            platform = item;
        }
    }
    return platform;
}

/**
 * 獲取token
 */
private static String getToken(WebDriver driver) throws Exception {
    String current = driver.getCurrentUrl();
    if (StringUtils.isBlank(current)) {
        throw new Exception("獲取token鏈接有誤");
    }
    String token = current.split("token=")[1];
    if (StringUtils.isBlank(token)) {
        throw new Exception("token錯誤");
    }
    return token;
}

/**
 * 登錄模塊
 */
private static void weixinLogin(WebDriver driver) throws Exception {
    driver.get(targetPath);
    WebElement usernameWebElement = driver.findElement(By.name("account"));
    usernameWebElement.clear();
    usernameWebElement.sendKeys(username);

    WebElement passwordWebElement = driver.findElement(By.name("password"));
    passwordWebElement.clear();
    passwordWebElement.sendKeys(password);

    WebElement helpWebElement = driver.findElement(By.className("icon_checkbox"));
    helpWebElement.click();

    WebElement btnWebElement = driver.findElement(By.className("btn_login"));
    btnWebElement.click();

    System.out.println("請用手機微信掃碼二維碼登錄公眾號");
    Thread.sleep(15000);
}

4.總結(jié)

以前沒有用過selenium,這2天研究了一下,還是挺好玩的。搶票什么的,用這個應該可以做。這里貼的只是部分源碼,可以直接到碼云上,看我寫的ReptileDemo。

源碼地址:https://gitee.com/jsjack_wang/JavaDemo

最后編輯于
?著作權(quán)歸作者所有,轉(zhuǎn)載或內(nèi)容合作請聯(lián)系作者
【社區(qū)內(nèi)容提示】社區(qū)部分內(nèi)容疑似由AI輔助生成,瀏覽時請結(jié)合常識與多方信息審慎甄別。
平臺聲明:文章內(nèi)容(如有圖片或視頻亦包括在內(nèi))由作者上傳并發(fā)布,文章內(nèi)容僅代表作者本人觀點,簡書系信息發(fā)布平臺,僅提供信息存儲服務。

友情鏈接更多精彩內(nèi)容