Java爬蟲（HttpClient）

網(wǎng)絡(luò)爬蟲主要功能就是對網(wǎng)頁內(nèi)容進(jìn)行爬取，然后根據(jù)特定需求對內(nèi)容進(jìn)行過濾分析。
針對網(wǎng)頁內(nèi)容爬取，假設(shè)需求為要對一個(gè)網(wǎng)站進(jìn)行全站爬取，將爬取的文件按類型保存在本地磁盤，并提供配置網(wǎng)站爬取的最大層次、最大鏈接數(shù)、爬取類型范圍等。
這里使用kafka主題做爬蟲隊(duì)列，使用springboot做了一個(gè)簡單的實(shí)現(xiàn)。

任務(wù)創(chuàng)建接口

這里提供了兩個(gè)接口，一個(gè)是輸入網(wǎng)站爬取配置創(chuàng)建爬取任務(wù)，一個(gè)是根據(jù)任務(wù)id查詢?nèi)蝿?wù)狀態(tài)，沒有提供詳細(xì)結(jié)果查詢，結(jié)果可直接數(shù)據(jù)庫查看。

@RestController
public class CrawlerTaskController {

    @Autowired
    private WebsiteTaskService websiteTaskService;

    @Autowired
    private WebsiteTaskDao websiteTaskDao;

    @Autowired
    private TaskProducer taskProducer;

    @PostMapping("task/add")
    @ResponseBody
    public Map<String, Object> addWebsiteTask(WebsiteTask item) {
        item.setTaskCount(1);
        websiteTaskService.put(item);
        UrlTask task = new UrlTask();
        task.setUrl(item.getUrl());
        task.setParentId(-1);
        task.setRootId(task.getRootId());
        task.setLevel(0);
        taskProducer.sendUrlTask(task);
        Map<String, Object> map = new HashMap<>();
        map.put("id", item.getId());
        map.put("message", "爬蟲任務(wù)添加成功！");
        return map;
    }

    @PostMapping("task/get")
    @ResponseBody
    public WebsiteTask getWebsiteTask(int id) {
        return websiteTaskDao.findById(id).get();
    }
}

網(wǎng)站任務(wù)實(shí)體

@Entity
@EntityListeners(AuditingEntityListener.class)
public class WebsiteTask {

    @Id
    @GeneratedValue(strategy = GenerationType.IDENTITY)
    private int id;
    @Column(length = 1024)
    private String url;//網(wǎng)站url，一般為首頁鏈接
    private int maxLevel;//最大爬取層次
    private int maxCount;//最大爬取鏈接數(shù)
    private int outerLevel;//最大爬取外鏈層次
    private String range;//爬取類型范圍
    private int taskCount;//任務(wù)數(shù)
    private int finishCount;//爬取完成任務(wù)數(shù)
    private int state = 1;//狀態(tài)：1=執(zhí)行中;2=已完成
    @CreatedDate
    private Date createTime;//創(chuàng)建時(shí)間
    private Date finishTime;//完成時(shí)間

    @Transient
    private List<String> ranges;

    ......
}

url任務(wù)實(shí)體

@Entity
@EntityListeners(AuditingEntityListener.class)
public class UrlTask {

    @Id
    @GeneratedValue(strategy = GenerationType.IDENTITY)
    private int id;
    private int parentId;//父頁面任務(wù)id
    private int rootId;//網(wǎng)站任務(wù)id
    @Column(length = 1024)
    private String url;
    private String contentType;//網(wǎng)頁類型
    private long contentLength;//內(nèi)容長度
    private int level;//當(dāng)前層次
    private long useMillis;//爬取用時(shí)
    private int respCode;//響應(yīng)狀態(tài)碼
    private String remark;//備注
    private String filePath;//保存磁盤文件路徑
    @CreatedDate
    private Date createTime;//創(chuàng)建時(shí)間

    ......
}

爬取的網(wǎng)頁模型定義

public class WebPageModel {

    public int respCode = 200;//響應(yīng)狀態(tài)碼
    public String message;//異常信息
    public Document document;//html頁面Document對象
    public String encoding;//頁面編碼
    public String contentType;//網(wǎng)頁類型
    public long contentLength;//內(nèi)容長度
    public String filePath;//文件路徑
    public String fileExt;//文件后綴

    public PageFormat format = PageFormat.OTHER;

    public enum PageFormat {
        HTML, IMAGE, AUDIO, VIDEO, TXT, WORD, EXCEL, PPT, PDF, COMPRESS, APK, IPA, OTHER
    }

    public void updateFormat() {
        String type = contentType;
        if (ContentTypeUtil.OCTET_STREAM_TYPE.equalsIgnoreCase(contentType)) {
            type = ContentTypeUtil.getContentType(fileExt);
        }
        if (ContentTypeUtil.isHtml(type)) {
            format = PageFormat.HTML;
        } else if (ContentTypeUtil.isImage(type)) {
            format = PageFormat.IMAGE;
        } else if (ContentTypeUtil.isAudio(type)) {
            format = PageFormat.AUDIO;
        } else if (ContentTypeUtil.isVideo(type)) {
            format = PageFormat.VIDEO;
        } else if (ContentTypeUtil.isTxt(type)) {
            format = PageFormat.TXT;
        } else if (ContentTypeUtil.isWord(type)) {
            format = PageFormat.WORD;
        } else if (ContentTypeUtil.isExcel(type)) {
            format = PageFormat.EXCEL;
        } else if (ContentTypeUtil.isPpt(type)) {
            format = PageFormat.PPT;
        } else if (ContentTypeUtil.isPdf(type)) {
            format = PageFormat.PDF;
        } else if (ContentTypeUtil.isCompress(type)) {
            format = PageFormat.COMPRESS;
        } else if (ContentTypeUtil.isApk(type)) {
            format = PageFormat.APK;
        } else if (ContentTypeUtil.isIpa(type)) {
            format = PageFormat.IPA;
        }
    }
}

url去重

public class UrlDuplicateFilter {

    private final Object lock = new Object();
    private final Set<String> set = new HashSet<>();
    private final int maxCount; // 最大不相同數(shù)量

    public UrlDuplicateFilter(int maxCount) {
        this.maxCount = maxCount;
    }

    /**
     * 過濾重復(fù)url
     */
    public boolean filter(String url) {
        if (StringUtils.isBlank(url)) {
            return false;
        }
        synchronized (lock) {
            if (reachMaxCount() || set.contains(url)) {
                return false;
            }
            set.add(url);
        }
        return true;
    }

    /**
     * 判斷數(shù)量是否達(dá)到上限
     */
    private boolean reachMaxCount() {
        return set.size() >= maxCount;
    }
}

一條url的爬取執(zhí)行過程

public class CrawlerTask implements Runnable {

    private static final Logger LOG = LoggerFactory.getLogger(CrawlerTask.class);
    private UrlTask task;
    private WebsiteTaskService websiteTaskService;
    private UrlTaskDao urlTaskDao;
    private TaskProducer taskProducer;

    public CrawlerTask(UrlTask task, WebsiteTaskService websiteTaskService, UrlTaskDao urlTaskDao,
                       TaskProducer taskProducer) {
        this.task = task;
        this.websiteTaskService = websiteTaskService;
        this.urlTaskDao = urlTaskDao;
        this.taskProducer = taskProducer;
    }

    @Override
    public void run() {
        long millis = System.currentTimeMillis();
        WebsiteTask website = websiteTaskService.getWebsiteTask(task.getRootId());
        //爬取鏈接內(nèi)容
        WebPageModel page = PageDownloadUtil.executeGet(task.getUrl(), website.getRanges());
        task.setContentLength(page.contentLength);
        task.setContentType(page.contentType);
        task.setRespCode(page.respCode);
        task.setRemark(page.message);
        task.setFilePath(page.filePath);
        task.setUseMillis(System.currentTimeMillis() - millis);
        urlTaskDao.saveAndFlush(task);

        if (task.getLevel() < website.getMaxLevel()) {
            Set<String> childUrls = new UrlExtract(page.document, task.getUrl()).extractFromA().extractFromFrame()
                    .extractFromIframe().extractFromImg().getUrls();
            if (!childUrls.isEmpty()) {
                UrlDuplicateFilter dupFilter = websiteTaskService.getUrlDuplicateFilter(task.getRootId());
                int addCount = 0;
                for (String childUrl : childUrls) {
                    if (CrawlerUtil.isOuterUrl(task.getUrl(), childUrl) && task.getLevel() >= website.getOuterLevel()) {
                        continue;
                    }
                    //提取出的子鏈接去重
                    if (dupFilter.filter(childUrl)) {
                        UrlTask childTask = new UrlTask();
                        childTask.setUrl(childUrl);
                        childTask.setParentId(task.getId());
                        childTask.setRootId(task.getRootId());
                        childTask.setLevel(task.getLevel() + 1);
                        taskProducer.sendUrlTask(childTask);
                        addCount++;
                    }
                }
                //任務(wù)數(shù)更新
                websiteTaskService.addTaskCount(task.getRootId(), addCount);
            }
        }
        //完成任務(wù)數(shù)更新
        websiteTaskService.addFinishCount(task.getRootId());
        LOG.info(String.format("爬取用時(shí)=%s，url=%s", System.currentTimeMillis() - millis, task.getUrl()));
    }
}

網(wǎng)頁爬取工具

public class PageDownloadUtil {

    private static final Logger LOG = LoggerFactory.getLogger(PageDownloadUtil.class);
    private static final int MAX_HTML_LENGTH = 20 * 1024 * 1024;//html頁面限制20M
    private static final int MAX_FILE_LENGTH = 500 * 1024 * 1024;//其它附件類型限制500M
    private static final String FOLDER_NAME = "d:/temp/" + UUID.randomUUID().toString().replace("-", "") + "/";
    private static final AtomicInteger INDEX = new AtomicInteger();
    private static final CloseableHttpClient client = HttpClientUtil.createHttpClient();
    //自定義錯(cuò)誤返回值
    private static final Map<Integer,String> CODE_MAP = new HashMap<>();
    static{
        CODE_MAP.put(-501,"uri解析異常");
        CODE_MAP.put(-502,"網(wǎng)絡(luò)協(xié)議異常");
        CODE_MAP.put(-503,"域名解析異常");
        CODE_MAP.put(-504,"http連接異常");
        CODE_MAP.put(-505,"網(wǎng)絡(luò)IO異常");
        CODE_MAP.put(-506,"頁面解析異常");
        CODE_MAP.put(-507,"編碼格式異常");
        CODE_MAP.put(-508,"內(nèi)容長度超出限制");
        CODE_MAP.put(-509,"網(wǎng)頁類型超出可爬取范圍");
    }

    public static WebPageModel executeGet(String url, List<String> ranges) {
        WebPageModel page = new WebPageModel();
        int redirectTimes = 0;
        boolean redirect;
        URI uri = CrawlerUtil.urlConvertToUri(url);
        if (uri == null) {
            page.respCode = -501;
            page.message = CODE_MAP.get(page.respCode);
            return page;
        }
        do {
            redirectTimes++;
            redirect = false;
            HttpGet method = new HttpGet(uri);
            HttpClientUtil.setHeader(method, url);
            CloseableHttpResponse response = null;
            long millis = System.currentTimeMillis();
            try {
                response = client.execute(method);
                page.respCode = response.getStatusLine().getStatusCode();
                if (page.respCode == HttpStatus.SC_OK) {
                    download(page, url, response, ranges);
                } else if (page.respCode >= 300 && page.respCode < 400) {// 頁面跳轉(zhuǎn)
                    Header[] locationHeader = response.getHeaders("location");
                    if (locationHeader != null && locationHeader.length > 0) {
                        String redirectUrl = locationHeader[0].getValue();
                        if (StringUtils.isNotBlank(redirectUrl) && !url.equals(redirectUrl)) {
                            uri = CrawlerUtil.urlConvertToUri(redirectUrl);
                            redirect = true;
                        }
                    }
                }
            } catch (ClientProtocolException e) {
                LOG.error("", e);
                page.respCode = -502;
                page.message = CODE_MAP.get(page.respCode);
            } catch (UnknownHostException e) {
                LOG.error("", e);
                page.respCode = -503;
                page.message = CODE_MAP.get(page.respCode);
            } catch (HttpHostConnectException e) {
                LOG.error("", e);
                page.respCode = -504;
                page.message = CODE_MAP.get(page.respCode);
            } catch (IOException e) {//連接超時(shí)嘗試重連3次
                redirectTimes++;
                redirect = true;
                LOG.error(String.format("第%s次鏈接失敗，executeusetime=%s", redirectTimes / 2,
                        System.currentTimeMillis() - millis), e);
                page.respCode = -505;
                page.message = CODE_MAP.get(page.respCode);
            } finally {
                if (response != null) {
                    EntityUtils.consumeQuietly(response.getEntity());
                    try {
                        response.close();
                    } catch (IOException e) {
                        LOG.error("responseclose", e);
                    }
                }
                method.releaseConnection();
            }
        } while (redirect && redirectTimes <= 5);
        return page;
    }

    private static void download(WebPageModel page, String url, CloseableHttpResponse response,
                                 List<String> ranges) {
        HttpEntity entity = response.getEntity();
        page.contentLength = entity.getContentLength();// 此方法不準(zhǔn)確，經(jīng)常返回-1，后面重新賦值

        // ContentType.getOrDefault(entity).getMimeType()提取可能會因?yàn)榉侵С值腸harset類型而報(bào)錯(cuò)，所以這里改為手工提取mimeType
        Header header = entity.getContentType();
        if (header != null) {
            HeaderElement[] headerElements = header.getElements();
            if (headerElements != null && headerElements.length > 0) {
                page.contentType = headerElements[0].getName();
            }
        }
        if (ContentTypeUtil.OCTET_STREAM_TYPE.equalsIgnoreCase(page.contentType)) {
            page.fileExt = HttpClientUtil.getOctetStreamFileExt(url, response);
        } else if (page.contentType == null) {
            //若未從header中取到contentType，根據(jù)url后綴判斷
            if (url.lastIndexOf("/") > 8) {
                String name = url.substring(url.lastIndexOf("/"));
                if (name.contains(".")) {
                    page.contentType = ContentTypeUtil.getContentType(name.substring(name.lastIndexOf(".")));
                }
            }
        }
        page.updateFormat();

        if (ranges.contains(page.format.toString())) {
            if (page.format == WebPageModel.PageFormat.HTML) {
                if (page.contentLength == 0 || page.contentLength > MAX_HTML_LENGTH) {
                    page.respCode = -508;
                    page.message = CODE_MAP.get(page.respCode);
                    return;
                }
                try {
                    String html = null;
                    Document document = null;
                    String charset = null;
                    if (header != null) {
                        charset = CrawlerUtil.judgeCharset(header.toString());
                    }
                    if (charset != null) {
                        html = EntityUtils.toString(entity, charset);
                        document = Jsoup.parse(html);
                    } else {
                        byte[] data = EntityUtils.toByteArray(entity);
                        html = new String(data, CrawlerUtil.UTF_8);
                        document = Jsoup.parse(html);
                        charset = CrawlerUtil.getCharsetFromMeta(document);
                        if (charset != null && !CrawlerUtil.UTF_8.equals(charset)) {
                            html = new String(data, charset);
                            document = Jsoup.parse(html);
                        }
                    }
                    byte[] data = html.getBytes(CrawlerUtil.UTF_8);
                    page.contentLength = data.length;
                    if (page.contentLength <= 0 || page.contentLength > MAX_HTML_LENGTH) {
                        page.respCode = -508;
                        page.message = CODE_MAP.get(page.respCode);
                        return;
                    }

                    page.encoding = CrawlerUtil.UTF_8;
                    page.document = document;

                    createFilePath(page);
                    HttpClientUtil.exportDataAsFile(data, page.filePath);
                } catch (ParseException e) {
                    LOG.error("", e);
                    page.respCode = -506;
                    page.message = CODE_MAP.get(page.respCode);
                } catch (UnsupportedEncodingException e) {
                    LOG.error("", e);
                    page.respCode = -507;
                    page.message = CODE_MAP.get(page.respCode);
                } catch (IOException e) {
                    LOG.error("", e);
                    page.respCode = -505;
                    page.message = CODE_MAP.get(page.respCode);
                }
            } else {// 如果是非html頁面直接下載
                if (page.contentLength == 0 || page.contentLength > MAX_FILE_LENGTH) {
                    page.respCode = -508;
                    page.message = CODE_MAP.get(page.respCode);
                    return;
                }
                if (page.fileExt == null) {
                    page.fileExt = ContentTypeUtil.getExtendFileName(page.contentType);
                }
                createFilePath(page);
                HttpClientUtil.exportEntityAsFile(entity, page.filePath);
            }
        } else {
            page.respCode = -509;
            page.message = CODE_MAP.get(page.respCode);
        }
    }

    private static void createFilePath(WebPageModel page) {
        String filePath = FOLDER_NAME + page.format.toString() + "/";
        File file = new File(filePath);
        file.mkdirs();
        page.filePath = filePath + INDEX.getAndIncrement() + page.fileExt;
    }
}

項(xiàng)目地址

詳細(xì)項(xiàng)目代碼可到我的Github上查看下載：https://github.com/DexterQY/website-crawler

?著作權(quán)歸作者所有,轉(zhuǎn)載或內(nèi)容合作請聯(lián)系作者
【社區(qū)內(nèi)容提示】社區(qū)部分內(nèi)容疑似由AI輔助生成，瀏覽時(shí)請結(jié)合常識與多方信息審慎甄別。
平臺聲明：文章內(nèi)容（如有圖片或視頻亦包括在內(nèi)）由作者上傳并發(fā)布，文章內(nèi)容僅代表作者本人觀點(diǎn)，簡書系信息發(fā)布平臺，僅提供信息存儲服務(wù)。

相關(guān)閱讀更多精彩內(nèi)容

友情鏈接更多精彩內(nèi)容