通過網(wǎng)上爬蟲獲取了全國所有企業(yè)，然后就需要補(bǔ)充企業(yè)信息，首先想到的就是企查查，啟信寶等專業(yè)網(wǎng)站，最終選擇了企查查，嘗試了多種方法：

1、selenium爬蟲，繞過企查查的登錄驗(yàn)證，但賬號和IP限制太大，最終放棄

2、通過requests直接請求+cookies，遇到了cookie有效期和限制問題

不斷的嘗試和修改參數(shù)，最終發(fā)現(xiàn)一種有效方式selenium + wep

只需要IP代理，不需要賬號，沒有限制，因?yàn)槭菦]有登錄，拿到的信息有限，能展示的都能獲取。

image

一、初始化selenium

sysstr = platform.system()

if(sysstr =="Windows"):
    chromedriver_path = os.getcwd() + "\\utools\\chromedriver.exe"
else: #mac
    chromedriver_path = os.getcwd() + "/mac_chromedriver"

logger.info("chromedriver_path: %s" %(chromedriver_path,))

default_agent = '--user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36"'


class wap_QiChaCha(object):
    def __init__(self, user_agent_header=default_agent, chromedriver_path=chromedriver_path):
        self.options = webdriver.ChromeOptions()  # 創(chuàng)建一個配置對象
        self.options.add_argument('--no-sandbox')
        self.options.add_argument('--disable-dev-shm-usage')
        self.options.add_argument('--proxy-server=http://47.98.154.206:3008')
        self.options.add_argument("--headless") # 開啟無界面模式cd
        self.options.add_argument("--disable-gpu")  # 可選項：禁用gpu，可以解決一些莫名的問題
        self.options.add_argument(user_agent_header)
        mobileEmulation = {'deviceName': 'iPhone X'}
        self.options.add_experimental_option('mobileEmulation', mobileEmulation)


    def init(self):
        self.driver = webdriver.Chrome(executable_path=chromedriver_path, chrome_options=self.options)

        # 打開登錄頁面
        self.driver.get('https://m.qichacha.com/')
        self.error_encounter = 0

二、判斷公司存在

  def search_company(self, company_name):
        #time.sleep(0.3)

        try:
            result = {}

            result[COMPANY.NAME] = utils.normalizeCompanyName(company_name)

            logger.info("search for: %s" %(company_name,))

            '''
            self.driver.get('https://m.qichacha.com/')

            self.driver.find_element_by_id('searchkey').send_keys(company_name)
            # 單擊搜索按鈕
            srh_btn = self.driver.find_element_by_xpath('//*[@id="V3_Index_S"]//span')
            srh_btn.click()
            '''

            self.driver.get('https://m.qcc.com/search?key=%s' %(company_name))

            utils.alertWait(WebDriverWait(self.driver, 3).until,
                            expected_conditions.presence_of_element_located(
                                (By.XPATH, '//*[contains(@class,"text-danger") or contains(@class,"nodata")]')), 5, 0,
                                    "not found text-danger or nodata")

            # 檢測企業(yè)是不是存在
            inc_full = self.driver.find_element_by_xpath('//*[@class="text-danger"]').text

            self.error_encounter = 0

            if inc_full == "0":
                logger.error("company %s not found" %(company_name,))
                return None

            # 獲取首個企業(yè)文本
            cname = self.driver.find_element_by_xpath('//div[@class="list-item-name"]').text
            href = self.driver.find_element_by_xpath('//a[@class="a-decoration"]').get_attribute("href")

            # 曾用名
            cym = None
            try:
                stock_or_others = self.driver.find_element_by_xpath('//div[@class="list-item-bottom"]').text
                # print(stock_or_others)
                # 稱呼不同：曾用名 或 歷史股東等
                if utils.normalizeCompanyName(company_name) in stock_or_others:
                    company_bottom = stock_or_others.replace("：", ":")
                    cym = company_bottom.split(":")[1]
            except:
                # 獲取下面顯示失敗
                pass

            if utils.normalizeCompanyName(cname) == utils.normalizeCompanyName(company_name) \
                    or utils.normalizeCompanyName(cym) == utils.normalizeCompanyName(company_name):
                result[COMPANY.URL] = href

                # time.sleep(0.2)
                return self.company_detail(href, result)

        except Exception as err:
            # self.driver.delete_all_cookies()
            logger.error(err)
            self.error_encounter = self.error_encounter + 1
            if self.error_encounter >= 3:
                self.driver.quit()
                self.init()

            return None
        finally:
            pass

image

三、獲取公司信息

 def company_detail(self, href, result):

        self.driver.get(href)

        utils.alertWait(WebDriverWait(self.driver, 3).until,
                        expected_conditions.presence_of_element_located((By.XPATH, '//*[@class="company-name"]')), 5, 0,
                        "not found text-danger")

        try:
            phone = self.driver.find_element_by_xpath('/html/body/div[1]/div[2]/div[1]/div[3]/a[1]').text
            if phone and len(phone) > 0:
                result[COMPANY.TEL] = phone.strip()
        except Exception as e:
            pass
            # logger.info("沒有手機(jī)號")

        try:
            email = self.driver.find_element_by_xpath('/html/body/div[1]/div[2]/div[1]/div[3]/a[2]').text
            if email and len(email) > 0:
                result[COMPANY.EMAIL] = email.strip()
        except Exception as e:
            pass
            # logger.info("沒有郵箱")

        try:
            address = self.driver.find_element_by_xpath('/html/body/div[1]/div[2]/div[1]/div[4]').text
            if address and len(address) > 0:
                result[COMPANY.ADDRESS] = address.strip()
        except Exception as e:
            pass
            # logger.info("沒有地址")

        try:
            infos = self.driver.find_element_by_xpath('//div[@class="basic-wrap"]/table')
            # infos = self.driver.find_element_by_xpath('//*[@id="Cominfo"]/table')
        except:
            return result


        result[COMPANY.TAX_LEVEL] = "稅務(wù)等級&&"
        try:
            taxcreditlist = self.driver.find_element_by_xpath('//div[@id="taxcreditlist"]').text
            info = str(taxcreditlist).replace("\n", "&").strip()
            result[COMPANY.TAX_LEVEL] = result[COMPANY.TAX_LEVEL] + info
        except:
            return result

        # 轉(zhuǎn)為etree
        data = etree.HTML(infos.get_property("innerHTML"))
        data_info = data.xpath('.//tr')

        result[COMPANY.BUSINESS] = "工商信息"
        for info in data_info:
            info_list = info.xpath(".//td//text()")

            new_info_list = []
            for info in list(info_list):
                new_info = str(info).replace("\n", "").strip()
                new_info_list.append(new_info)

            new_info_list = [i for i in new_info_list if i != '']
            self.retrieveInfo(new_info_list, result)

            result[COMPANY.BUSINESS] = result[COMPANY.BUSINESS] + " && " + " && ".join(
                map(str, new_info_list))  # 以 && 分割 連接 list 內(nèi)容

        return result

image