如何使用python中的多处理来爬行数百万个URL

Question

我已经实施了三层递归来生成URL的种子列表，然后从每个URL中刮除信息。我想使用多处理来利用系统的所有内核来加快爬行的速度。这是我到目前为止已经实施的轨道代码。

# -*- coding: utf-8 -*-
import scrapy
from scrapy.selector import HtmlXPathSelector
from scrapy.http.request import Request
from CompanyInfoGrabber.Utility.utils import getAddress, getCompanyStatus, getDirectorDetail, getRegNumber

class CompanyInfoGrabberSpider(scrapy.Spider):
    name = 'CompanyDetail'
    allowed_domains = ['example.com']
    start_urls = ['http://example.com']
    def parse(self, response):
        counter = 0
        print("User Agent in parse() is : ", response.request.headers['User-Agent'])
        hxp = HtmlXPathSelector(response)
        URL_LIST = hxp.select('//sitemapindex/sitemap/loc/text()').extract()
        print("URL LIST: ", URL_LIST)
        for URL in URL_LIST[:2]:
           next_page = response.urljoin(URL)
           yield Request(next_page, self.parse_page)
    def parse_page(self, response):
        print("User Agent in parse_page is : ", response.request.headers['User-Agent'])
        hxp = HtmlXPathSelector(response)
        # create seed list of company-url
        COMPANY_URL_LIST = hxp.select('//urlset/url/loc/text()').extract()
        print("Company url: ", COMPANY_URL_LIST[:20])
        """
        Here I want to use multiprocessing like this
        pool = Pool(processes=8)
        pool.map(parse_company_detail, COMPANY_URL_LIST)
        """
        for company_url in COMPANY_URL_LIST[:5]:
            next_page = response.urljoin(company_url)
            yield Request(next_page, self.parse_company_detail)
    def parse_company_detail(self, response):
        COMPANY_DATA = dict()
        print("User Agent in parse_company_page() is : ", response.request.headers['User-Agent'])
        hxp = HtmlXPathSelector(response)
        _ABOUT_ = ''.join(hxp.xpath('normalize-space(//div[@class="panel-body"]/text())').extract())
        for node in hxp.xpath('//div[@class="panel-body"]//p'):
            _ABOUT_ += ''.join(node.xpath('string()').extract())
        COMPANY_DATA['About'] = _ABOUT_
        # Get company data.
    COMPANY_DATA = getDirectorDetail(COMPANY_DATA, hxp)
    print("Dictionary: ", COMPANY_DATA)
    return COMPANY_DATA

我如何使用多处理来抓取URL的种子列表？预先感谢。

更新：我的问题不是重复的。在这里，我只有一个蜘蛛。

问：

Om Prakash

Answer 1

我建议使用螺纹模块同时运行多个线程。您需要修改类以在 init 中进行URL参数。

import threading
sites = ['URL1','URL2','URL3']
def create_instance():
    global sites
    CompanyInfoGrabberSpider(scrapy.Spider,sites[0])
    sites.remove[sites[0]]
for site in sites:
    threading.Thread(target=create_instance).start() # Create and start thread