可以使用项管道将输出拆分为多个文件。请参阅下面的示例代码。

import scrapy
from scrapy.exporters import CsvItemExporter
from urllib.parse import urlparse

class PerFilenameExportPipeline:
    def open_spider(self, spider):
        self.filename_to_exporter = {}

    def close_spider(self, spider):
        for exporter in self.filename_to_exporter.values():
            exporter.finish_exporting()

    def _exporter_for_item(self, item):
        filename = item.get('url').strip()
        if filename not in self.filename_to_exporter:
            f = open(f'{filename}.csv', 'wb')
            exporter = CsvItemExporter(f, export_empty_fields=True)
            exporter.start_exporting()
            self.filename_to_exporter[filename] = exporter
        return self.filename_to_exporter[filename]

    def process_item(self, item, spider):
        exporter = self._exporter_for_item(item)
        exporter.export_item(item)
        return item

class SampleSpider(scrapy.Spider):
    name = 'sample'
    start_urls = ['https://example.com', "https://www.scrapethissite.com/pages/simple", "https://stackoverflow.com"]

    custom_settings = {
        'USER_AGENT': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36',
        "ITEM_PIPELINES": {PerFilenameExportPipeline: 100}
    }

    def parse(self, response):
        parsed =  urlparse(response.url)
        yield {
            "url": parsed.netloc,
            "title": response.css("title::text").get()
        }