我想限制刮到5页的数量与下面的代码,虽然该网站有50页。我在用Scrapy的爬行蜘蛛我怎么能这么做呢?
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
class BooksSpider(CrawlSpider):
name = "bookscraper"
allowed_domains = ["books.toscrape.com"]
start_urls = ["https://books.toscrape.com/"]
rules = (Rule(LinkExtractor(restrict_xpaths='//h3/a'), callback='parse_item', follow=True),
Rule(LinkExtractor(restrict_xpaths='//li[@class="next"]/a'), follow=True),)
def parse_item(self, response):
product_info = response.xpath('//table[contains(@class, "table-striped")]')
name = response.xpath('//h1/text()').get()
upc = product_info.xpath('(./tr/td)[1]/text()').get()
price = product_info.xpath('(./tr/td)[3]/text()').get()
availability = product_info.xpath('(./tr/td)[6]/text()').get()
yield {'Name': name, 'UPC': upc, 'Availability': availability, 'Price': price}
1条答案
按热度按时间yws3nbqq1#
deny
参数和蜘蛛将停止扫描第五页。https://docs.scrapy.org/en/latest/topics/link-extractors.html#module-scrapy.linkextractors.lxmlhtml