我试图刮一个动态生成的网站使用 selenium +Scrapy。
我已经成功地从第一页刮下了项目，但当我试图进入下一页时，似乎为 selenium 生成的浏览器总是在请求同一页。
我正在尝试：
1.-执行parse函数以提取第一页数据。
2.-提取完成后，搜索Next按钮并获取href属性。
3.-再次调用相同的函数发送新的url。

import scrapy
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

class IndeedSpiderSpider(scrapy.Spider):
    name = "indeed"
    allowed_domains = ["es.indeed.com"]

    def start_requests(self):
        url = "https://es.indeed.com/jobs?q&l=Barcelona"
        yield scrapy.Request(url=url, callback=self.parse_jobs)

    def parse_jobs(self, response):
        driver = webdriver.Firefox()
        driver.get("https://es.indeed.com/jobs?q&l=Barcelona")
        driver.implicitly_wait(10)
        offersnames=driver.find_elements(By.XPATH, "//td/div/h2/span")

        for i in range(len(offersnames)):
            yield {
                "name": offersnames[i].text
            }

        next_page_element = driver.find_element(By.CSS_SELECTOR, "ul.pagination-list > li:last-child > a")
        next_page_url=next_page_element.get_attribute("href")
        if next_page_url:
            next_page = response.urljoin(next_page_url)
            yield scrapy.Request(url=next_page, callback=self.parse_jobs)

        driver.quit()