Selenium滚动到该元素,但不单击

thtygnil  于 2022-11-24  发布在  其他
关注(0)|答案(2)|浏览(148)

尝试在python中使用selenium从网站“www.example.com“的导航栏单击”下一步“按钮https://uk.trustpilot.com/categories/bars_cafes?subcategories=cafe。

from selenium.webdriver import Chrome
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup
import time 

URL = "https://uk.trustpilot.com/categories/bars_cafes?subcategories=cafe"
driver = Chrome(ChromeDriverManager().install())

class Scraper:
    def __init__(self, website):
        self.website = website
    
    def get_website(self):
        return driver.get(self.website)

    def ignore_cookie(self):
        try:
            ignore_cookies = driver.find_element(by=By.XPATH, value='//*[@id="onetrust-reject-all- handler"]')
            ignore_cookies.click()
        except AttributeError:
            pass


    def next_page(self):
        driver.find_element(by=By.NAME, value="pagination-button-next").click()

ignore cookie函数工作正常,但next_page函数滚动到next按钮,但没有点击它。

xfyts7mz

xfyts7mz1#

包括以下导入:

from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time as t

编辑next_page函数,如下所示:

wait = WebDriverWait(driver, 25)

next_page_button = wait.until(EC.element_to_be_clickable((By.XPATH, '//a[@name="pagination-button-next"]')))
next_page_button.location_once_scrolled_into_view
t.sleep(2)
next_page_button.click()

请参见https://www.selenium.dev/documentation/上的Selenium文档

w8biq8rn

w8biq8rn2#

这应该可以做到:

from selenium.webdriver import Chrome
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

url = "https://uk.trustpilot.com/categories/bars_cafes?subcategories=cafe"

class Scraper:
    def __init__(self, website):
        self.driver = Chrome(ChromeDriverManager().install())
        self.driver.get(website)
        self.wait = WebDriverWait(self.driver,20)

    def ignore_cookie(self):
        self.driver.find_element(By.CSS_SELECTOR, "button[class^='onetrust-close-btn-handler']").click()

    def fetch_content(self):
        while True:
            for item in self.driver.find_elements(By.CSS_SELECTOR, "section > [class*='card_card']"):
                shop_name = item.find_element(By.CSS_SELECTOR, "a[name='business-unit-card'] p[class*='displayName']").text
                yield shop_name

            try:
                self.next_page()
                self.wait.until(EC.staleness_of(item))
            except Exception as err:
                self.driver.quit()
                return

    def next_page(self):
        next_page = self.driver.find_element(By.CSS_SELECTOR, "a[name='pagination-button-next']")
        self.driver.execute_script("arguments[0].click();", next_page)

scrape = Scraper(url)
scrape.ignore_cookie()
for title in scrape.fetch_content():
    print(title)

相关问题