Selenium按钮未加载或无法点击

Question

我正试图让页面点击"加载更多"；按钮，直到它到达页面底部以提取所有数据。然而，尽管Selenium有明确的等待，但按钮有时不会加载，或者它确实出现但没有可点击(我打印了"加载更多点击"(，看看自动化是否真的点击了，而且似乎在点击——只是按钮不可点击(。我使用的是Selenium 4.1.5和Chrome驱动程序104。这是代码：

import pandas as pd
import time
import csv
import selenium.webdriver as webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException, 
ElementClickInterceptedException, NoSuchElementException, ElementNotInteractableException
from selenium.webdriver.chrome.service import Service as ChromeService
from selenium.webdriver.chrome.options import Options as ChromeOptions
from selenium.webdriver.support.ui import WebDriverWait, Select
from webdriver_manager.chrome import ChromeDriverManager
def ChromeDriver():
options = ChromeOptions()
options.add_argument('--disable-gpu')
options.add_argument("force-device-scale-factor=1.00")
options.headless = False
service = ChromeService("C:Python39chromedriver.exe")
driver = webdriver.Chrome(options=options, service=service)
#driver.maximize_window()
return driver
def toggle_button():
toggle_button = driver.find_element(By.XPATH, "//input[@type='checkbox']")
toggle_button.click()
def set_currency(currency):
active_currency = driver.find_element(By.XPATH, "//span[contains(@class, 'modeTitleActive-AXg2YMF0')]").text
if active_currency!= currency:
toggle_button()
else:
pass
def accept_all():
try:
promtpw = driver.find_element(By.XPATH, "//div[@class='item-x59qvYIc']")
promtpw = promtpw.find_element(By.XPATH, "//button[@class='acceptAll-ofWyKNKk button-KXqj4Lf5 size-xsmall-KXqj4Lf5 color-brand-KXqj4Lf5 variant-primary-KXqj4Lf5']").click()
except NoSuchElementException:
pass
except ElementNotInteractableException:
pass
def save_as_csv(data_list):
for idx, data in enumerate(data_list[0:4]):
file_title = data_list[4][idx]
file = f"csv/tv_crypto_prices_{file_title}.csv"
with open(file, "w", newline="") as csv_file:
tv_writer = csv.writer(csv_file)
tv_writer.writerow([file_title.upper()])
data.to_csv(file, mode="a")
print("csv files saved")
def quit_driver():
driver.close()
driver.quit()
def get_categories():
categories = driver.find_elements(By.XPATH, "//div[starts-with(@class, 'item-x59qvYIc')]")
return categories
def get_categories_list(categories_objs):
categories_list = []
for category in categories:
append_category_text = categories_list.append(category.text)
return categories_list
def fetch_data():
data = []
categories_list = get_categories_list(categories)
#categories_list = categories_list.append(categories_list.pop(categories_list.index("Overview")))
accept_all()
for category in categories:
category.click()
time.sleep(2)
print(category.text)

#Load data
load_more = True
while load_more:
try:
time.sleep(2)
load_more_button = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//span[@class='tv-load-more__btn']")))
load_more_button.click()
print("Load more clicked")
except NoSuchElementException:
load_more = False
except ElementNotInteractableException:
load_more = False

crypto = pd.read_html(driver.page_source)[1]
crypto.columns.values[0] = "Name" #Filter column values
append_data = data.append(crypto)
append_categories = data.append(categories_list)
quit_driver()
return data
def run_tvc():
#Get driver
driver = ChromeDriver()
driver.get("https://www.tradingview.com/markets/cryptocurrencies/prices-all/")

#Set currency
currency = "BTC"
set_currency(currency)

#Get data
d = fetch_data()
save_as_csv(d)
run_tvc()

我还尝试将load_more_button = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//span[@class='tv-load-more__btn']")))替换为load_more_button = driver.find_element(By.XPATH, "//span[@class='tv-load-more__btn']")，这似乎对所有类别都起到了作用，除了第一个类别("概述"(，其中页面首先与url一起加载。我希望它能以任何一种方式工作，简而言之，问题是：

1. 如何让它与WebDriverWait一起工作，每次都以TimeOutException结束
2. 为什么加载更多按钮不出现在第一类"；概述"；硒，你是怎么解决的？我中途停止了这个过程，并尝试用完全相同的Selenium的Chrome窗口重新加载页面，按钮总是出现并起作用

Answer 1

真的需要硒吗？如果不是这样，那么另一种解决方案是通过API加载所需的数据。您可以从API获取整个数据，其中方法为post，数据格式为json，仅使用requests模块。

脚本示例：

import requests
import pandas as pd
payload = {"filter":[{"left":"market_cap_calc","operation":"nempty"},{"left":"sector","operation":"nempty"},{"left":"name","operation":"match","right":"BTC$"}],"options":{"lang":"en"},"markets":["crypto"],"symbols":{"query":{"types":[]},"tickers":[]},"columns":["base_currency_logoid","sector","market_cap_calc","market_cap_diluted_calc","close","total_shares_outstanding","total_shares_diluted","total_value_traded","change","pricescale","minmov","fractional","minmove2"],"sort":{"sortBy":"market_cap_calc","sortOrder":"desc"},"range":[0,450]}
url= 'https://scanner.tradingview.com/crypto/scan'
headers= {
"content-type": "application/x-www-form-urlencoded",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36",
}
lst=[]
req=requests.post(url,headers=headers,json=payload).json()
for item in req['data']:
lst.append({
"Name":item['d'][1]})

df = pd.DataFrame(lst)#.to_csv('out.csv',index=False)#to store data just uncomment
print(df)

输出：

Name
0                         Bitcoin
1                        Ethereum
2                          Tether
3                        USD Coin
4                    Binance Coin
..                            ...
445                  Tornado Cash
446                      Vertcoin
447                     TokenClub
448                    TrustVerse
449  Atletico De Madrid Fan Token
[450 rows x 1 columns]