Selenium webdriver:我想点击下一页直到最后一页

Question

from selenium import webdriver
from bs4 import BeautifulSoup as bs
import time
url = 'https://curecity.in/vendor-list.php?category=Doctor&filters_location=Jaipur&filters%5Bsubareas_global%5D=&filters_speciality='
driver = webdriver.Chrome('C:chromedriver.exe')
driver.get(url)
driver.maximize_window()

next_page_number=1
next_page = True
while next_page == True:
soup = bs(driver.page_source, 'html.parser')
for link in soup.find_all('div',class_='col-md-9 feature-info'):
link1 = link.find('a')
print(link1['href'])
try:
driver.find_element_by_link_text(">").click()
next_page_number+=1
time.sleep(1)
except:
print ('No more pages')
next_page=False
driver.close()

我正在尝试单击下一页，但上面编写的代码正在单击备用页面。我想从第一页点击到最后一页。

Answer 1

我在这个页面上发现了两个问题。

1(它加载页面非常慢，所以我不得不在获取数据和click()按钮之前甚至睡10秒。

2(按钮>的工作方式与我预期的不同 - 它会跳转3页(即使我在浏览器中手动单击它(，所以我搜索带有下一页编号的按钮并单击它。

driver.find_element_by_xpath('//a[@data-page="{}"]'.format(next_page_number)).click()

---

完整代码。即使没有BeautifulSoup也能工作

from selenium import webdriver
#from bs4 import BeautifulSoup as bs
import time
url = 'https://curecity.in/vendor-list.php?category=Doctor&filters_location=Jaipur&filters%5Bsubareas_global%5D=&filters_speciality='
driver = webdriver.Chrome('C:chromedriver.exe')
#driver = webdriver.Firefox()
driver.maximize_window()
driver.get(url)
next_page_number = 1
while True:
print('page:', next_page_number)
time.sleep(10)
#soup = bs(driver.page_source, 'html.parser')
#for link in soup.find_all('div',class_='col-md-9 feature-info'):
#    link1 = link.find('a')
#    print(link1['href'])
for link in driver.find_elements_by_xpath('//div[@class="col-md-2 feature-icon"]/a'):
print(link.get_attribute('href'))
try:
next_page_number += 1
driver.find_element_by_xpath('//a[@data-page="{}"]'.format(next_page_number)).click()
except:
print('No more pages')
break # exit loop
#driver.close()