Python selenium PhantomJS proxy

Question

这是我的代码：

from selenium import webdriver
proxylist=['58.12.12.12:80','69.12.12.12:80']
weblist=['https://www.google.com','https://www.facebook.com','https://www.yahoo.com','https://aol.com']
for s in range (len(proxylist)):
    service_args = ['--proxy=%s'%(proxylist[s]),'--proxy-type=socks5']
    driver = webdriver.PhantomJS('phantomjs.exe', service_args = service_args)
    for s in weblist:
        driver.get(s)

这个想法是浏览器首先将使用代理列表[0]来访问这些站点。如果 proxylist[0] 在网站 [2] 超时，则代理列表 [1] 将继续对网站 [3] 执行该工作。我想我应该使用尝试，除了但不知道把它们放在哪里。很高兴你帮忙！

Answer 1

试试这样的事情。 基本上，我们正在切换内循环和外循环并添加一个 try/except

for s in weblist:
    for s in range (len(proxylist)):
        try
            service_args = ['--proxy=%s'%(proxylist[s]),'--proxy-type=socks5']
            driver = webdriver.PhantomJS('phantomjs.exe', service_args = service_args)
            driver.get(s)
            break
        except TimeOutException:
            print 'timed out'

Answer 2

超时的尝试捕获如下所示：

try:
    driver.set_page_load_timeout(1)
    driver.get("http://www.example.com")
except TimeoutException as ex:
    print("Exception has been thrown. " + str(ex))

对于您的代码，添加它将如下所示：

from selenium import webdriver
from selenium.common.exceptions import TimeoutException
proxylist=['58.12.12.12:80','69.12.12.12:80']
weblist=['https://www.google.com','https://www.facebook.com','https://www.yahoo.com','https://aol.com']

def test():
    temp_count_proxy = 0
    driver_opened = 0
    for url in weblist:
        if temp_count_proxy > len(proxylist):
            print("Out of proxy")
            return
        if driver_opened == 0:
            service_args = ['--proxy={}'.format(proxylist[temp_count_proxy]),'--proxy-type=socks5']
            driver = webdriver.PhantomJS('phantomjs.exe', service_args = service_args)
            driver_opened = 1
        try:
            driver.set_page_load_timeout(2)
            driver.get(url)
        except TimeoutException as ex:
            driver.close()
            driver_opened = 0
            temp_count_proxy += 1
            continue
test()

请注意，如果它无法获取一个 url，它将更改代理，并获取下一个 url(按照您的要求(，但不会获得相同的 url。

如果您希望它在使用当前 URL 重试失败时更改代理，请使用以下内容：

from selenium import webdriver
from selenium.common.exceptions import TimeoutException
proxylist=['58.12.12.12:80','69.12.12.12:80']
weblist=['https://www.google.com','https://www.facebook.com','https://www.yahoo.com','https://aol.com']

def test():
    temp_count_proxy = 0
    driver_opened = 0
    for url in weblist:
        while True:
            if temp_count_proxy > len(proxylist):
                print("Out of proxy")
                return
            if driver_opened == 0:
                service_args = ['--proxy={}'.format(proxylist[temp_count_proxy]),'--proxy-type=socks5']
                driver = webdriver.PhantomJS('phantomjs.exe', service_args = service_args)
                driver_opened = 1
            try:
                driver.set_page_load_timeout(2)
                driver.get(url)
                # Your code to process here
            except TimeoutException as ex:
                driver.close()
                driver_opened = 0
                temp_count_proxy += 1
                continue
            break