Selenium webdriver设置代理(有密码&无密码)

踩坑日记:日前需要用到代理来整理点数据,发现老是失败,经过度娘解决了问题,故写下一贴。

1、当代理没有密码得处理方式

from selenium import webdriverfrom selenium.webdriver.support.ui import WebDriverWaitfrom selenium.webdriver.support import expected_conditions as ECfrom selenium.webdriver.common.by import BychromeOptions = webdriver.ChromeOptions()ip = "202.20.16.82"port = "888"# 设置代理chromeOptions.add_argument("--proxy-server=http://{}:{}".format(ip, port))driver = webdriver.Chrome(chrome_options=chromeOptions)# 最大等待时间10swait = WebDriverWait(driver, 10)driver.get(https:www.baidu.com)# 根据页面加载元素来判断是否加载到自己想要的源码位置wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, .bg s_btn)))text = driver.page_sourceprint(text)

2、当代理有账号和密码的时候

源码:

from selenium import webdriverimport stringimport zipfilefrom selenium.webdriver.chrome.options import Options# 代理隧道验证信息(账号+密码)proxyUser = ""proxyPass = ""ip = "202.20.16.82"port = "888"# 有密码得ip代理def create_proxy_auth_extension(proxy_host, proxy_port, proxy_username, proxy_password, scheme=http, plugin_path=None):if plugin_path is None:plugin_path = r{}_{}@http-dyn.dobel.com_9020.zip.format(proxy_username, proxy_password)manifest_json = """{"version": "1.0.0","manifest_version": 2,"name": "Dobel Proxy","permissions": ["proxy","tabs","unlimitedStorage","storage","","webRequest","webRequestBlocking"],"background": {"scripts": ["background.js"]},"minimum_chrome_version":"22.0.0"}"""background_js = string.Template("""var config = {mode: "fixed_servers",rules: {singleProxy: {scheme: "${scheme}",host: "${host}",port: parseInt(${port})},bypassList: ["http://foobar.com"]}};chrome.proxy.settings.set({value: config, scope: "regular"}, function() {});function callbackFn(details) {return {authCredentials: {username: "${username}",password: "${password}"}};}chrome.webRequest.onAuthRequired.addListener(callbackFn,{urls: [""]},[blocking]);""").substitute(host=proxy_host,port=proxy_port,username=proxy_username,password=proxy_password,scheme=scheme,)with zipfile.ZipFile(plugin_path, w) as zp:zp.writestr("manifest.json", manifest_json)zp.writestr("background.js", background_js)return plugin_pathproxy_auth_plugin_path = create_proxy_auth_extension(proxy_host=ip,proxy_port=port,proxy_username=proxyUser,proxy_password=proxyPass)chrome_options = Options()chrome_options.add_extension(proxy_auth_plugin_path)driver = webdriver.Chrome(chrome_options=chrome_options)url = "https:http://www.baidu.com"driver.get(url)# 如需隐式设置等待加载时间,参考无密码的处理方式text = driver.page_sourceprint("源码:", text)

上面就是chrome驱动器关于IP代理有密码和没密码的处理方式,如需使用,替换ip以及port就可以直接使用啦(有密码的还要更换上面的账户和密码)