scrapy 代理ip池怎么使用?

phantomjs 可通过以下设置代理ip

#coding=utf-8

import os

import re

import time

import requests

from scrapy.selector import HtmlXPathSelector

from scrapy.http import HtmlResponse

from selenium import webdriver

from selenium.webdriver.common.proxy import ProxyType

import sys

reload(sys)

sys.setdefaultencoding("utf-8")

import warnings

warnings.filterwarnings("ignore")

if __name__ == __main__:

PATH_PHANTOMJS=rD:\phantomjs\bin\phantomjs.exe

browser=webdriver.PhantomJS(PATH_PHANTOMJS)

#真实ip

browser.get(http://icanhazip.com/) #获取ip网址

response = HtmlResponse(url=,body=str(browser.page_source))

hxs = HtmlXPathSelector(response)

print your ip is:,.join(hxs.select(//text()).extract()).strip()

#代理ip

proxy=webdriver.Proxy()

proxy.proxy_type=ProxyType.MANUAL

proxy.http_proxy=220.248.229.45:3128

#将代理设置添加到webdriver.DesiredCapabilities.PHANTOMJS中

proxy.add_to_capabilities(webdriver.DesiredCapabilities.PHANTOMJS)

browser.start_session(webdriver.DesiredCapabilities.PHANTOMJS)

browser.get(http://icanhazip.com/) #获取ip网址

response = HtmlResponse(url=,body=str(browser.page_source))

hxs = HtmlXPathSelector(response)

print your proxy ip is:,.join(hxs.select(//text()).extract()).strip()

重点接入代码:

Phantomjs http/socks5

phantomjs --proxy=ip:port --proxy-type=[http|socks5|none] demo.js

上海、北京、广州、深圳等全国300+城市地区,测试送10000个ip使用,支持数据采集,网络投票,seo监控等等

品易HTTP-爬虫代理IP-千万动态HTTP代理IP​http.py.cn/?utm-source=py&utm-keyword=?1007