python爬虫--requests库常用实例

发送GET请求

import requests url = "" headers = { "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", } req = requests.get(url, headers=headers) print(req.content.decode("utf-8")) # content为bytes类型

发送POST请求

import requests url = "?px=default&city=%E4%B8%8A%E6%B5%B7&needAddtionalResult=false" headers = { "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", "Referer": "?px=default&city=%E4%B8%8A%E6%B5%B7" } data = { "first": "true", "pn": "1", "kd": "python" } req = requests.post(url, headers=headers, data=data) print(req.json()) # 转换json为字典

使用代理

import requests url = "" headers = { "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", } data = { "first": "true", "pn": "1", "kd": "python" } proxy = { "http": "117.191.11.107:8080" } req = requests.get(url, headers=headers, proxies=proxy) print(req.text){ "origin": "117.191.11.107" }

使用cookie

import requests url = "" headers = { "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", } req = requests.get(url, headers=headers) print(req.cookies.get_dict()) # 获取cookies

使用session

import requests login_url = "" headers = { "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", } data = { "email": "xxx", "password": "xxx" } sess = requests.session() sess.post(login_url, headers=headers, data=data) r = sess.get("", headers=headers) with open("me.html", "w", encoding="utf-8") as fp: fp.write(r.content.decode("utf-8"))

访问不受信任的站点

req = requests.get(url, headers=headers, verify=false) # verify设置为false,否则会报错