Python requests proxy, In this article, How to use bulk proxy in proxy rotation.You can use Proxy Rotation in 2 Efficient ways(automatic generate and own proxy file) and much more detailed explanation.
Contents
hide
Required Modules:
pip install requests
pip install colorama
urllib could be a package that collects several modules for working with URLs
urllib.request
for opening and reading URLsurllib.error
containing the exceptions raised byurllib.request
urllib.parse
for parsing URLsurllib.robotparser
for parsingrobots.txt
files
urllib may be a standard library, don’t need to install it
Now simply use the module import urllib in our Python Program.
#1 Proxy Rotatation with Automatically Generate Bulk Proxy:
- Automatically generate proxy list from proxyscrape.com
- Save file in your File Explorer in the give name of “local_file” in the code.
- Check and Valid Proxy via ipecho.net
- Automatically timeout(3 sec) from proxy checking.
import requests from colorama import init from colorama import Fore, Back, Style init(autoreset=True) import os , time , sys , subprocess from urllib import request import shutil def generate_proxy(): #If Run this Code, Everytime Generate a new Proxy List File #Remove Old Proxy List File if os.path.exists('proxy_list.txt'): os.remove('proxy_list.txt') #Download proxy list remote_url = 'https://api.proxyscrape.com/v2/?request=getproxies&protocol=http&timeout=10000&country=all&ssl=all&anonymity=all' #Save Proxy Filename local_file = 'proxy_list.txt' #Save Proxy List File In File Explorer request.urlretrieve(remote_url, local_file) generate_proxy() def rotating_proxy(): proxy_read = open("proxy_list.txt" , "r") while True: proxies = proxy_read.readline() if not proxies: proxy_read.close() time.sleep(1) print("Please Wait..Generating Proxy") generate_proxy() time.sleep(2) try: proxy = f'http://{proxies}' url = 'https://ipecho.net/plain/' #Automatically Timout, While Taking more Time page = requests.get(url, proxies={"http": proxy, "https": proxy} , timeout = 3) print(Style.BRIGHT + Fore.GREEN + "Valid Proxy Address : ", pages.text) except: print(Style.BRIGHT + Fore.RED + "Invalid Proxy " + proxies) rotating_proxy()
#2 Proxy Rotation your own Proxy List:
- Enter your Proxy List filename from the input().
- Add any Proxy socks5 , socks4 , elite etc..
- Check and Valid Proxy via ipecho.net
- Automatically timeout(3 sec) from proxy checking.
import requests from colorama import init from colorama import Fore, Back, Style init(autoreset=True) import os , time , sys , subprocess from urllib import request import shutil def rotating_proxy():. #Add proxy list file in your Current Directory own_file = input("Enter Proxy List Filename") proxy_read = open(own_file , "r") while True: proxies = proxy_read.readline() if not proxies: proxy_read.close() time.sleep(1) print("Please Wait..Generating Proxy") generate_proxy() time.sleep(2) try: proxy = f'http://{proxies}' url = 'https://ipecho.net/plain/' #Automatically Timout, While Taking more Time page = requests.get(url, proxies={"http": proxy, "https": proxy} , timeout = 3) print(Style.BRIGHT + Fore.GREEN + "Valid Proxy Address : ", pages.text) except: print(Style.BRIGHT + Fore.RED + "Invalid Proxy " + proxies) rotating_proxy()
#3 Speedup Rotating Proxy:
import time import requests import urllib.request import socket import socks import locale import warnings start_time = time.time() locale.setlocale(locale.LC_ALL, '') warnings.filterwarnings('ignore') url = 'http://ifconfig.io/ip' socks.set_default_proxy(socks.SOCKS5, "127.0.0.1", port=9050) socket.socket = socks.socksocket res = urllib.request.urlopen(url).read() print (res.decode()) end = round(time.time() - start_time, 2) print(f'--- {end} seconds ---') start_time = time.time() proxy = { 'http': 'http://111.68.26.237:8080', 'https': 'https://111.68.26.237:8080' } addr = requests.get(url, proxies=proxy).text print (addr) end = round(time.time() - start_time, 2) print(f'--- {end} seconds ---')
Detailed Python Requests Proxy
Prerequisites & Installation:
pip freeze
pip freeze will display installed in your current modules
pip install requests
How to use a Python Requests Proxy
import requests proxies = { 'http': 'http://proxy.example.com:8080', 'https': 'http://secureproxy.example.com:8090', } url = 'http://mywebsite.com/example' response = requests.post(url, proxies=proxies)
Requests Methods
response = requests.get(url) response = requests.post(url, data={"a": 1, "b": 2}) response = requests.put(url, data=put_body) response = requests.delete(url) response = requests.patch(url, data=patch_update) response = requests.head(url) response = requests.options(url)
Proxy Authentication:
response = requests.get(url, auth=('user', 'pass'))
Proxy Sessions:
import requests session = requests.Session() session.proxies = { 'http': 'http://10.10.10.10:8000', 'https': 'http://10.10.10.10:8000', } url = 'http://mywebsite.com/example' response = session.get(url)
Environmental Variables:
export HTTP_PROXY='http://10.10.10.10:8000'
export HTTPS_PROXY='http://10.10.10.10:1212'
Reading Responses:
response = requests.get(url)
text_resp = response.text
response = requests.get(url)
json_resp = response.json()
How to Rotate IPs with Requests
import requests ip_addresses = [ "mysuperproxy.com:5000", "mysuperproxy.com:5001", "mysuperproxy.com:5100", "mysuperproxy.com:5010", "mysuperproxy.com:5050", "mysuperproxy.com:8080", "mysuperproxy.com:8001", "mysuperproxy.com:8000", "mysuperproxy.com:8050" ] def proxy_request(request_type, url, **kwargs): while True: try: proxy = random.randint(0, len(ip_addresses) - 1) proxies = {"http": ip_addresses(proxy), "https": ip_addresses(proxy)} response = requests.get(request_type, url, proxies=proxies, timeout=5, **kwargs) print(f"Proxy currently being used: {proxy['https']}") break except: print("Error, looking for another proxy") return response