Previously, I was using selenium webdriver for web scraping purposes for various websites. But on a whim, I decided to try to use rotating proxy IP addresses for web scraping purposes, because I wanted to learn what that was. For that purpose, I searched online and found this article, and decided to try it out:
https://medium.com/ml-book/multiple-proxy-servers-in-selenium-web-driver-python-4e856136199d
But when I used it in my code, I am not even able to go to any damn website; not even the 'get' statement works :(
I get this message in my Anaconda Spyder console .
Note: I took a screenshot and put its here:
Then, I removed the code that I had copied from this article. Even then, my code won't stop connecting by proxy!!!! It simply refuses to not use proxy, it's like my code is taking revenge on me.
Here is my code:
import xlrd
import pandas as pd
import datetime as dt
import xlwings as xw
import sys
import math
import xlwt
from xlwt import Workbook
import openpyxl
from openpyxl import load_workbook
from collections import Counter
import shutil as shu
import os
import time
from selenium import webdriver
#from http_request_randomizer.requests.proxy.requestProxy import RequestProxy
sz = ('Coast_Retail_-_Auto_Weekly_Update.xlsx')
sz1 = xlrd.open_workbook(sz)
sz2 = sz1.sheet_by_index(0)
hz='Coast_Retail_-_Auto_Weekly_Update.xlsx'
hz1=load_workbook(hz)
hz2=hz1.worksheets[0]
req_proxy = RequestProxy() #you may get different number of proxy when you run this at each time
proxies = req_proxy.get_proxy_list() #this will create proxy list
PROXY = proxies[0].get_address()
webdriver.DesiredCapabilities.CHROME['proxy']={
"httpProxy":PROXY,
"ftpProxy":PROXY,
"sslProxy":PROXY,
"proxyType":"MANUAL",
}
d = webdriver.Chrome(executable_path=r'R:\Sulaiman\temp_code_vineet\nick\chromedriver.exe')
time.sleep(5)
d.get("https://tfl.compass.inovatec.ca")
time.sleep(5)
un = d.find_element_by_id("UserName")
un.send_keys("vpande")
pw = d.find_element_by_id("Password")
pw.send_keys("v123456A")
sb = d.find_element_by_class_name("red-btn")
sb.click()
time.sleep(5)
qz=[]
for i in range(4,sz2.nrows):
try:
if(sz2.cell_value(i,13)=="Booked"):
fn=sz2.cell_value(i,0)
ln=sz2.cell_value(i,1)
fun=fn+""+ln
sch = d.find_element_by_class_name("search")
sch.send_keys(fun)
sch.send_keys(u'\ue007')
time.sleep(5)
d.find_element_by_xpath('//*[@id="body"]/section/div/div[2]/div[1]/div[2]/a[2]').click()
time.sleep(5)
x=d.find_element_by_xpath('/html/body/div[5]/section/div/div[2]/div/div[2]/div[2]/div[4]/span[2]').text
print(x)
qz.append(x)
d.get("https://tfl.compass.inovatec.ca")
time.sleep(5)
except:
print("err at "+str(i))
pass
print(qz)