I'm trying to scrape information from a more dynamic web page, however the code that I'm using is not working. i believe there's some kind of problem with the webdriver or my proxy, as I get a mistake with the following text: Message: invalid argument: user data directory is already in use, please specify a unique value for --user-data-dir argument, or don't use --user-data-dir
When I run the script it only opens a new tab in chrome, not the requested url, and the pages doesn't close.
My code is the following:
from selenium import webdriver
from selenium.webdriver.common.proxy import Proxy, ProxyType
from selenium.webdriver.chrome.options import Options
from datetime import datetime
myProxy = {
"http" : "http://10.120.118.49:8080",
"https" : "https://10.120.118.49:8080"
}
headers={'User-agent' : 'Mozilla/5.0'}
startTime=datetime.now()
options = Options()
proxy = Proxy()
proxy.proxyType = ProxyType.MANUAL
proxy.autodetect = False
proxy.httpProxy = proxy.sslProxy = proxy.socksProxy = "http://10.120.118.49:8080"
options.Proxy = proxy
options.add_argument('--ignore-certificate-errors')
options.add_argument("--test-type")
options.add_argument("test-type");
options.add_argument("start-maximized");
options.add_argument("--window-size=1920,1080");
options.add_argument("--enable-precise-memory-info");
options.add_argument("--disable-popup-blocking");
options.add_argument("--disable-default-apps");
options.add_argument("test-type=browser");
options.add_argument("--no-sandbox");
options.add_argument("--user-data-dir=C:/Users/C71220/AppData/Local/Google/Chrome/hello");
driver = webdriver.Chrome(r"C:/Users/C71220/chromedriver.exe", options=options)
url='https://www.delhaize.be/nl-be/shop/Dranken-en-alcohol/Frisdrank/c/v2DRISOF?q=:relevance:manufacturerNameFacet:Fanta:manufacturerNameFacet:Coca-Cola:manufacturerNameFacet:Schweppes:manufacturerNameFacet:Fuze%20Tea:manufacturerNameFacet:Monster:manufacturerNameFacet:Sprite:manufacturerNameFacet:Aquarius:manufacturerNameFacet:Finley:manufacturerNameFacet:Honest&sort=relevance&pageNumber=0'
driver.get(url)
product_name="""/html/body/main/div[1]/div[3]/ul/li[1]/div/div/div[2]/div[1]/header/a/div/p[2]"""
product= driver.find_element_by_xpath(product_name)
print(product.text)
driver.close()
endtime=f'''Execution Time:datetime.now() - startTime'''***Edit: some words