Quantcast
Channel: Active questions tagged selenium - Stack Overflow
Viewing all articles
Browse latest Browse all 98221

Scraping data through API after actions with buttons

$
0
0

I wanna grab data from https response, but they data appears only after clicking on the button with opens the dialog window (look at the screen bellow), so I have to open all necessary windows and then run request.get. The main difficulty is I don't understand how to make request.get for the same window where I run driver.get command.

import requests
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("start-maximized")
driver = webdriver.Chrome(options=chrome_options)
driver.get(url)
XPATH = "//*[name()='svg' and contains(@class, 'jss350')]"
XPATH2 = '/html/body/div[3]/div[3]/div/div[3]/button/span[1]'

# Wait for all svg elements to be clickable
WebDriverWait(driver, 20).until(EC.presence_of_all_elements_located((By.XPATH, XPATH)))

# find list of svg elements
svg_elements = driver.find_elements_by_xpath(XPATH)

for element in svg_elements[3:4]:
    element.click() #open windows
    WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, XPATH2))).click() #close

#here is I want to run the request.get function to grab data from the same window

driver.quit()

params = {
    'products': 'RU',
}


url = 'my url'

r = requests.get(url, params=params, headers={
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36'})
json_data = r.json()
print(r.url)
print(r.text)

enter image description here


Viewing all articles
Browse latest Browse all 98221

Trending Articles



<script src="https://jsc.adskeeper.com/r/s/rssing.com.1596347.js" async> </script>