Quantcast
Channel: Active questions tagged selenium - Stack Overflow
Viewing all articles
Browse latest Browse all 97762

Making Webscraping with selenium faster (or faster alternatives)

$
0
0

Guys i currently have a working script that scrapes ajax content from an certain page...the thing is that it take +- 12 seconds to run and for my purposes i would need it to be faster.

Any tips?

from urllib.parse import urlencode
import requests
import time
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver import Chrome
from selenium.webdriver.chrome.options import Options

def search_char():
    char_name_input = str(input('Search Character: ')) # User input / Character Name

    start_time = time.time()
    browser = webdriver.PhantomJS()


    search_url = 'https://www.tibia.com/community/?subtopic=characters' # URL 

    r = browser.get(search_url) # Searched Character Page

    element = browser.find_element_by_name("name")
    element.send_keys(char_name_input)
    element2 = browser.find_element_by_name("Submit").click()


    page = browser.find_element_by_id('Content')
    rendered_page = page.get_attribute('innerHTML')

    soup = BeautifulSoup(rendered_page, 'html.parser')



    selection = soup.find_all('td')

    lista = []
    for item in selection:
        lista.append(item.get_text())

    browser.close()

    print("--- %s seconds ---" % (time.time() - start_time))

    for i in lista:
        print(i,'\n')

search_char()

Viewing all articles
Browse latest Browse all 97762

Trending Articles



<script src="https://jsc.adskeeper.com/r/s/rssing.com.1596347.js" async> </script>