Quantcast
Channel: Active questions tagged selenium - Stack Overflow
Viewing all articles
Browse latest Browse all 98814

How to click all replies in youtube

$
0
0

I am trying to click all 'more-replies' and get all replies from one youtube page. Now I can get all comments, but replies are problem.So I tried to do this in 2 ways

First:

# -*- encoding: utf-8 -*-
import time
import csv
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
import selenium.common.exceptions as exc

def crawling(url, store_save):
    chrome_options = webdriver.ChromeOptions()
    chrome_options.add_argument('disable_infobars')

    chrome_driver_path = "./chromedriver.exe"
    driver = webdriver.Chrome(chrome_driver_path, options=chrome_options)
    driver.get(url)
    time.sleep(2)
    title = driver.find_element_by_xpath('//*[@id="container"]/h1/yt-formatted-string')
    print(title)

    html = driver.find_element_by_tag_name('html')
    html.send_keys(Keys.PAGE_DOWN)
    html.send_keys(Keys.PAGE_DOWN)
    time.sleep(2)

    last_page_height = driver.execute_script("return document.documentElement.scrollHeight")

    while True:
        driver.execute_script("window.scrollTo(0, document.documentElement.scrollHeight);") 
        time.sleep(3.0) 
        new_page_height = driver.execute_script("return document.documentElement.scrollHeight") 
        if new_page_height == last_page_height:
            break 
        last_page_height = new_page_height

    while True:
        try:
            loadMoreButton = driver.find_element_by_xpath('//*[@id="more-replies"]')
            time.sleep(2)
            loadMoreButton.click()
            time.sleep(5)
        except Exception as e:
            print(e)
            break

    time.sleep(10)

    comment_elems = driver.find_elements_by_xpath('//*[@id="content-text"]')
    all_comments = [elem.text for elem in comment_elems]
    write_file = "output_comment.txt"
    with open(write_file, "w", -1, "utf-8") as output:
        for line in all_comments:
            print(line)
            output.write(line + '\n')


url = input("input URL")
store_save = "./yt_comments.csv"
crawling(url, store_save)

Second:

WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.XPATH, '//*[@id="more-replies"]'))).click() 

Then the first 'more replies' button was clicked, but others were not. What should I do?


Viewing all articles
Browse latest Browse all 98814

Trending Articles



<script src="https://jsc.adskeeper.com/r/s/rssing.com.1596347.js" async> </script>