I am trying to click all 'more-replies' and get all replies from one youtube page. Now I can get all comments, but replies are problem.So I tried to do this in 2 ways
First:
# -*- encoding: utf-8 -*-
import time
import csv
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
import selenium.common.exceptions as exc
def crawling(url, store_save):
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('disable_infobars')
chrome_driver_path = "./chromedriver.exe"
driver = webdriver.Chrome(chrome_driver_path, options=chrome_options)
driver.get(url)
time.sleep(2)
title = driver.find_element_by_xpath('//*[@id="container"]/h1/yt-formatted-string')
print(title)
html = driver.find_element_by_tag_name('html')
html.send_keys(Keys.PAGE_DOWN)
html.send_keys(Keys.PAGE_DOWN)
time.sleep(2)
last_page_height = driver.execute_script("return document.documentElement.scrollHeight")
while True:
driver.execute_script("window.scrollTo(0, document.documentElement.scrollHeight);")
time.sleep(3.0)
new_page_height = driver.execute_script("return document.documentElement.scrollHeight")
if new_page_height == last_page_height:
break
last_page_height = new_page_height
while True:
try:
loadMoreButton = driver.find_element_by_xpath('//*[@id="more-replies"]')
time.sleep(2)
loadMoreButton.click()
time.sleep(5)
except Exception as e:
print(e)
break
time.sleep(10)
comment_elems = driver.find_elements_by_xpath('//*[@id="content-text"]')
all_comments = [elem.text for elem in comment_elems]
write_file = "output_comment.txt"
with open(write_file, "w", -1, "utf-8") as output:
for line in all_comments:
print(line)
output.write(line + '\n')
url = input("input URL")
store_save = "./yt_comments.csv"
crawling(url, store_save)
Second:
WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.XPATH, '//*[@id="more-replies"]'))).click()
Then the first 'more replies' button was clicked, but others were not. What should I do?