from selenium import webdriver from selenium.webdriver.common.by import By import streamlit as st from selenium.webdriver import FirefoxOptions import re import logging def scrape_text(url, n_words=15): try: driver = None logging.warning("Initiated Scraping") user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36" opts = FirefoxOptions() opts.add_argument("--headless") opts.add_argument(f"user-agent={user_agent}") driver = webdriver.Firefox(options=opts) driver.set_page_load_timeout(60) driver.get(url) elem = driver.find_element(By.TAG_NAME, "body").text # h1 = driver.find_element(By.TAG_NAME, "h1").text # elem = h1 + "\n" + elem sents = elem.split("\n") sentence_list = [] for sent in sents: sent = sent.strip() if (len(sent.split()) >= n_words) and (len(re.findall(r"^\w.+[^\w\)\s]$", sent))>0): sentence_list.append(sent) driver.quit() logging.warning("Closed Webdriver") logging.warning("Successfully scraped text") if len(sentence_list) < 3: raise Exception("Found nothing to scrape.") return " \n\n\n".join(sentence_list) except: if driver: driver.close() logging.warning("Closed Webdriver") raise