dependencies

requirements.txt

@@ -8,7 +8,7 @@ llama-index-embeddings-openai
 llama-index-llms-openai
 # needed for simpledirectoryreader to work
 llama-index-readers-file
-selenium==4.10.0
+selenium==4.22.0
 unstructured
 requests
 chromium

web_search.py

@@ -72,16 +72,24 @@ def process_url(url, timeout):
             options.add_argument("--headless")
             options.add_argument("--no-sandbox")
             options.add_argument("--disable-dev-shm-usage")
+
+            options.add_argument("start-maximized")
+            options.add_argument("disable-infobars")
+            options.add_argument("--disable-extensions")
+            options.add_argument("--disable-gpu")
+            options.add_argument("--disable-dev-shm-usage")
             result = ""
-            with webdriver.Chrome(service=ChromeService(ChromeDriverManager().install()), options=options) as dr:
-                logger.info(f"*****setting page load timeout {timeout}")
-                dr.set_page_load_timeout(timeout)
-                try:
-                    dr.get(url)
-                    response = dr.page_source
-                    result = response_text_extract(url=url, response=response)
-                except selenium.common.exceptions.TimeoutException:
-                    return "", url
+
+            driver = webdriver.Chrome(service=ChromeService(ChromeDriverManager().install()), options=options)
+            logger.info(f"*****setting page load timeout {timeout}")
+            driver.set_page_load_timeout(timeout)
+
+            try:
+                driver.get(url)
+                response = driver.page_source
+                result = response_text_extract(url=url, response=response)
+            except selenium.common.exceptions.TimeoutException:
+                return "", url
     except Exception:
         traceback.print_exc()
         logger.info(f"{site} err")