Update app.py

app.py

@@ -177,10 +177,13 @@ async def chat(
 
 def extract_text_from_webpage(html_content):
     """Extracts visible text from HTML content using BeautifulSoup."""
-    soup = BeautifulSoup(html_content)
-    for tag in soup(["script", "style", "header", "footer"]):
+    soup = BeautifulSoup(html_content, "html.parser")
+    # Remove unwanted tags
+    for tag in soup(["script", "style", "header", "footer", "nav"]):
         tag.extract()
-    return soup.get_text(strip=True)
+    # Get the remaining visible text
+    visible_text = soup.get_text(strip=True)
+    return visible_text
 
 async def fetch_and_extract(url, max_chars, proxy: Optional[str] = None):
     """Fetches a URL and extracts text asynchronously."""
@@ -245,15 +248,19 @@ async def web_search_and_extract(
 
 def extract_text_from_webpage2(html_content):
     """Extracts visible text from HTML content using BeautifulSoup."""
-    soup = BeautifulSoup(html_content)
-    for tag in soup(["script", "style", "header", "footer"]):
+    soup = BeautifulSoup(html_content, "html.parser")
+    # Remove unwanted tags
+    for tag in soup(["script", "style", "header", "footer", "nav"]):
         tag.extract()
-    return soup.get_text(strip=True)
+    # Get the remaining visible text
+    visible_text = soup.get_text(strip=True)
+    return visible_text
 
-def fetch_and_extract2(url, max_chars):
+def fetch_and_extract2(url, max_chars, proxy: Optional[str] = None):
     """Fetches a URL and extracts text using threading."""
+    proxies = {'http': proxy, 'https': proxy} if proxy else None
     try:
-        response = requests.get(url, headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/111.0"})
+        response = requests.get(url, headers={"User-Agent": "Mozilla/5.0"}, proxies=proxies)
         response.raise_for_status()
         html_content = response.text
         visible_text = extract_text_from_webpage2(html_content)
@@ -267,19 +274,20 @@ def fetch_and_extract2(url, max_chars):
 @app.get("/api/websearch-and-extract-threading")
 def web_search_and_extract_threading(
     q: str,
-    max_results: int = 10,
+    max_results: int = 3,
     timelimit: Optional[str] = None,
     safesearch: str = "moderate",
     region: str = "wt-wt",
     backend: str = "html",
-    max_chars: int = 10000,
-    extract_only: bool = True
+    max_chars: int = 6000,
+    extract_only: bool = True,
+    proxy: Optional[str] = None
 ):
     """
     Searches using WEBS, extracts text from the top results using threading, and returns both.
     """
     try:
-        with WEBS() as webs:
+        with WEBS(proxy=proxy) as webs:
             # Perform WEBS search
             search_results = webs.text(keywords=q, region=region, safesearch=safesearch,
                                      timelimit=timelimit, backend=backend, max_results=max_results)
@@ -289,7 +297,7 @@ def web_search_and_extract_threading(
             threads = []
             for result in search_results:
                 if 'href' in result:
-                    thread = threading.Thread(target=lambda: extracted_results.append(fetch_and_extract2(result['href'], max_chars)))
+                    thread = threading.Thread(target=lambda: extracted_results.append(fetch_and_extract2(result['href'], max_chars, proxy)))
                     threads.append(thread)
                     thread.start()