google search once again

Dockerfile

@@ -11,8 +11,6 @@ WORKDIR /app
 
 COPY --chown=user ./requirements.txt requirements.txt
 RUN pip install --no-cache-dir --upgrade -r requirements.txt
-RUN pip install --no-cache-dir trafilatura
-RUN pip install --no-cache-dir "lxml[html_clean]"
 
 COPY --chown=user . /app
 CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

nuse_modules/google_search.py

@@ -4,27 +4,35 @@ import os
 import requests
 import time
 from typing import List
-from trafilatura import fetch_url, extract
+from boilerpy3 import extractors          # ← switched library
 
 GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
-GOOGLE_CX_ID = os.getenv("GOOGLE_CX_ID")
+GOOGLE_CX_ID   = os.getenv("GOOGLE_CX_ID")
+
+# initialise once (thread-safe)
+article_extractor = extractors.ArticleExtractor()
 
 
 def extract_full_text(url: str) -> str:
+    """
+    Download a page and return its readable main text.
+    Falls back to empty string on any failure.
+    """
     try:
-        downloaded = fetch_url(url)
-        if downloaded:
-            content = extract(downloaded, include_comments=False, include_tables=False)
-            return content or ""
+        return article_extractor.get_content_from_url(url) or ""
     except Exception as e:
         print(f"[SCRAPER ERROR] {url}: {e}")
-    return ""
+        return ""
 
 
 def search_google_news(keywords: List[str], num_results: int = 5) -> List[dict]:
+    """
+    Run a Google Custom Search and return a list of dicts with:
+        title, link, snippet, content (full article text)
+    """
     query = " ".join(keywords)
     url = (
-        f"https://www.googleapis.com/customsearch/v1"
+        "https://www.googleapis.com/customsearch/v1"
         f"?key={GOOGLE_API_KEY}&cx={GOOGLE_CX_ID}"
         f"&q={query}&num={num_results}"
     )
@@ -33,17 +41,17 @@ def search_google_news(keywords: List[str], num_results: int = 5) -> List[dict]:
         res = requests.get(url, timeout=10)
         res.raise_for_status()
         data = res.json()
-        results = []
 
+        results = []
         for item in data.get("items", []):
             link = item.get("link")
             article_text = extract_full_text(link)
 
             results.append({
-                "title": item.get("title"),
-                "link": link,
+                "title":   item.get("title"),
+                "link":    link,
                 "snippet": item.get("snippet"),
-                "content": article_text
+                "content": article_text,
             })
 
         return results

requirements.txt

@@ -6,4 +6,5 @@ redis
 transformers
 accelerate
 torch
-huggingface_hub
+huggingface_hub
+boilerpy3==1.0.6