Update helper_functions_api.py

helper_functions_api.py

@@ -196,43 +196,27 @@ class Scraper:
             print(f"Error fetching page content for {url}: {e}")
         return None
 
-def extract_main_content(url):
-    if url:
-        try:
-            result = urlparse(url)
-            if all([result.scheme, result.netloc]):
-                # Prepare query parameters
-                params = {
-                    "url": url,
-                    "favor_precision": False,
-                    "favor_recall": False,
-                    "output_format": "markdown",
-                    "target_language": "en",
-                    "include_tables": True,
-                    "include_images": False,
-                    "include_links": False,
-                    "deduplicate": True,
-                }
-
-                # Make request to FastAPI endpoint
-                response = requests.get("https://pvanand-web-scraping.hf.space/extract-article", params=params)
-
-                if response.status_code == 200:
-                    return response.json()["article"]
-                else:
-                    return ""
-        except:
-            return ""
+def extract_main_content(html):
+    if html:
+        plain_text = ""
+        soup = BeautifulSoup(html, 'lxml')
+        for element in soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'table']):
+            plain_text += element.get_text(separator=" ", strip=True) + "\n"
+        return plain_text
+    return ""
 
 def process_content(data_format, url, query):
-    content = extract_main_content(url)
-    if content:
-        rephrased_content = rephrase_content(
-            data_format=data_format,
-            content=limit_tokens(remove_stopwords(content), token_limit=4000),
-            query=query,
-        )
-        return rephrased_content, url
+    scraper = Scraper()
+    html_content = scraper.fetch_content(url)
+    if html_content:
+        content = extract_main_content(html_content)
+        if content:
+            rephrased_content = rephrase_content(
+                data_format=data_format,
+                content=limit_tokens(remove_stopwords(content), token_limit=1000),
+                query=query,
+            )
+            return rephrased_content, url
     return "", url
 
 def fetch_and_extract_content(data_format, urls, query):