Update helper_functions_api.py

helper_functions_api.py

@@ -4,6 +4,7 @@ from mistune.plugins.table import table
 from jinja2 import Template
 import re
 import os
+from urllib.parse import urlparse
 
 def md_to_html(md_text):
     renderer = mistune.HTMLRenderer()
@@ -70,7 +71,16 @@ from together import Together
 llm_default_small = "meta-llama/Llama-3-8b-chat-hf"
 llm_default_medium = "meta-llama/Llama-3-70b-chat-hf"
 
-SysPromptData = "You are an information retriever and summarizer, return only the factual information regarding the user query"
+SysPromptData = """You are expert in information extraction from the given context.
+                    Steps to follow:
+                    1. Check if relevant factual data regarding <USER QUERY> is present in the <SCRAPED DATA>.
+                       - IF YES, extract the maximum relevant factual information related to <USER QUERY> from the <SCRAPED DATA>.
+                       - IF NO, then return "N/A"
+                    
+                    Rules to follow:
+                    - Return N/A if information is not present in the scraped data.
+                    - FORGET EVERYTHING YOU KNOW, Only output information that is present in the scraped data, DO NOT MAKE UP INFORMATION
+                """
 SysPromptDefault = "You are an expert AI, complete the given task. Do not add any additional comments."
 SysPromptSearch = """You are a search query generator, create a concise Google search query, focusing only on the main topic and omitting additional redundant details, include year if necessory, 2024, Do not add any additional comments. OUTPUT ONLY THE SEARCH QUERY
                         #Additional instructions:
@@ -152,11 +162,11 @@ def remove_stopwords(text):
 def rephrase_content(data_format, content, query):
 
     if data_format == "Structured data":
-        return together_response(
-            f"return only the factual information regarding the query: {{{query}}}. Output should be concise chunks of \
-    paragraphs or tables or both, using the scraped context:{{{limit_tokens(content)}}}",
+        return together_response(f"""
+            <SCRAPED DATA>{content}</SCRAPED DATA>
+            extract the maximum relevant factual information covering all aspects of <USER QUERY>{query}</USER QUERY> ONLY IF AVAILABLE in the scraped data.""",
             SysPrompt=SysPromptData,
-            max_tokens=500,
+            max_tokens=900,
         )
     elif data_format == "Quantitative data":
         return together_response(
@@ -171,42 +181,44 @@ def rephrase_content(data_format, content, query):
             max_tokens=500,
         )
         
-class Scraper:
-    def __init__(self, user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"):
-        self.session = requests.Session()
-        self.session.headers.update({"User-Agent": user_agent})
-
-    @retry(tries=3, delay=1)
-    def fetch_content(self, url):
+def extract_main_content(url):
+    if url:
         try:
-            response = self.session.get(url, timeout=2)
-            if response.status_code == 200:
-                return response.text
-        except requests.exceptions.RequestException as e:
-            print(f"Error fetching page content for {url}: {e}")
-        return None
-
-def extract_main_content(html):
-    if html:
-        plain_text = ""
-        soup = BeautifulSoup(html, 'lxml')
-        for element in soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'table']):
-            plain_text += element.get_text(separator=" ", strip=True) + "\n"
-        return plain_text
+            result = urlparse(url)
+            if all([result.scheme, result.netloc]):
+                # Prepare query parameters
+                params = {
+                    "url": url,
+                    "favor_precision": False,
+                    "favor_recall": False,
+                    "output_format": "markdown",
+                    "target_language": "en",
+                    "include_tables": True,
+                    "include_images": False,
+                    "include_links": False,
+                    "deduplicate": True,
+                }
+
+                # Make request to FastAPI endpoint
+                response = requests.get("https://pvanand-web-scraping.hf.space/extract-article", params=params)
+
+                if response.status_code == 200:
+                    return response.json()["article"]
+                else:
+                    return ""
+        except:
+            return ""
     return ""
 
 def process_content(data_format, url, query):
-    scraper = Scraper()
-    html_content = scraper.fetch_content(url)
-    if html_content:
-        content = extract_main_content(html_content)
-        if content:
-            rephrased_content = rephrase_content(
-                data_format=data_format,
-                content=limit_tokens(remove_stopwords(content), token_limit=1000),
-                query=query,
-            )
-            return rephrased_content, url
+    content = extract_main_content(url)
+    if content:
+        rephrased_content = rephrase_content(
+            data_format=data_format,
+            content=limit_tokens(content, token_limit=4000),
+            query=query,
+        )
+        return rephrased_content, url
     return "", url
 
 def fetch_and_extract_content(data_format, urls, query):