Update app.py

app.py

@@ -16,6 +16,9 @@ import requests
 from bs4 import BeautifulSoup
 from apify_client import ApifyClient
 import urllib.parse
+import openai
+from openai import OpenAI
+import re
 
 
 load_dotenv()
@@ -32,6 +35,12 @@ if not APIFY_API_TOKEN:
 client = ApifyClient(APIFY_API_TOKEN)
 # Initialize the ApifyClient with the API token
 
+# Initialize OpenAI client
+OPENAI_API_KEY = os.environ.get('OPENAI_API_KEY')
+if not OPENAI_API_KEY:
+    st.error("OPENAI_API_KEY is not set in the environment variables. Please set it and restart the application.")
+client = OpenAI(api_key=OPENAI_API_KEY)
+
 # Configuration: Set to True if running locally, False if running on Streamlit Cloud
 IS_LOCAL = False
 
@@ -128,9 +137,12 @@ def get_serp_results(query):
        # logger.info(f"Fetched {len(results)} results from Apify dataset")
         
         if results and 'organicResults' in results[0]:
-            urls = [item['url'] for item in results[0]['organicResults']]
-           # logger.info(f"Extracted {len(urls)} URLs from organic results")
-            return urls
+            serp_data = []
+            for item in results[0]['organicResults'][:5]:  # Limit to top 5 results
+                url = item['url']
+                content = fetch_content(url, query)
+                serp_data.append({'url': url, 'content': content})
+            return serp_data
         else:
            # logger.warning("No organic results found in the SERP data.")
             st.warning("No organic results found in the SERP data.")
@@ -143,14 +155,47 @@ def get_serp_results(query):
 
 
         
-def fetch_content(url):
+def extract_relevant_content(full_content, query):
+    try:
+        response = client.chat.completions.create(
+            model="gpt-4o-mini",
+            messages=[
+                {"role": "system", "content": "You are a helpful assistant that extracts the most relevant content from web pages."},
+                {"role": "user", "content": f"Given the following web page content and search query, extract only the most relevant parts of the content that answer or relate to the query.If there's no relevant content, say 'No relevant content found.'\n\nQuery: {query}\n\nContent: {full_content[:4000]}"}  # Limit input to 4000 characters
+            ],
+            max_tokens=3000  # Adjust as needed
+        )
+        return response.choices[0].message.content.strip()
+    except Exception as e:
+        st.error(f"Error in GPT content extraction: {str(e)}")
+        return "Error in content extraction"
+
+def fetch_content(url, query):
     try:
         decoded_url = urllib.parse.unquote(url)
         response = requests.get(decoded_url, timeout=10)
         response.raise_for_status()
         soup = BeautifulSoup(response.text, 'html.parser')
-        content = soup.get_text(separator=' ', strip=True)
-        return content
+        
+        # Remove unwanted elements
+        for unwanted in soup(['nav', 'header', 'footer', 'sidebar', 'menu', 'aside']):
+            unwanted.decompose()
+        
+        # Try to find the main content
+        main_content = soup.find('main') or soup.find('article') or soup.find('div', class_=re.compile('content|main|body'))
+        
+        if main_content:
+            content = main_content.get_text(separator=' ', strip=True)
+        else:
+            # Fallback to body if no main content is found
+            content = soup.body.get_text(separator=' ', strip=True)
+        
+        # Clean up the content
+        content = re.sub(r'\s+', ' ', content)  # Replace multiple spaces with single space
+        
+        # Use GPT to extract relevant content
+        relevant_content = extract_relevant_content(content, query)
+        return relevant_content
     except requests.RequestException:
         return ""
 
@@ -176,19 +221,21 @@ def analyze_competitors(row, co, custom_url=None):
     query = row['query']
     our_url = row['page']
     
-    competitor_urls = get_serp_results(query)
-    competitor_urls = [url for url in competitor_urls if not url.startswith('/search')][:5]  # Get top 5 valid competitors
+    competitor_data = get_serp_results(query)
     
-    if custom_url and custom_url not in competitor_urls:
-        competitor_urls.append(custom_url)
+    if custom_url and custom_url not in [data['url'] for data in competitor_data]:
+        custom_content = fetch_content(custom_url, query)
+        competitor_data.append({'url': custom_url, 'content': custom_content})
     
     results = []
-    for url in competitor_urls:
-        content = fetch_content(url)
-        score = calculate_relevance_score(content, query, co)
-        results.append({'url': url, 'relevancy_score': score})
+    for data in competitor_data:
+        score = calculate_relevance_score(data['content'], query, co)
+        results.append({'url': data['url'], 'relevancy_score': score})
+    
+    our_content = fetch_content(our_url, query)
+    our_score = calculate_relevance_score(our_content, query, co)
+    results.append({'url': our_url, 'relevancy_score': our_score})
     
-    results.append({'url': our_url, 'relevancy_score': row['relevancy_score']})
     results_df = pd.DataFrame(results).sort_values('relevancy_score', ascending=False)
     
     return results_df
@@ -445,7 +492,7 @@ def show_model_type_selector():
     )
 
 def calculate_single_relevancy(row):
-    page_content = fetch_content(row['page'])
+    page_content = fetch_content(row['page'], row['query'])
     query = row['query']
     score = calculate_relevance_score(page_content, query, co)
     return score