Update app.py

app.py

@@ -24,6 +24,7 @@ logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %
 
 load_dotenv()
 logging.info("Environment variables loaded")
+logger = logging.getLogger(__name__)
 
 # Initialize Cohere client
 APIFY_API_TOKEN = os.environ.get('APIFY_API_TOKEN')
@@ -31,10 +32,13 @@ COHERE_API_KEY = os.environ["COHERE_API_KEY"]
 co = cohere.Client(COHERE_API_KEY)
 logging.info("Cohere client initialized")
 if not APIFY_API_TOKEN:
+    logger.error("APIFY_API_TOKEN is not set in the environment variables.")
     st.error("APIFY_API_TOKEN is not set in the environment variables. Please set it and restart the application.")
 
 # Initialize the ApifyClient with the API token
 client = ApifyClient(APIFY_API_TOKEN)
+# Initialize the ApifyClient with the API token
+logger.info("ApifyClient initialized")
 
 # Configuration: Set to True if running locally, False if running on Streamlit Cloud
 IS_LOCAL = False
@@ -91,7 +95,9 @@ def init_session_state():
 # -------------
 
 def get_serp_results(query):
+    logger.info(f"Getting SERP results for query: {query}")
     if not APIFY_API_TOKEN:
+        logger.error("Apify API token is not set. Unable to fetch SERP results.")
         st.error("Apify API token is not set. Unable to fetch SERP results.")
         return []
 
@@ -108,56 +114,89 @@ def get_serp_results(query):
     }
 
     try:
+        logger.debug(f"Calling Apify Actor with input: {run_input}")
         # Run the Actor and wait for it to finish
         run = client.actor("nFJndFXA5zjCTuudP").call(run_input=run_input)
+        logger.info(f"Apify Actor run completed. Run ID: {run.get('id')}")
         
         # Fetch results from the run's dataset
+        logger.debug(f"Fetching results from dataset ID: {run.get('defaultDatasetId')}")
         results = list(client.dataset(run["defaultDatasetId"]).iterate_items())
+        logger.info(f"Fetched {len(results)} results from Apify dataset")
         
         if results and 'organicResults' in results[0]:
-            return [item['url'] for item in results[0]['organicResults']]
+            urls = [item['url'] for item in results[0]['organicResults']]
+            logger.info(f"Extracted {len(urls)} URLs from organic results")
+            return urls
         else:
+            logger.warning("No organic results found in the SERP data.")
             st.warning("No organic results found in the SERP data.")
             return []
     except Exception as e:
+        logger.exception(f"Error fetching SERP results: {str(e)}")
         st.error(f"Error fetching SERP results: {str(e)}")
         return []
 
+
+
+        
 def fetch_content(url):
+    logger.info(f"Fetching content from URL: {url}")
     try:
         response = requests.get(url)
         response.raise_for_status()
         soup = BeautifulSoup(response.text, 'html.parser')
-        return soup.get_text(separator=' ', strip=True)
+        content = soup.get_text(separator=' ', strip=True)
+        logger.debug(f"Fetched {len(content)} characters from {url}")
+        return content
     except requests.RequestException as e:
+        logger.error(f"Error fetching content from {url}: {e}")
         st.warning(f"Error fetching content from {url}: {e}")
         return ""
 
-def generate_embeddings(text_list, model_type):
-    logging.debug(f"Generating embeddings for model type: {model_type}")
-    if not text_list:
-        logging.warning("Text list is empty, returning empty embeddings")
-        return []
-    model = 'embed-english-v3.0' if model_type == 'english' else 'embed-multilingual-v3.0'
-    input_type = 'search_document'
-    response = co.embed(model=model, texts=text_list, input_type=input_type)
-    embeddings = response.embeddings
-    logging.debug(f"Embeddings generated successfully for model type: {model_type}")
-    return embeddings
-
 def calculate_relevance_score(page_content, query, co):
-    page_embedding = co.embed(texts=[page_content], model='embed-english-v3.0', input_type='search_document').embeddings[0]
-    query_embedding = co.embed(texts=[query], model='embed-english-v3.0', input_type='search_query').embeddings[0]
-    return cosine_similarity([query_embedding], [page_embedding])[0][0]
+    logger.info(f"Calculating relevance score for query: {query}")
+    try:
+        page_embedding = co.embed(texts=[page_content], model='embed-english-v3.0', input_type='search_document').embeddings[0]
+        query_embedding = co.embed(texts=[query], model='embed-english-v3.0', input_type='search_query').embeddings[0]
+        score = cosine_similarity([query_embedding], [page_embedding])[0][0]
+        logger.debug(f"Relevance score calculated: {score}")
+        return score
+    except Exception as e:
+        logger.exception(f"Error calculating relevance score: {str(e)}")
+        st.error(f"Error calculating relevance score: {str(e)}")
+        return 0
+
+def analyze_competitors(row, co):
+    logger.info(f"Analyzing competitors for query: {row['query']}")
+    query = row['query']
+    our_url = row['page']
+    our_score = row['relevancy_score']
+    
+    competitor_urls = get_serp_results(query)
+    
+    results = []
+    for url in competitor_urls:
+        content = fetch_content(url)
+        score = calculate_relevance_score(content, query, co)
+        results.append({'url': url, 'relevancy_score': score})
+    
+    results.append({'url': our_url, 'relevancy_score': our_score})
+    results_df = pd.DataFrame(results).sort_values('relevancy_score', ascending=False)
+    
+    logger.info(f"Competitor analysis completed. {len(results)} results obtained.")
+    return results_df
 
 def show_competitor_analysis(row, co):
     if st.button("Check Competitors", key=f"comp_{row['page']}"):
+        logger.info(f"Competitor analysis requested for page: {row['page']}")
         with st.spinner('Analyzing competitors...'):
             results_df = analyze_competitors(row, co)
             st.write("Relevancy Score Comparison:")
             st.dataframe(results_df)
             
             our_rank = results_df.index[results_df['url'] == row['page']].tolist()[0] + 1
+            logger.info(f"Our page ranks {our_rank} out of {len(results_df)} in terms of relevancy score.")
             st.write(f"Our page ranks {our_rank} out of {len(results_df)} in terms of relevancy score.")
 
 
@@ -523,9 +562,11 @@ def main():
                 st.write("Data fetched successfully. Click the button below to calculate relevancy scores.")
                 
                 if st.button("Calculate Relevancy Scores"):
+                    logger.info("Calculating relevancy scores for all rows")
                     st.session_state.report_data = calculate_relevancy_scores(st.session_state.report_data, model_type)
                 
                 for index, row in st.session_state.report_data.iterrows():
+                    logger.debug(f"Processing row {index}: Query: {row['query']}, Page: {row['page']}")
                     st.write(f"Query: {row['query']}")
                     st.write(f"Page: {row['page']}")
                     st.write(f"Relevancy Score: {row['relevancy_score']:.4f}")
@@ -534,10 +575,11 @@ def main():
                 
                 download_csv_link(st.session_state.report_data)
             elif st.session_state.report_data is not None:
+                logger.warning("No data found for the selected criteria.")
                 st.warning("No data found for the selected criteria.")
-                logging.warning("No data found for the selected criteria")
 
 
 if __name__ == "__main__":
     logging.info("Running main function")
-    main()
+    main()
+    logger.info("Script completed")