update app

app.py

@@ -4,7 +4,6 @@ from sentence_transformers import SentenceTransformer
 from transformers import pipeline
 from typing import List, Tuple
 import os
-from dotenv import load_dotenv
 import logging
 import json
 import gradio as gr
@@ -14,13 +13,27 @@ import pandas as pd
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
-load_dotenv()  # This loads the variables from .env
-
 # Initialize Cohere client, SentenceTransformer model, and QA pipeline
-co = cohere.Client(api_key = os.environ.get("COHERE_API_KEY"))
+co = cohere.Client(api_key=os.environ.get("COHERE_API_KEY"))
 sentence_model = SentenceTransformer('all-MiniLM-L6-v2')
 qa_pipeline = pipeline("question-answering", model="distilbert-base-cased-distilled-squad")
 
+# Define sample inputs
+samples = [
+    {
+        "context": "Albert Einstein is an Austrian scientist, who has completed his higher education in ETH Zurich in Zurich, Switzerland. He was later a faculty at Princeton University.",
+        "answer": "Switzerland"
+    },
+    {
+        "context": "The Eiffel Tower, located in Paris, France, is one of the most famous landmarks in the world. It was constructed in 1889 as the entrance arch to the 1889 World's Fair. The tower is 324 meters (1,063 ft) tall and is the tallest structure in Paris.",
+        "answer": "Paris"
+    },
+    {
+        "context": "The Great Wall of China is a series of fortifications and walls built across the historical northern borders of ancient Chinese states and Imperial China to protect against nomadic invasions. It is the largest man-made structure in the world, with a total length of more than 13,000 miles (21,000 kilometers).",
+        "answer": "China"
+    }
+]
+
 def generate_questions(context: str, answer: str) -> List[str]:
     try:
         response = co.chat(
@@ -93,7 +106,13 @@ def check_answer_precision(context: str, questions: List[str], original_answer:
     return precision_scores, generated_answers
 
 def calculate_composite_scores(sd_scores: List[float], sr_scores: List[float], ap_scores: List[float]) -> List[float]:
-    return [0.2 * sd + 0.4 * sr + 0.4 * ap for sd, sr, ap in zip(sd_scores, sr_scores, ap_scores)]
+    # Normalize other scores based on answer precision
+    max_other_score = max(max(sd_scores), max(sr_scores))
+    normalized_sd_scores = [sd * (ap / max_other_score) for sd, ap in zip(sd_scores, ap_scores)]
+    normalized_sr_scores = [sr * (ap / max_other_score) for sr, ap in zip(sr_scores, ap_scores)]
+    
+    # Calculate composite scores with higher weight for answer precision
+    return [0.6 * ap + 0.2 * sd + 0.2 * sr for ap, sd, sr in zip(ap_scores, normalized_sd_scores, normalized_sr_scores)]
 
 def rank_questions_with_details(context: str, answer: str) -> Tuple[pd.DataFrame, List[pd.DataFrame], str]:
     questions = generate_questions(context, answer)
@@ -107,16 +126,16 @@ def rank_questions_with_details(context: str, answer: str) -> Tuple[pd.DataFrame
     # Create detailed scores dataframe
     detailed_scores = pd.DataFrame({
         'Question': questions,
+        'Answer Precision': ap_scores,
         'Composite Score': composite_scores,
         'Structural Diversity': sd_scores,
         'Semantic Relevance': sr_scores,
-        'Answer Precision': ap_scores,
         'Generated Answer': generated_answers
     })
-    detailed_scores = detailed_scores.sort_values('Composite Score', ascending=False).reset_index(drop=True)
+    detailed_scores = detailed_scores.sort_values('Answer Precision', ascending=False).reset_index(drop=True)
     
     # Create separate ranking dataframes for each metric
-    metrics = ['Composite Score', 'Structural Diversity', 'Semantic Relevance', 'Answer Precision']
+    metrics = ['Answer Precision', 'Composite Score', 'Structural Diversity', 'Semantic Relevance']
     rankings = []
     
     for metric in metrics:
@@ -125,36 +144,22 @@ def rank_questions_with_details(context: str, answer: str) -> Tuple[pd.DataFrame
             'Question': [questions[i] for i in np.argsort(detailed_scores[metric])[::-1]],
             f'{metric}': sorted(detailed_scores[metric], reverse=True)
         })
+        if metric == 'Answer Precision':
+            df['Generated Answer'] = [generated_answers[i] for i in np.argsort(detailed_scores[metric])[::-1]]
         rankings.append(df)
     
     best_question = detailed_scores.iloc[0]['Question']
     
     return detailed_scores, rankings, best_question
 
-# Define sample inputs
-samples = [
-    {
-        "context": "Albert Einstein is an Austrian scientist, who has completed his higher education in ETH Zurich in Zurich, Switzerland. He was later a faculty at Princeton University.",
-        "answer": "Switzerland"
-    },
-    {
-        "context": "The Eiffel Tower, located in Paris, France, is one of the most famous landmarks in the world. It was constructed in 1889 as the entrance arch to the 1889 World's Fair. The tower is 324 meters (1,063 ft) tall and is the tallest structure in Paris.",
-        "answer": "Paris"
-    },
-    {
-        "context": "The Great Wall of China is a series of fortifications and walls built across the historical northern borders of ancient Chinese states and Imperial China to protect against nomadic invasions. It is the largest man-made structure in the world, with a total length of more than 13,000 miles (21,000 kilometers).",
-        "answer": "China"
-    }
-]
-
 def gradio_interface(context: str, answer: str) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame, str]:
     detailed_scores, rankings, best_question = rank_questions_with_details(context, answer)
     return (
         detailed_scores,
-        rankings[0],  # Composite Score Ranking
-        rankings[1],  # Structural Diversity Ranking
-        rankings[2],  # Semantic Relevance Ranking
-        rankings[3],  # Answer Precision Ranking
+        rankings[0],  # Answer Precision Ranking
+        rankings[1],  # Composite Score Ranking
+        rankings[2],  # Structural Diversity Ranking
+        rankings[3],  # Semantic Relevance Ranking
         f"Best Question: {best_question}"
     )
 
@@ -181,25 +186,25 @@ with gr.Blocks(theme=gr.themes.Default()) as iface:
     
     with gr.Row():
         with gr.Column():
-            composite_ranking_output = gr.DataFrame(label="Composite Score Ranking")
+            answer_precision_ranking_output = gr.DataFrame(label="Answer Precision Ranking")
         with gr.Column():
-            structural_diversity_ranking_output = gr.DataFrame(label="Structural Diversity Ranking")
+            composite_ranking_output = gr.DataFrame(label="Composite Score Ranking")
     
     with gr.Row():
         with gr.Column():
-            semantic_relevance_ranking_output = gr.DataFrame(label="Semantic Relevance Ranking")
+            structural_diversity_ranking_output = gr.DataFrame(label="Structural Diversity Ranking")
         with gr.Column():
-            answer_precision_ranking_output = gr.DataFrame(label="Answer Precision Ranking")
+            semantic_relevance_ranking_output = gr.DataFrame(label="Semantic Relevance Ranking")
 
     submit_button.click(
         fn=gradio_interface,
         inputs=[context_input, answer_input],
         outputs=[
             detailed_scores_output,
+            answer_precision_ranking_output,
             composite_ranking_output,
             structural_diversity_ranking_output,
             semantic_relevance_ranking_output,
-            answer_precision_ranking_output,
             best_question_output
         ]
     )
@@ -211,5 +216,5 @@ with gr.Blocks(theme=gr.themes.Default()) as iface:
             outputs=[context_input, answer_input]
         )
 
-
+# Launch the app
 iface.launch()