Update my_model/results/evaluation.py

my_model/results/evaluation.py

@@ -8,10 +8,38 @@ import streamlit as st
 from my_model.config import evaluation_config as config 
 
 class KBVQAEvaluator:
-    def __init__(self):
+    """
+    A class to evaluate Knowledge-Based Visual Question Answering (KB-VQA) models.
+    
+    This class provides methods for syntactic and semantic evaluation of the KB-VQA model,
+    using both exact match and VQA scores. The evaluation results can be saved to an 
+    Excel file for further analysis.
+
+    Attributes:
+        data_path (str): Path to the evaluation data.
+        use_fuzzy (bool): Flag to determine if fuzzy matching should be used.
+        stemmer (PorterStemmer): Instance of PorterStemmer for stemming answers.
+        scores_df (pd.DataFrame): DataFrame containing scores.
+        df (pd.DataFrame): Main DataFrame containing evaluation data.
+        vqa_scores (Dict[str, float]): Dictionary to store VQA scores for different model configurations.
+        exact_match_scores (Dict[str, float]): Dictionary to store exact match scores for different model configurations.
+        fuzzy_threshold (int): Threshold for fuzzy matching score.
+        openai_api_key (str): API key for OpenAI GPT-4.
+        model_names (List[str]): List of model names to be evaluated.
+        model_configurations (List[str]): List of model configurations to be evaluated.
+        gpt4_seed (int): Seed for GPT-4 evaluation.
+        gpt4_max_tokens (int): Maximum tokens for GPT-4 responses.
+        gpt4_temperature (float): Temperature setting for GPT-4 responses.
+    """
+    
+    def __init__(self): -> None
         """
-        Initialize the VQA Processor with the dataset and configuration settings.
+        Initialize the KBVQAEvaluator with the dataset and configuration settings.
+        
+        Reads data from the specified paths in the configuration and initializes
+        various attributes required for evaluation.
         """
+
         self.data_path = config.EVALUATION_DATA_PATH
         self.use_fuzzy = config.USE_FUZZY
         self.stemmer = PorterStemmer()
@@ -30,17 +58,32 @@ class KBVQAEvaluator:
     def stem_answers(self, answers: Union[str, List[str]]) -> Union[str, List[str]]:
         """
         Apply Porter Stemmer to either a single string or a list of strings.
+
+        Args:
+            answers (Union[str, List[str]]): A single answer string or a list of answer strings.
+
+        Returns:
+            Union[str, List[str]]: Stemmed version of the input string or list of strings.
         """
+        
         if isinstance(answers, list):
             return [" ".join(self.stemmer.stem(word.strip()) for word in answer.split()) for answer in answers]
         else:
             words = answers.split()
             return " ".join(self.stemmer.stem(word.strip()) for word in words)
 
-    def calculate_vqa_score(self, ground_truths, model_answer):
+   def calculate_vqa_score(self, ground_truths: List[str], model_answer: str) -> float:
         """
         Calculate VQA score based on the number of matching answers, with optional fuzzy matching.
+
+        Args:
+            ground_truths (List[str]): List of ground truth answers.
+            model_answer (str): Model's answer to be evaluated.
+
+        Returns:
+            float: VQA score based on the number of matches.
         """
+       
         if self.use_fuzzy:
             fuzzy_matches = sum(fuzz.partial_ratio(model_answer, gt) >= self.fuzzy_threshold for gt in ground_truths)
             return min(fuzzy_matches / 3, 1)
@@ -48,19 +91,29 @@ class KBVQAEvaluator:
             count = Counter(ground_truths)
             return min(count.get(model_answer, 0) / 3, 1)
 
-    def calculate_exact_match_score(self, ground_truths, model_answer):
+    def calculate_exact_match_score(self, ground_truths: List[str], model_answer: str) -> int:
         """
         Calculate Exact Match score, with optional fuzzy matching.
+
+        Args:
+            ground_truths (List[str]): List of ground truth answers.
+            model_answer (str): Model's answer to be evaluated.
+
+        Returns:
+            int: Exact match score (1 if there is a match, 0 otherwise).
         """
+        
         if self.use_fuzzy:
             return int(any(fuzz.partial_ratio(model_answer, gt) >= self.fuzzy_threshold for gt in ground_truths))
         else:
             return int(model_answer in ground_truths)
 
-    def syntactic_evaluation(self):
+    def syntactic_evaluation(self) -> None:
         """
         Process the DataFrame: stem answers, calculate scores, and store results.
+
         """
+        
         self.df['raw_answers_stemmed'] = self.df['raw_answers'].apply(literal_eval).apply(self.stem_answers)
         
         for name in self.model_names:
@@ -74,10 +127,19 @@ class KBVQAEvaluator:
                 self.vqa_scores[full_config] = round(self.df[f'vqa_score_{full_config}'].mean()*100, 2)
                 self.exact_match_scores[full_config] = round(self.df[f'exact_match_score_{full_config}'].mean()*100, 2)
 
-    def create_GPT4_messages_template(self, question, ground_truths, model_answer):
+    def create_GPT4_messages_template(self, question: str, ground_truths: List[str], model_answer: str) -> List[dict]:
         """
         Create a message list for the GPT-4 API call based on the question, ground truths, and model answer.
+
+        Args:
+            question (str): The question being evaluated.
+            ground_truths (List[str]): List of ground truth answers.
+            model_answer (str): Model's answer to be evaluated.
+
+        Returns:
+            List[dict]: Messages formatted for GPT-4 API call.
         """
+        
         system_message = {
             "role": "system",
             "content": """You are an AI trained to evaluate the equivalence of AI-generated answers to a set of ground truth answers for a given question. Upon reviewing a model's answer, determine if it matches the ground truths. Use the following rating system: 1 if you find that the model answer matches more than 25% of the ground truth answers, 2 if you find that the model answer matches only less than 25% of the ground truth answers, and 3 if the model answer is incorrect. Respond in the format below for easy parsing:
@@ -93,7 +155,7 @@ class KBVQAEvaluator:
         return [system_message, user_message]
         
 
-    def semantic_evaluation(self):
+    def semantic_evaluation(self) -> None:
         """
         Perform semantic evaluation using GPT-4 for each model configuration.
         """
@@ -109,7 +171,14 @@ class KBVQAEvaluator:
                     rating = int(evaluation.split('\n')[0].split(":")[1].strip())
                     self.df.at[index, f'gpt4_rating_{config}'] = rating    
                 
-    def save_results(self, save_filename):
+     def save_results(self, save_filename: str) -> None:
+        """
+        Save the evaluation results to an Excel file.
+
+        Args:
+            save_filename (str): The filename to save the results.
+        """
+         
         # Create a DataFrame for the scores
         scores_data = {
             'Model Configuration': list(self.vqa_scores.keys()),
@@ -123,10 +192,15 @@ class KBVQAEvaluator:
             self.df.to_excel(writer, sheet_name='Main Data', index=False)
             scores_df.to_excel(writer, sheet_name='Scores', index=False)
 
-def run_evaluation(save=False, save_filename="results"):
+def run_evaluation(save: bool = False, save_filename: str = "results") -> None:
     """
     Run the full evaluation process using KBVQAEvaluator and save the results to an Excel file.
+
+    Args:
+        save (bool): Whether to save the results to an Excel file. Defaults to False.
+        save_filename (str): The filename to save the results if save is True. Defaults to "results".
     """
+    
     # Instantiate the evaluator
     evaluator = KBVQAEvaluator()