File size: 4,053 Bytes
636ef92 db8f22f 636ef92 d6cf797 636ef92 e72d911 a73ed42 636ef92 e72d911 636ef92 2abf95b 3bcd977 044c2d1 2abf95b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 |
import pandas as pd
from fuzzywuzzy import fuzz
from collections import Counter
from nltk.stem import PorterStemmer
from ast import literal_eval
from typing import Union, List
import streamlit as st
class KBVQAEvaluator:
def __init__(self):
"""
Initialize the VQA Processor with the dataset and configuration settings.
"""
self.data_path = 'Files/evaluation_results_final.xlsx'
self.use_fuzzy = False
self.stemmer = PorterStemmer()
self.scores_df = pd.read_excel(self.data_path, sheet_name="Scores")
self.df = pd.read_excel(self.data_path, sheet_name="Main Data")
self.vqa_scores = {}
self.exact_match_scores = {}
def stem_answers(self, answers: Union[str, List[str]]) -> Union[str, List[str]]:
"""
Apply Porter Stemmer to either a single string or a list of strings.
"""
if isinstance(answers, list):
return [" ".join(self.stemmer.stem(word.strip()) for word in answer.split()) for answer in answers]
else:
words = answers.split()
return " ".join(self.stemmer.stem(word.strip()) for word in words)
def calculate_vqa_score(self, ground_truths, model_answer):
"""
Calculate VQA score based on the number of matching answers, with optional fuzzy matching.
"""
if self.use_fuzzy:
fuzzy_matches = sum(fuzz.partial_ratio(model_answer, gt) >= 80 for gt in ground_truths)
return min(fuzzy_matches / 3, 1)
else:
count = Counter(ground_truths)
return min(count.get(model_answer, 0) / 3, 1)
def calculate_exact_match_score(self, ground_truths, model_answer):
"""
Calculate Exact Match score, with optional fuzzy matching.
"""
if self.use_fuzzy:
return int(any(fuzz.partial_ratio(model_answer, gt) >= 80 for gt in ground_truths))
else:
return int(model_answer in ground_truths)
def evaluate(self):
"""
Process the DataFrame: stem answers, calculate scores, and store results.
"""
self.df['raw_answers_stemmed'] = self.df['raw_answers'].apply(literal_eval).apply(self.stem_answers)
model_configurations = ['caption+detic', 'caption+yolov5', 'only_caption', 'only_detic', 'only_yolov5']
model_names = ['13b', '7b']
for name in model_names:
for config in model_configurations:
full_config = f'{name}_{config}'
self.df[f'{full_config}_stemmed'] = self.df[full_config].apply(self.stem_answers)
self.df[f'vqa_score_{full_config}'] = self.df.apply(lambda x: self.calculate_vqa_score(x['raw_answers_stemmed'], x[f'{full_config}_stemmed']), axis=1)
self.df[f'exact_match_score_{full_config}'] = self.df.apply(lambda x: self.calculate_exact_match_score(x['raw_answers_stemmed'], x[f'{full_config}_stemmed']), axis=1)
self.vqa_scores[full_config] = round(self.df[f'vqa_score_{full_config}'].mean()*100, 2)
self.exact_match_scores[full_config] = round(self.df[f'exact_match_score_{full_config}'].mean()*100, 2)
def save_results(self):
# Create a DataFrame for the scores
scores_data = {
'Model Configuration': list(self.vqa_scores.keys()),
'VQA Score': list(self.vqa_scores.values()),
'Exact Match Score': list(self.exact_match_scores.values())
}
scores_df = pd.DataFrame(scores_data)
# Saving the scores DataFrame to an Excel file
with pd.ExcelWriter('evaluation_results_final.xlsx', engine='openpyxl', mode='w') as writer:
scores_df.to_excel(writer, sheet_name='Scores', index=False)
def run_evaluator(self):
#evaluator.evaluate()
st.table(self.scores_df)
st.dataframe(self.scores_df, width=900, height=500)
#print(evaluator.exact_match_scores)
#evaluator.save_results()
|