feat: Use t-tests to determine win ratios

app.py

@@ -12,6 +12,8 @@ import requests
 import random
 import logging
 import datetime as dt
+import scipy.stats as stats
+import itertools as it
 
 
 fmt = "%(asctime)s [%(levelname)s] <%(name)s> %(message)s"
@@ -408,13 +410,19 @@ def produce_radial_plot(
             for language in languages:
                 if model_id not in results_dfs_filtered[language].index:
                     continue
-                score = results_dfs_filtered[language].loc[model_id][task]
+                score_list = results_dfs_filtered[language].loc[model_id][task]
                 win_ratio = 100 * np.mean([
-                    score >= other_score
-                    for other_score in results_dfs_filtered[language][task].dropna()
+                    stats.ttest_rel(
+                        a=score_list, b=other_scores, alternative="greater"
+                    ).pvalue < 0.05
+                    for other_scores in results_dfs_filtered[language][task].dropna().drop(index=model_id)
                 ])
                 win_ratios.append(win_ratio)
-                scores.append(score)
+
+                if all(score < 1 for score in score_list):
+                    score_list = [100 * score for score in score_list]
+
+                scores.append(np.mean(score_list))
             if use_win_ratio:
                 result_list.append(np.mean(win_ratios))
             else:
@@ -515,18 +523,23 @@ def fetch_results() -> dict[Language, pd.DataFrame]:
                 dataset = next(
                     dataset for dataset in DATASETS if dataset.name == dataset_name
                 )
-                results_dict = record['results']['total']
-                score = results_dict.get(
-                    f"test_{dataset.task.metric}", results_dict.get(dataset.task.metric)
-                )
+                scores = [
+                    test_score_dict.get(
+                        f"test_{dataset.task.metric}",
+                        test_score_dict.get(dataset.task.metric)
+                    )
+                    for test_score_dict in record["results"]["raw"]["test"]
+                ]
                 if dataset.task in data_dict[model_name]:
-                    data_dict[model_name][dataset.task].append(score)
+                    data_dict[model_name][dataset.task].append(scores)
                 else:
-                    data_dict[model_name][dataset.task] = [score]
+                    data_dict[model_name][dataset.task] = [scores]
         results_df = pd.DataFrame(data_dict).T.map(
-            lambda list_or_nan:
-            np.mean(list_or_nan) if list_or_nan == list_or_nan else list_or_nan
-        ).dropna()
+            lambda lists_or_nan:
+                list(it.chain(lists_or_nan))
+                if lists_or_nan == lists_or_nan
+                else lists_or_nan
+        ).dropna().map(lambda lst: lst[0])
         results_dfs[language] = results_df
 
     logger.info("Successfully fetched results from ScandEval benchmark.")

requirements.txt

@@ -52,6 +52,7 @@ requests==2.31.0
 rich==13.7.0
 rpds-py==0.17.1
 ruff==0.1.14
+scipy==1.12.0
 semantic-version==2.10.0
 shellingham==1.5.4
 six==1.16.0