normalize scores to majority class baseline

src/leaderboard/read_evals.py

@@ -160,13 +160,23 @@ class EvalResult:
 
         baselines = {task.value.benchmark: task.value.baseline*100 for task in Tasks}
 
-        average = sum([v for task, v in self.results.items() if v is not None and task in all_tasks]) / len(all_tasks)
-        average_g = sum([v for task, v in self.results.items() if v is not None and task in g_tasks]) / len(g_tasks)
-        average_mc = sum([v for task, v in self.results.items() if v is not None and task in mc_tasks]) / len(mc_tasks)
-
-        # average = sum([(v-baselines.get(task,0))/(100-baselines.get(task,0))*100 for task, v in self.results.items() if v is not None and task in all_tasks]) / len(all_tasks)
-        # average_g = sum([(v-baselines.get(task,0))/(100-baselines.get(task,0))*100 for task, v in self.results.items() if v is not None and task in g_tasks]) / len(g_tasks)
-        # average_mc = sum([(v-baselines.get(task,0))/(100-baselines.get(task,0))*100 for task, v in self.results.items() if v is not None and task in mc_tasks]) / len(mc_tasks)
+        # average = sum([v for task, v in self.results.items() if v is not None and task in all_tasks]) / len(all_tasks)
+        # average_g = sum([v for task, v in self.results.items() if v is not None and task in g_tasks]) / len(g_tasks)
+        # average_mc = sum([v for task, v in self.results.items() if v is not None and task in mc_tasks]) / len(mc_tasks)
+        # print('XXXXXXXXXXXX')
+        # print(self.eval_name)
+        # print(all_tasks)
+        # print(baselines)
+        # print(self.results)
+        # print('XXXXXXXXXXXX')
+
+        # average = sum([((v if v is not None else 0)-baselines.get(task,0))/(100-baselines.get(task,0))*100 for task, v in self.results.items() if  task in all_tasks]) / len(all_tasks)
+        # average_g = sum([((v if v is not None else 0)-baselines.get(task,0))/(100-baselines.get(task,0))*100 for task, v in self.results.items() if  task in g_tasks]) / len(g_tasks)
+        # average_mc = sum([((v if v is not None else 0)-baselines.get(task,0))/(100-baselines.get(task,0))*100 for task, v in self.results.items() if  task in mc_tasks]) / len(mc_tasks)
+
+        average = sum([(self.results.get(task,0) - baselines.get(task, 0)) / (100 - baselines.get(task, 0)) * 100 for task in all_tasks]) / len(all_tasks)
+        average_g = sum([(self.results.get(task,0) - baselines.get(task, 0)) / (100 - baselines.get(task, 0)) * 100 for task in g_tasks]) / len(g_tasks)
+        average_mc = sum([(self.results.get(task,0) - baselines.get(task, 0)) / (100 - baselines.get(task, 0)) * 100 for task in mc_tasks]) / len(mc_tasks)
 
         data_dict = {}
         # data_dict = {