update

tasks.py

@@ -85,6 +85,9 @@ class Task:
             }
         self.label_column = self.label_column or self.input_column
 
+    def __eq__(self, __value: object) -> bool:
+        return self.name == __value.name
+
     @cached_property
     def samples(self):
         return self.dataset[self.input_column]

tlem.py

@@ -13,6 +13,7 @@ import pandas as pd
 from .tasks import *
 from .utils import *
 from itertools import chain
+from copy import deepcopy
 
 
 class ReasoningMetric(evaluate.Metric):
@@ -78,26 +79,29 @@ class Suite(EvaluationSuite):
             # case _:
             #     return list(chain(*self.suite.values()))[key]
 
+    def aggregate(self, suite):
+        for cate, tasks in suite.items():
+            if isinstance(tasks, dict):
+                suite[cate] = self.aggregate(tasks)
+            else:
+                result = []
+                for task in tasks:
+                    result.extend(task.result.values())
+                suite[cate] = np.mean(result)
+
+        return suite
+
     def run(
         self,
         model_or_pipeline: Any,
-        suite=None,
     ) -> dict[str, float]:
         self.assert_suite_nonempty()
-        if suite is None:
-            suite = self.suite
 
         self.suite: dict[str, list[Task]]
-        results = defaultdict(dict)
-        for category, tasks in (bar := tqdm(self.suite.items())):
-            bar.desc = f"complete {category}."
-            if isinstance(tasks, dict):
-                results[category] = self.run(model_or_pipeline, tasks)
-            else:
-                for task in tasks:
-                    results[category].update(task.run(model_or_pipeline))
-                results[category] = np.mean(list(results[category].values()))
-        return results
+        for task in (bar := tqdm(self.tasks)):
+            bar.desc = f"complete {task.name}."
+            _ = task.run(model_or_pipeline)
+        return self.aggregate(deepcopy(self.suite))
 
     def get_suite(self, name) -> dict[str, Task]:
         chat = False
@@ -144,6 +148,20 @@ class Suite(EvaluationSuite):
                     input_column="problem",
                     label_column="solution",
                 )
+
+            case "open-leaderboard":
+                suite = {}
+                for name in [
+                    "arc",
+                    "hellaswag",
+                    "mmlu-chat",
+                    "winogrande",
+                    "gsm8k",
+                    # "truthful_qa",
+                    "drop",
+                ]:
+                    suite[name] = self.get_suite(name)
+
         if isinstance(suite, Task):
             suite = [suite]
         if isinstance(suite, list):