clean

tasks.py

@@ -14,6 +14,7 @@ from .utils import *
 from evaluate import load
 from collections import defaultdict
 import sys
+from pathlib import Path
 
 
 # if sys.version_info >= (3, 9):
@@ -73,7 +74,7 @@ class Task:
             if isinstance(self.dataset_name, str)
             else list(self.dataset_name)
         )
-        names[0] = names[0].split("/")[-1]
+        names[0] = Path(names[0]).name
 
         self.name = "-".join(names) + f"-{self.split}"
         if isinstance(self.prompt, str):
@@ -652,31 +653,15 @@ Anawer:"""
     def suite(
         cls,
     ):
-        finer_categories = (
-            pd.Series(cls.categories)  # noqa # type: ignore
-            .explode()
-            .reset_index()
-            .set_index(0)
-            .groupby(0)
-            .agg(list)["index"]
-            .to_dict()
+        return Task(
+            "drop",
+            metric_name=("sustech/tlem", "drop"),
+            input_column=cls.input_column,
+            label_column=cls.label_column,
+            prompt=partial(cls.prompt_drop),
+            few_shot=0,
+            split="validation",
         )
-        suite = defaultdict(list)
-        categories = list(finer_categories.keys())
-        for cate in categories:
-            suite[cate].append(
-                Task(
-                    ("drop", cate),
-                    metric_name=("sustech/tlem", "drop"),
-                    input_column=cls.input_column,
-                    label_column=cls.label_column,
-                    prompt=partial(cls.prompt_drop),
-                    few_shot=0,
-                    split="validation",
-                )
-            )
-
-        return suite
 
 
 class HellaSwag:

tlem.py

@@ -81,15 +81,12 @@ class Suite(EvaluationSuite):
             case slice() | int():
                 return self.tasks[key]
 
-    def aggregate(self, suite):
+    def agg(self, suite):
         for cate, tasks in suite.items():
             if isinstance(tasks, dict):
-                suite[cate] = self.aggregate(tasks)
+                suite[cate] = self.agg(tasks)
             else:
-                result = []
-                for task in tasks:
-                    result.extend(task.result.values())
-                suite[cate] = np.mean(result)
+                suite[cate] = np.mean([pd.Series(task.result).mean() for task in tasks])
 
         return suite
 
@@ -103,7 +100,8 @@ class Suite(EvaluationSuite):
         for task in (bar := tqdm(self.tasks)):
             bar.desc = f"complete {task.name}."
             _ = task.run(model_or_pipeline)
-        return self.aggregate(deepcopy(self.suite))
+            logging.info(f"{task.name} {task.result=}")
+        return self.agg(deepcopy(self.suite))
 
     def arun(self, model_or_pipeline):
         async def sync_function():
@@ -113,7 +111,7 @@ class Suite(EvaluationSuite):
 
         asyncio.run(sync_function())
 
-        return self.aggregate(deepcopy(self.suite))
+        return self.agg(deepcopy(self.suite))
 
     def get_suite(self, name) -> dict[str, Task]:
         chat = False