update

.gitignore

@@ -1,2 +1,4 @@
 __pycache__
-tlem.ju.py
+*.ju.py
+tests
+

pyproject.toml

@@ -0,0 +1,14 @@
+[tool.poetry]
+name = "tlem"
+version = "0.1.0"
+description = ""
+authors = ["fecet <xiezej@gmail.com>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

tasks.py

@@ -225,14 +225,11 @@ class Metrics:
             return {"error": "predictions and references have different " "length"}
         responses = [general_postprocess(pred) for pred in responses]
         processed_answers = [[general_postprocess(j) for j in i] for i in answers]
-        matched_answers = []
-        for pred, ans, origin_ans in zip(responses, processed_answers, answers):
-            if pred in ans or pred in origin_ans:
-                matched_answers.append(pred)
-            else:
-                matched_answers.append(ans[0])
-
-        return responses, matched_answers
+        scores = []
+        for pred, ans in zip(responses, processed_answers):
+            score = np.mean([1 if a in pred else 0 for a in ans])
+            scores.append(score)
+        return {"em": np.mean(scores)}
 
     def bbh_mcq(responses: list[str], answers: list[str | int]):
         if len(responses) != len(answers):
@@ -624,8 +621,6 @@ Text: [PROMPT]
 Question: [QUESTION]
 Anawer:"""
 
-    categories = ["validation"]
-
     @classmethod
     def prompt_drop(cls, example):
         prompt = cls.icl_prompt.replace("[PROMPT]", example["passage"]).replace(
@@ -633,19 +628,7 @@ Anawer:"""
         )
 
         validated_answers = example["answers_spans"]["spans"]
-        validated_types = example["answers_spans"]["types"]
-        answers = []
-        for answer_item, answer_type in zip(validated_answers, validated_types):
-            # if answer_type == "number":
-            #     answers.append(answer_item)
-            # elif any(answer_item['date'][i] for i in ['day', 'month', 'year']):
-            #     d = [answer_item['date'][i] for i in ['day', 'month', 'year']]
-            #     answers.append(' '.join(d).strip())
-            # else:
-            #     for span in answer_item['spans']:
-            # answers.append(span)
-            answers.append(answer_item)
-        answers = list(set(answers))
+        answers = list(set(validated_answers))
 
         return {cls.input_column: prompt, cls.label_column: answers}
 

tlem.py

@@ -49,8 +49,7 @@ class ReasoningMetric(evaluate.Metric):
     def _compute(self, responses, references):
         return_value = getattr(Metrics, self.config_name)(responses, references)
         match return_value:
-            case tuple():
-                extract_responses, extract_references = return_value
+            case extract_responses, extract_references:
                 results = {
                     self.config_name: np.mean(
                         sync_pipe(lambda x, y: x == y)(
@@ -182,8 +181,7 @@ class Suite(EvaluationSuite):
     def singleton(self, task):
         try:
             return self.tasks[self.tasks.index(task)]
-
-        except Exception as e:
+        except ValueError:
             self.tasks.append(task)
             return self.tasks[-1]