update

src/backend/run_eval_suite.py

@@ -9,7 +9,7 @@ from src.backend.tasks.xsum.task_v2 import XSumv2
 from src.backend.tasks.cnndm.task import CNNDM
 from src.backend.tasks.cnndm.task_v2 import CNNDMv2
 
-from src.backend.tasks.selfcheckgpt.task import SelfCheckGpt
+from src.backend.tasks.selfcheckgpt.task import SelfCheckGPT
 
 
 def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_size, device, use_cache=None, limit=None, max_nb_samples=100) -> dict:

src/backend/tasks/__init__.py

@@ -4,4 +4,4 @@ from src.backend.tasks.xsum.task_v2 import XSumv2
 from src.backend.tasks.cnndm.task import CNNDM
 from src.backend.tasks.cnndm.task_v2 import CNNDMv2
 
-from src.backend.tasks.selfcheckgpt.task import SelfCheckGpt
+from src.backend.tasks.selfcheckgpt.task import SelfCheckGPT

src/backend/tasks/cnndm/cnndm.yaml

@@ -0,0 +1,2 @@
+task: cnndm
+class: !function task.CNNDM

src/backend/tasks/cnndm/task.py

@@ -59,14 +59,14 @@ def rouge(refs, preds):
     return {type: result[type].mid.fmeasure * 100 for type in rouge_types}
 
 
-@register_task("cnndm")
+# @register_task("cnndm")
 class CNNDM(Task):
     VERSION = 0
     DATASET_PATH = "cnn_dailymail"
     DATASET_NAME = "3.0.0"
 
-    def __init__(self, data_dir=None, cache_dir=None, download_mode=None, config=None):
-        super().__init__(data_dir=data_dir, cache_dir=cache_dir, download_mode=download_mode, config=config)
+    def __init__(self):
+        super().__init__(config={'metadata': {'version': self.VERSION}})
         self.factkb_tokenizer = None
         self.factkb_model = None
         self.bert_score = None

src/backend/tasks/cnndm/task_v2.py

@@ -59,14 +59,14 @@ def rouge(refs, preds):
     return {type: result[type].mid.fmeasure * 100 for type in rouge_types}
 
 
-@register_task("cnndm_v2")
+# @register_task("cnndm_v2")
 class CNNDMv2(Task):
     VERSION = 0
     DATASET_PATH = "cnn_dailymail"
     DATASET_NAME = "3.0.0"
 
-    def __init__(self, data_dir=None, cache_dir=None, download_mode=None, config=None):
-        super().__init__(data_dir=data_dir, cache_dir=cache_dir, download_mode=download_mode, config=config)
+    def __init__(self):
+        super().__init__(config={'metadata': {'version': self.VERSION}})
         self.factkb_tokenizer = None
         self.factkb_model = None
         self.bert_score = None

src/backend/tasks/cnndm/xsum_v2.yaml

@@ -0,0 +1,2 @@
+task: cnndm_v2
+class: !function task_v2.CNNDM_v2

src/backend/tasks/selfcheckgpt/selfcheckgpt.yaml

@@ -0,0 +1,2 @@
+task: selfcheckgpt
+class: !function task.SelfCheckGPT

src/backend/tasks/selfcheckgpt/task.py

@@ -12,15 +12,15 @@ import spacy
 from selfcheckgpt.modeling_selfcheck import SelfCheckMQAG, SelfCheckNLI, SelfCheckBERTScore, SelfCheckNgram
 
 
-@register_task("selfcheckgpt")
-class SelfCheckGpt(Task):
+# @register_task("selfcheckgpt")
+class SelfCheckGPT(Task):
     VERSION = 0.0
     DATASET_PATH = "potsawee/wiki_bio_gpt3_hallucination"
     DATASET_NAME = None
     OUTPUT_TYPE = 'generate_until'
 
-    def __init__(self, data_dir=None, cache_dir=None, download_mode=None, config=None):
-        super().__init__(data_dir=data_dir, cache_dir=cache_dir, download_mode=download_mode, config=config)
+    def __init__(self):
+        super().__init__(config={'metadata': {'version': self.VERSION}})
         # these end tokens are hard coded because of the current limitaion of the llm-eval.
         self.generation_kwargs = {"until": ["\n\n", "<unk>", "<|im_end|>", "</s>"], "max_length": 512}
         self.generation_kwargs_sampling_number = 5  # the number of sampling for self-consistence

src/backend/tasks/xsum/task.py

@@ -1,6 +1,5 @@
-from lm_eval.api.task import Task
+from lm_eval.api.task import Task, ConfigurableTask
 from lm_eval.api.instance import Instance
-from lm_eval.api.registry import register_task
 from lm_eval.api.metrics import mean
 
 import torch
@@ -51,14 +50,14 @@ def rouge(refs, preds):
     return {type: result[type].mid.fmeasure * 100 for type in rouge_types}
 
 
-@register_task("xsum")
+# @register_task("xsum")
 class XSum(Task):
     VERSION = 0
     DATASET_PATH = "EdinburghNLP/xsum"
     DATASET_NAME = None
 
-    def __init__(self, data_dir=None, cache_dir=None, download_mode=None, config=None):
-        super().__init__(data_dir=data_dir, cache_dir=cache_dir, download_mode=download_mode, config=config)
+    def __init__(self):
+        super().__init__(config={'metadata': {'version': self.VERSION}})
         self.factkb_tokenizer = None
         self.factkb_model = None
         self.bert_score = None

src/backend/tasks/xsum/task_v2.py

@@ -1,4 +1,4 @@
-from lm_eval.api.task import Task
+from lm_eval.api.task import ConfigurableTask, Task, TaskConfig
 from lm_eval.api.instance import Instance
 from lm_eval.api.registry import register_task
 from lm_eval.api.metrics import mean
@@ -51,14 +51,14 @@ def rouge(refs, preds):
     return {type: result[type].mid.fmeasure * 100 for type in rouge_types}
 
 
-@register_task("xsum_v2")
-class XSumv2(Task):
+# @register_task("xsum_v2")
+class XSumv2(ConfigurableTask):
     VERSION = 0
     DATASET_PATH = "EdinburghNLP/xsum"
     DATASET_NAME = None
 
-    def __init__(self, data_dir=None, cache_dir=None, download_mode=None, config=None):
-        super().__init__(data_dir=data_dir, cache_dir=cache_dir, download_mode=download_mode, config=config)
+    def __init__(self):
+        super().__init__(config={'metadata': {'version': self.VERSION}})
         self.factkb_tokenizer = None
         self.factkb_model = None
         self.bert_score = None
@@ -92,15 +92,14 @@ class XSumv2(Task):
     def test_docs(self):
         return self.dataset["test"]
 
-    def prompt(self):
+    def custom_prompt(self):
         res = "Provide a summary of the provided document."
         return res
 
     def doc_to_text(self, doc):
-        return f'{self.prompt()}\n\nDocument: {doc["document"]}\nSummary:'
+        return f'{self.custom_prompt()}\n\nDocument: {doc["document"]}\nSummary:'
 
-    @staticmethod
-    def should_decontaminate():
+    def should_decontaminate(self):
         return True
 
     def doc_to_decontamination_query(self, doc):

src/backend/tasks/xsum/xsum.yaml

@@ -0,0 +1,2 @@
+task: xsum
+class: !function task.XSum

src/backend/tasks/xsum/xsum_v2.yaml

@@ -0,0 +1,2 @@
+task: xsum_v2
+class: !function task_v2.XSumv2