update

src/backend/envs.py

@@ -51,8 +51,9 @@ class Tasks(Enum):
     task17 = Task("truefalse_cieacf", "acc", "TrueFalse", 8)
 
     task18 = Task("faithdial_hallu", "acc", "FaithDial", 8)
+    task19 = Task("faithdial_hallu_v2", "acc", "FaithDial", 8)
 
-    task19 = Task("race", "acc", "RACE", 0)
+    task20 = Task("race", "acc", "RACE", 0)
 
 # NUM_FEWSHOT = 64  # Change with your few shot
 

src/backend/tasks/faithdial/faithdial_v2.yaml

@@ -0,0 +1,15 @@
+group: faithdial
+task: faithdial_hallu_v2
+dataset_path: McGill-NLP/FaithDial
+training_split: train
+validation_split: validation
+test_split: test
+output_type: multiple_choice
+doc_to_text: !function utils.doc_to_text_v2
+doc_to_target: !function utils.doc_to_target
+doc_to_choice: ["false", "true"]
+metric_list:
+  - metric: acc
+    higher_is_better: True
+metadata:
+  version: 0.0

src/backend/tasks/faithdial/utils.py

@@ -5,12 +5,15 @@ ValueType = Union[str, List[str]]
 def doc_to_text(doc: dict[str, ValueType]) -> str:
     history_str = " ".join([f'[{"Human" if i % 2 == 0 else "Assistant"}] {m}' for i, m in enumerate(doc['history'])])
     doc_text = f'#Knowledge#: {doc["knowledge"]}\n#Dialogue History#: {history_str}\n#Response#: {doc["response"]}\n#Hallucinated#:'
-    # breakpoint()
+    return doc_text
+
+
+def doc_to_text_v2(doc: dict[str, ValueType]) -> str:
+    history_str = " ".join([f'[{"Human" if i % 2 == 0 else "Assistant"}] {m}' for i, m in enumerate(doc['history'])])
+    doc_text = f'#Knowledge#: {doc["knowledge"]}\n#Dialogue History#: {history_str}\n#Response#: {doc["original_response"]}\n#Hallucinated#:'
     return doc_text
 
 
 def doc_to_target(doc: dict[str, ValueType]) -> str:
     res = "true" if "Hallucination" in doc["BEGIN"] else "false"
-    # breakpoint()
     return res
-