update

src/backend/envs.py

@@ -17,13 +17,10 @@ class Task:
 
 
 class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
-    # task0 = Task("anli_r1", "acc", "ANLI")
-    # task1 = Task("logiqa", "acc_norm", "LogiQA")
     task0 = Task("nq_open", "em", "NQ Open", 64)  # 64, as in the ATLAS paper
     task1 = Task("triviaqa", "em", "TriviaQA", 64)  # 64, as in the ATLAS paper
-    # TruthfulQA is intended as a zero-shot benchmark [5, 47]. https://owainevans.github.io/pdfs/truthfulQA_lin_evans.pdf
 
+    # TruthfulQA is intended as a zero-shot benchmark [5, 47]. https://owainevans.github.io/pdfs/truthfulQA_lin_evans.pdf
     task2 = Task("truthfulqa_gen", "rougeL_acc", "TruthfulQA Gen", 0)
     task3 = Task("truthfulqa_mc1", "acc", "TruthfulQA MC1", 0)
     task4 = Task("truthfulqa_mc2", "acc", "TruthfulQA MC2", 0)

src/backend/tasks/selfcheckgpt/task.py

@@ -18,6 +18,7 @@ class SelfCheckGpt(Task):
     DATASET_PATH = "potsawee/wiki_bio_gpt3_hallucination"
     DATASET_NAME = None
     OUTPUT_TYPE = 'generate_until'
+
     def __init__(self, data_dir=None, cache_dir=None, download_mode=None, config=None):
         super().__init__(data_dir=data_dir, cache_dir=cache_dir, download_mode=download_mode, config=config)
         self.generation_kwargs = {"temperature": 0.0, "do_sample": False}
@@ -54,7 +55,8 @@ class SelfCheckGpt(Task):
         doc_text = doc["wiki_bio_text"]
         doc_text = doc_text.split()
         doc_text = " ".join(doc_text[:5])
-        doc_text = f"Please generating a Wikipedia passage starting with: {doc_text}\n"
+        # prompt = f"This is a passage from Wikipedia about {context}:\n\n"
+        doc_text = f"Please generate a Wikipedia passage starting with: {doc_text}\n"
         return doc_text
 
     def doc_to_target(self, doc):
@@ -82,35 +84,32 @@ class SelfCheckGpt(Task):
         sentences = self.selfcheckgpt_nlp(response_temperature_0)
         sentences = [sent.text.strip() for sent in sentences.sents]
         if self.selfcheckgpt_type == 'SelfCheckNgram':
-            selfcheckgpt_scores = self.selfcheckgpt.predict(
-                sentences = sentences,
-                passage = response_temperature_0,
-                sampled_passages = other_responses,
-                )
-            return {'avg-selfcheckgpt': selfcheckgpt_scores['doc_level']['avg_neg_logprob'],
-                    'max-selfcheckgpt': selfcheckgpt_scores['doc_level']['avg_max_neg_logprob']}
+            selfcheckgpt_scores = self.selfcheckgpt.predict(sentences=sentences, passage=response_temperature_0, sampled_passages=other_responses)
+            return {
+                'avg-selfcheckgpt': selfcheckgpt_scores['doc_level']['avg_neg_logprob'],
+                'max-selfcheckgpt': selfcheckgpt_scores['doc_level']['avg_max_neg_logprob']
+            }
 
         elif self.selfcheckgpt_type == 'SelfCheckBERTScore':
             selfcheckgpt_scores = self.selfcheckgpt.predict(sentences=sentences, sampled_passages=other_responses)
         elif self.selfcheckgpt_type == 'SelfCheckMQAG':
             selfcheckgpt_scores = self.selfcheckgpt.predict(
-                sentences = sentences,
-                passage = response_temperature_0,
-                sampled_passages = other_responses,
-                num_questions_per_sent = 5,          # number of questions to be drawn
-                scoring_method = 'bayes_with_alpha', # options = 'counting', 'bayes', 'bayes_with_alpha'
-                beta1 = 0.8, beta2 = 0.8,            # additional params depending on scoring_method
-                )
+                sentences=sentences,
+                passage=response_temperature_0,
+                sampled_passages=other_responses,
+                num_questions_per_sent=5,          # number of questions to be drawn
+                scoring_method='bayes_with_alpha', # options = 'counting', 'bayes', 'bayes_with_alpha'
+                beta1=0.8, beta2=0.8)            # additional params depending on scoring_method
         elif self.selfcheckgpt_type == 'SelfCheckNLI':
-            selfcheckgpt_scores = self.selfcheckgpt.predict(
-                sentences = sentences,
-                sampled_passages = other_responses,
-                )
+            selfcheckgpt_scores = self.selfcheckgpt.predict(sentences=sentences, sampled_passages=other_responses)
             
             if len(selfcheckgpt_scores) == 0:
                 self.SelfCheckNLI_error_cnt += 1
                 print(f"SelfCheckNLI Warning.SelfCheckNLI_error_cnt:{self.SelfCheckNLI_error_cnt}. This instance is marked as hallucinated with 1.0.")
-                result = {'avg-selfcheckgpt': 1.0, 'max-selfcheckgpt': 1.0}
+                result = {
+                    'avg-selfcheckgpt': 1.0,
+                    'max-selfcheckgpt': 1.0
+                }
 
             else:
                 threshold = 0.5