Update selfcheckgpt

After we discuss with the author of selfcheckgpt Potasawee, we change the prompt of this evaluation for better faithful and factual evaluataion.

src/backend/tasks/selfcheckgpt/task.py

@@ -53,11 +53,15 @@ class SelfCheckGPT(ConfigurableTask):
         return self.dataset["evaluation"]
 
     def doc_to_text(self, doc):
-        doc_text = doc["wiki_bio_text"]
-        doc_text = doc_text.split()
-        doc_text = " ".join(doc_text[:5])
-        # prompt = f"This is a passage from Wikipedia about {context}:\n\n"
-        doc_text = f"Please generate a Wikipedia passage starting with: {doc_text}\n"
+        if not hasattr(self, 'selfcheckgpt_nlp'):
+            self.selfcheckgpt_nlp = spacy.load("en_core_web_sm")
+
+        sentences = [x.text.strip() for x in self.selfcheckgpt_nlp(doc['wiki_bio_text']).sents]
+        if len(sentences) < 2:
+            raise ValueError("This wikipedia passage is too short for self-consistency check: {sentences}")
+            # disscussed with Potsawee
+
+        doc_text = f"Please generate a Wikipedia passage that consists of at least two sentences, starting with the following sentence: {sentences[0]}\n"
         return doc_text
 
     def doc_to_target(self, doc):
@@ -104,21 +108,22 @@ class SelfCheckGPT(ConfigurableTask):
         elif self.selfcheckgpt_type == 'SelfCheckNLI':
             selfcheckgpt_scores = self.selfcheckgpt.predict(sentences=sentences, sampled_passages=other_responses)
 
-            if len(selfcheckgpt_scores) == 0:
+            if len(selfcheckgpt_scores) < 2:
+                # at least two sentences
                 self.SelfCheckNLI_error_cnt += 1
-                print(f"SelfCheckNLI Warning.SelfCheckNLI_error_cnt:{self.SelfCheckNLI_error_cnt}. This instance is marked as hallucinated with 0.0.")
                 result = {
                     'avg-selfcheckgpt': 0.0,
                     'max-selfcheckgpt': 0.0
                 }
 
             else:
-                threshold = 0.6 # https://huggingface.co/blog/dhuynh95/automatic-hallucination-detection
+                threshold = 0.7 # https://huggingface.co/blog/dhuynh95/automatic-hallucination-detection
                 # passage is hallucianted if one sentence is hallucinated. It's very strict.
                 selfcheckgpt_scores_max = 0.0 if max(selfcheckgpt_scores) > threshold else 1.0
                 # passage is hallucianted if average score of all sentences is hallucinated.
                 selfcheckgpt_scores_avg = 0.0 if sum(selfcheckgpt_scores) / len(selfcheckgpt_scores) > threshold else 1.0
                 result = {'avg-selfcheckgpt': selfcheckgpt_scores_avg, 'max-selfcheckgpt': selfcheckgpt_scores_max}
+
             return result
 
         selfcheckgpt_scores_avg = sum(selfcheckgpt_scores) / len(selfcheckgpt_scores) if len(selfcheckgpt_scores) > 0 else 0