fix generation bugs

backend-cli.py

@@ -94,7 +94,7 @@ def request_to_result_name(request: EvalRequest) -> str:
 
 def process_evaluation(task: Task, eval_request: EvalRequest) -> dict:
     batch_size = "auto"
-
+    
     try:
         results = run_evaluation(eval_request=eval_request, task_names=[task.benchmark], num_fewshot=task.num_fewshot,
                                  batch_size=batch_size, device=DEVICE, use_cache=None, limit=LIMIT)
@@ -266,6 +266,20 @@ def process_pending_requests() -> bool:
 if __name__ == "__main__":
     wait = True
     hard_task_lst = None
+    local_debug = True
+    #debug specific task by ping
+    if local_debug:
+        debug_model_names = ['TinyLlama/TinyLlama-1.1B-Chat-v0.6']
+        # debug_task_name = 'ifeval'
+        debug_task_name = 'selfcheckgpt'
+        task_lst = TASKS_HARNESS.copy()
+        for task in task_lst:
+            for debug_model_name in debug_model_names:
+                task_name = task.benchmark
+                if task_name != debug_task_name:
+                    continue
+                eval_request = EvalRequest(model=debug_model_name, private=False, status='', json_filepath='', precision='float16')
+                results = process_evaluation(task, eval_request)
 
     if socket.gethostname() in {'hamburg', 'neuromancer'} or os.path.isdir("/home/pminervi"):
         wait = False

src/backend/manage_requests.py

@@ -24,7 +24,6 @@ class EvalRequest:
     likes: Optional[int] = 0
     params: Optional[int] = None
     license: Optional[str] = ""
-    
     def get_model_args(self) -> str:
         model_args = f"pretrained={self.model},revision={self.revision},parallelize=True" # ,max_length=4096"
 
@@ -41,7 +40,6 @@ class EvalRequest:
             pass
         else:
             raise Exception(f"Unknown precision {self.precision}.")
-        
         return model_args
 
 

src/backend/run_eval_suite.py

@@ -11,6 +11,7 @@ from src.backend.tasks.cnndm.task_v2 import CNNDMv2
 
 from src.backend.tasks.selfcheckgpt.task import SelfCheckGPT
 
+from src.backend.huggingface_generate_until import HFLMwithChatTemplate
 
 def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_size, device, use_cache=None, limit=None, max_nb_samples=100) -> dict:
     if limit:
@@ -32,7 +33,7 @@ def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_siz
     print(f"Selected Tasks: {task_names}")
     print(f"Eval Request: {eval_request.get_model_args()}")
 
-    results = evaluator.simple_evaluate(model="hf-auto",  # "hf-causal-experimental",  # "hf-causal"
+    results = evaluator.simple_evaluate(model="hf-chat",  # "hf-causal-experimental",  # "hf-causal"
                                         model_args=eval_request.get_model_args(),
                                         tasks=task_names,
                                         num_fewshot=num_fewshot,

src/backend/tasks/selfcheckgpt/task.py

@@ -22,7 +22,7 @@ class SelfCheckGPT(ConfigurableTask):
     def __init__(self):
         super().__init__(config={'metadata': {'version': self.VERSION}})
         # these end tokens are hard coded because of the current limitaion of the llm-eval.
-        self.generation_kwargs = {"until": ["\n\n", "<unk>", "<|im_end|>", "</s>"], "max_length": 512}
+        self.generation_kwargs = {"until": ["\n\n", "<unk>", "<|im_end|>", "</s>", "<|endoftext|>"], "max_length": 512}
         self.generation_kwargs_sampling_number = 5  # the number of sampling for self-consistence
         self.generation_kwargs_sampling = {"temperature": 0.99, "do_sample": True, "until": ["\n\n", "<unk>", "<|im_end|>", "</s>"], "max_length": 512}
 

src/display/utils.py

@@ -45,7 +45,7 @@ class Tasks(Enum):
     halueval_dial = Task("halueval_dialogue", "acc", "HaluDial/Acc")
 
     # XXX include me back at some point
-    # selfcheck = Task("selfcheckgpt", "max-selfcheckgpt", "SelfCheckGPT")
+    selfcheck = Task("selfcheckgpt", "max-selfcheckgpt", "SelfCheckGPT")
 
 
 # These classes are for user facing column names,
@@ -157,7 +157,7 @@ class Precision(Enum):
         if precision in ["GPTQ", "None"]:
             return Precision.qt_GPTQ
         return Precision.Unknown
-        
+
 
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]

src/leaderboard/read_evals.py

@@ -24,7 +24,7 @@ class EvalResult:
     # Also see src.display.utils.AutoEvalColumn for what will be displayed.
     eval_name: str  # org_model_precision (uid)
     full_model: str  # org/model (path on hub)
-    org: str 
+    org: str
     model: str
     revision: str  # commit hash, "" if main
     results: dict