update

halueval-cli.py

@@ -28,13 +28,15 @@ def main():
     eval_requests: list[EvalRequest] = get_eval_requests(job_status=status, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND)
     eval_request = [r for r in eval_requests if 'bloom-560m' in r.model][0]
 
-    # task_names = ['halueval_qa']
+    TASKS_HARNESS = [t.value for t in Tasks if 'halueval_qa' in t.value.benchmark]
     # task_names = ['triviaqa']
-    TASKS_HARNESS = [task.value for task in Tasks]
+    # TASKS_HARNESS = [task.value for task in Tasks]
 
     include_task_folder("src/backend/tasks/")
     initialize_tasks('INFO')
 
+    # breakpoint()
+
     print(tasks.ALL_TASKS)
 
     for task in TASKS_HARNESS:
@@ -43,7 +45,7 @@ def main():
                                             batch_size=1, device=DEVICE, use_cache=None, limit=8, write_out=True)
         print('AAA', results["results"])
 
-        # breakpoint()
+        breakpoint()
 
 
 if __name__ == "__main__":

src/backend/run_eval_suite.py

@@ -8,7 +8,7 @@ import logging
 logging.getLogger("openai").setLevel(logging.WARNING)
 
 
-def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_size, device, use_cache=None, limit=None) -> dict:
+def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_size, device, use_cache=None, limit=None, max_nb_samples=100) -> dict:
     if limit:
         print("WARNING: --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.")
 
@@ -29,6 +29,13 @@ def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_siz
     results["config"]["model_name"] = eval_request.model
     results["config"]["model_sha"] = eval_request.revision
 
+    if max_nb_samples is not None:
+        if 'samples' in results:
+            samples = results['samples']
+            for task_name in samples.keys():
+                if len(samples['task_name']) > max_nb_samples:
+                    results['samples'][task_name] = results['samples'][task_name][:max_nb_samples]
+
     print(evaluator.make_table(results))
 
     return results