update

cli/halueval-cli.py

@@ -28,7 +28,7 @@ def main():
     eval_requests: list[EvalRequest] = get_eval_requests(job_status=status, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND)
     eval_request = [r for r in eval_requests if 'bloom-560m' in r.model][0]
 
-    TASKS_HARNESS = [t.value for t in Tasks if 'halueval_qa' in t.value.benchmark]
+    TASKS_HARNESS = [t.value for t in Tasks if 'xsum' in t.value.benchmark]
     # task_names = ['triviaqa']
     # TASKS_HARNESS = [task.value for task in Tasks]
 
@@ -41,8 +41,8 @@ def main():
 
     for task in TASKS_HARNESS:
         print(f"Selected Tasks: [{task}]")
-        results = evaluator.simple_evaluate(model="hf", model_args=eval_request.get_model_args(), tasks=[task.benchmark], num_fewshot=0,
-                                            batch_size=1, device=DEVICE, use_cache=None, limit=8, write_out=True)
+        results = evaluator.simple_evaluate(model="hf", model_args=eval_request.get_model_args(), tasks=[task.benchmark], num_fewshot=1,
+                                            batch_size=1, device="mps", use_cache=None, limit=1, write_out=True)
         print('AAA', results["results"])
 
         breakpoint()