update

backend-cli.py

@@ -112,13 +112,14 @@ def process_finished_requests() -> bool:
         result_name: str = request_to_result_name(eval_request)
 
         # Check the corresponding result
-        eval_result: EvalResult = result_name_to_result[result_name]
+        from typing import Optional
+        eval_result: Optional[EvalResult] = result_name_to_result[result_name] if result_name in result_name_to_result else None
 
         # Iterate over tasks and, if we do not have results for a task, run the relevant evaluations
         for task in TASKS_HARNESS:
             task_name = task.benchmark
 
-            if task_name not in eval_result.results:
+            if eval_result is None or task_name not in eval_result.results:
                 eval_request: EvalRequest = result_name_to_request[result_name]
 
                 set_eval_request(api=API, eval_request=eval_request, set_to_status=RUNNING_STATUS, hf_repo=QUEUE_REPO,

src/leaderboard/read_evals.py

@@ -103,16 +103,6 @@ class EvalResult:
             mean_acc = np.mean(accs) * 100.0
             results[task.benchmark] = mean_acc
 
-        # print(json_filepath, results)
-
-        # XXX
-        # if 'nq_open' not in results:
-        #     results['nq_open'] = 0.0
-
-        # XXX
-        # if 'triviaqa' not in results:
-        #     results['triviaqa'] = 0.0
-
         return EvalResult(eval_name=result_key, full_model=full_model, org=org, model=model, results=results,
                           precision=precision, revision=config.get("model_sha", ""), still_on_hub=still_on_hub,
                           architecture=architecture)