updated with meg's suggestions + cleaned up a bit

app.py

@@ -1,5 +1,8 @@
 import logging
+from apscheduler.schedulers.background import BackgroundScheduler
+
 from src.logging import configure_root_logger
+
 logging.getLogger("httpx").setLevel(logging.WARNING)
 logging.getLogger("numexpr").setLevel(logging.WARNING)
 logging.getLogger("absl").setLevel(logging.WARNING)
@@ -36,8 +39,8 @@ links_md = f"""
 | Results Repo    | [{RESULTS_REPO}](https://huggingface.co/datasets/{RESULTS_REPO}) |
 """
 
-def button_auto_eval():
-    logger.info("Manually triggering Auto Eval")
+def auto_eval():
+    logger.info("Triggering Auto Eval")
     run_auto_eval()
 
 
@@ -55,10 +58,14 @@ with gr.Blocks(js=dark_mode_gradio_js) as demo:
         button = gr.Button("Manually Run Evaluation")
         gr.Markdown(links_md)
 
-        dummy = gr.Markdown(run_auto_eval, every=REFRESH_RATE, visible=False)
-
-        button.click(fn=button_auto_eval, inputs=[], outputs=[])
+        #dummy = gr.Markdown(auto_eval, every=REFRESH_RATE, visible=False)
 
+        button.click(fn=auto_eval, inputs=[], outputs=[])
 
 if __name__ == '__main__':
-    demo.queue(default_concurrency_limit=40).launch(server_name="0.0.0.0", show_error=True, server_port=7860)
+    scheduler = BackgroundScheduler()
+    scheduler.add_job(auto_eval, "interval", seconds=REFRESH_RATE)
+    scheduler.start()
+    demo.queue(default_concurrency_limit=40).launch(server_name="0.0.0.0",
+                                                          show_error=True,
+                                                          server_port=7860)

main_backend_harness.py

@@ -6,7 +6,7 @@ from huggingface_hub import snapshot_download
 logging.getLogger("openai").setLevel(logging.WARNING)
 
 from src.backend.run_eval_suite_harness import run_evaluation
-from src.backend.manage_requests import check_completed_evals, get_eval_requests, set_eval_request
+from src.backend.manage_requests import check_completed_evals, get_eval_requests, set_eval_request, PENDING_STATUS, RUNNING_STATUS, FINISHED_STATUS, FAILED_STATUS
 from src.backend.sort_queue import sort_models_by_priority
 
 from src.envs import QUEUE_REPO, EVAL_REQUESTS_PATH_BACKEND, RESULTS_REPO, EVAL_RESULTS_PATH_BACKEND, DEVICE, API, LIMIT, TOKEN
@@ -19,10 +19,6 @@ from src.logging import setup_logger
 logger = setup_logger(__name__)
 pp = pprint.PrettyPrinter(width=80)
 
-PENDING_STATUS = "PENDING"
-RUNNING_STATUS = "RUNNING"
-FINISHED_STATUS = "FINISHED"
-FAILED_STATUS = "FAILED"
 
 snapshot_download(repo_id=RESULTS_REPO, revision="main", local_dir=EVAL_RESULTS_PATH_BACKEND, repo_type="dataset", max_workers=60, token=TOKEN)
 snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60, token=TOKEN)

main_backend_lighteval.py

@@ -6,7 +6,7 @@ from huggingface_hub import snapshot_download
 logging.getLogger("openai").setLevel(logging.WARNING)
 
 from src.backend.run_eval_suite_lighteval import run_evaluation
-from src.backend.manage_requests import check_completed_evals, get_eval_requests, set_eval_request
+from src.backend.manage_requests import check_completed_evals, get_eval_requests, set_eval_request, PENDING_STATUS, RUNNING_STATUS, FINISHED_STATUS, FAILED_STATUS
 from src.backend.sort_queue import sort_models_by_priority
 
 from src.envs import QUEUE_REPO, EVAL_REQUESTS_PATH_BACKEND, RESULTS_REPO, EVAL_RESULTS_PATH_BACKEND, API, LIMIT, TOKEN, ACCELERATOR, VENDOR, REGION, TASKS_LIGHTEVAL
@@ -17,11 +17,6 @@ logger = setup_logger(__name__)
 # logging.basicConfig(level=logging.ERROR)
 pp = pprint.PrettyPrinter(width=80)
 
-PENDING_STATUS = "PENDING"
-RUNNING_STATUS = "RUNNING"
-FINISHED_STATUS = "FINISHED"
-FAILED_STATUS = "FAILED"
-
 snapshot_download(repo_id=RESULTS_REPO, revision="main", local_dir=EVAL_RESULTS_PATH_BACKEND, repo_type="dataset", max_workers=60, token=TOKEN)
 snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60, token=TOKEN)
 

src/backend/manage_requests.py

@@ -9,6 +9,11 @@ from src.logging import setup_logger
 
 logger = setup_logger(__name__)
 
+PENDING_STATUS = "PENDING"
+RUNNING_STATUS = "RUNNING"
+FINISHED_STATUS = "FINISHED"
+FAILED_STATUS = "FAILED"
+
 @dataclass
 class EvalRequest:
     """This class represents one evaluation request file.
@@ -34,18 +39,10 @@ class EvalRequest:
         """
         model_args = f"pretrained={self.model},revision={self.revision}"
 
-        if self.precision in ["float16", "bfloat16", "float32"]:
+        if self.precision in ["float16", "bfloat16"]:
             model_args += f",dtype={self.precision}"
 
         # Quantized models need some added config, the install of bits and bytes, etc
-
-        #elif self.precision == "8bit":
-        #    model_args += ",load_in_8bit=True"
-        #elif self.precision == "4bit":
-        #    model_args += ",load_in_4bit=True"
-        #elif self.precision == "GPTQ":
-            # A GPTQ model does not need dtype to be specified,
-            # it will be inferred from the config
         else:
             raise Exception(f"Unknown precision {self.precision}.")
         
@@ -95,6 +92,16 @@ def get_eval_requests(job_status: list, local_dir: str, hf_repo: str) -> list[Ev
     return eval_requests
 
 
+def eval_was_running(eval_request: EvalRequest):
+    """Checks whether a file says it's RUNNING to determine whether to FAIL"""
+    json_filepath = eval_request.json_filepath
+
+    with open(json_filepath) as fp:
+        data = json.load(fp)
+
+    status = data["status"]
+    return status == RUNNING_STATUS
+
 def check_completed_evals(
     api: HfApi,
     hf_repo: str,
@@ -106,7 +113,14 @@ def check_completed_evals(
     local_dir_results: str,
 ):
     """Checks if the currently running evals are completed, if yes, update their status on the hub."""
-    snapshot_download(repo_id=hf_repo_results, revision="main", local_dir=local_dir_results, repo_type="dataset", max_workers=60, token=TOKEN)
+    snapshot_download(
+        repo_id=hf_repo_results, 
+        revision="main", 
+        local_dir=local_dir_results, 
+        repo_type="dataset", 
+        max_workers=60, 
+        token=TOKEN
+    )
 
     running_evals = get_eval_requests(checked_status, hf_repo=hf_repo, local_dir=local_dir)
 
@@ -125,7 +139,8 @@ def check_completed_evals(
             )
             set_eval_request(api, eval_request, completed_status, hf_repo, local_dir)
         else:
-            logger.info(
-                f"No result file found for {model} setting it to {failed_status}"
-            )
-            set_eval_request(api, eval_request, failed_status, hf_repo, local_dir)
+            if eval_was_running(eval_request=eval_request):
+                logger.info(
+                    f"No result file found for {model} setting it to {failed_status}"
+                )
+                set_eval_request(api, eval_request, failed_status, hf_repo, local_dir)