update

backend-cli.py

@@ -7,7 +7,7 @@ import random
 from datetime import datetime
 
 from src.backend.run_eval_suite import run_evaluation
-from src.backend.manage_requests import check_completed_evals, get_eval_requests, set_eval_request
+from src.backend.manage_requests import check_completed_evals, get_eval_requests
 from src.backend.sort_queue import sort_models_by_priority
 from src.backend.envs import Tasks, EVAL_REQUESTS_PATH_BACKEND, EVAL_RESULTS_PATH_BACKEND, DEVICE, LIMIT, Task
 
@@ -15,7 +15,7 @@ from src.backend.manage_requests import EvalRequest
 from src.leaderboard.read_evals import EvalResult
 
 from src.envs import QUEUE_REPO, RESULTS_REPO, API
-from src.utils import my_snapshot_download
+from src.utils import my_snapshot_download, my_set_eval_request
 
 import logging
 import pprint
@@ -136,14 +136,12 @@ def process_finished_requests(thr: int) -> bool:
                     eval_request: EvalRequest = result_name_to_request[result_name]
 
                     my_snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
-                    set_eval_request(api=API, eval_request=eval_request, set_to_status=RUNNING_STATUS, hf_repo=QUEUE_REPO,
-                                     local_dir=EVAL_REQUESTS_PATH_BACKEND)
+                    my_set_eval_request(api=API, eval_request=eval_request, set_to_status=RUNNING_STATUS, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND)
 
                     results = process_evaluation(task, eval_request)
 
                     my_snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
-                    set_eval_request(api=API, eval_request=eval_request, set_to_status=FINISHED_STATUS, hf_repo=QUEUE_REPO,
-                                     local_dir=EVAL_REQUESTS_PATH_BACKEND)
+                    my_set_eval_request(api=API, eval_request=eval_request, set_to_status=FINISHED_STATUS, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND)
 
                     return True
 
@@ -171,8 +169,7 @@ def process_pending_requests() -> bool:
     pp.pprint(eval_request)
 
     my_snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
-    set_eval_request(api=API, eval_request=eval_request, set_to_status=RUNNING_STATUS, hf_repo=QUEUE_REPO,
-                     local_dir=EVAL_REQUESTS_PATH_BACKEND)
+    my_set_eval_request(api=API, eval_request=eval_request, set_to_status=RUNNING_STATUS, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND)
 
     task_lst = TASKS_HARNESS.copy()
     random.shuffle(task_lst)
@@ -181,8 +178,7 @@ def process_pending_requests() -> bool:
         results = process_evaluation(task, eval_request)
 
     my_snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
-    set_eval_request(api=API, eval_request=eval_request, set_to_status=FINISHED_STATUS, hf_repo=QUEUE_REPO,
-                     local_dir=EVAL_REQUESTS_PATH_BACKEND)
+    my_set_eval_request(api=API, eval_request=eval_request, set_to_status=FINISHED_STATUS, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND)
 
     return True
 

src/backend/envs.py

@@ -20,6 +20,9 @@ class Tasks(Enum):
     task0 = Task("nq_open", "em", "NQ Open", 64)  # 64, as in the ATLAS paper
     task1 = Task("triviaqa", "em", "TriviaQA", 64)  # 64, as in the ATLAS paper
 
+    task11 = Task("nq8", "em", "NQ Open 8", 8)
+    task12 = Task("tqa8", "em", "TriviaQA 8", 8)
+
     # TruthfulQA is intended as a zero-shot benchmark [5, 47]. https://owainevans.github.io/pdfs/truthfulQA_lin_evans.pdf
     task2 = Task("truthfulqa_gen", "rougeL_acc", "TruthfulQA Gen", 0)
     task3 = Task("truthfulqa_mc1", "acc", "TruthfulQA MC1", 0)
@@ -38,9 +41,6 @@ class Tasks(Enum):
     task10 = Task("memo-trap", "acc", "memo-trap", 0)
     task10_2 = Task("memo-trap_v2", "acc", "memo-trap", 0)
 
-    task11 = Task("nq8", "em", "NQ Open 8", 8)
-    task12 = Task("tqa8", "em", "TriviaQA 8", 8)
-
     task13 = Task("ifeval", "prompt_level_strict_acc", "IFEval", 0)
 
     task14 = Task("selfcheckgpt", "max-selfcheckgpt", "SelfCheckGPT", 0)

src/backend/tasks/nq8/nq8.yaml

@@ -3,7 +3,7 @@ dataset_path: nq_open
 output_type: generate_until
 training_split: train
 validation_split: validation
-description: "Answer these questions:\n"
+description: "Answer these questions:\n\n"
 doc_to_text: "Q: {{question}}?\nA:"
 doc_to_target: "{{answer}}" # TODO: should be multi-target
 fewshot_delimiter: "\n"
@@ -27,6 +27,6 @@ metric_list:
     ignore_case: true
     ignore_punctuation: true
     regexes_to_ignore:
-    - "\ban|a|the\b"
+    - "\\b(?:The |the |An |A |The |a |an )"
 metadata:
   - version: 0.0

src/utils.py

@@ -1,4 +1,6 @@
 from huggingface_hub import snapshot_download
+from src.backend.manage_requests import set_eval_request
+import time
 
 
 def my_snapshot_download(repo_id, revision, local_dir, repo_type, max_workers):
@@ -7,6 +9,15 @@ def my_snapshot_download(repo_id, revision, local_dir, repo_type, max_workers):
             snapshot_download(repo_id=repo_id, revision=revision, local_dir=local_dir, repo_type=repo_type, max_workers=max_workers)
             return
         except Exception:
-            import time
+            time.sleep(60)
+    return
+
+
+def my_set_eval_request(api, eval_request, set_to_status, hf_repo, local_dir):
+    for i in range(10):
+        try:
+            set_eval_request(api=api, eval_request=eval_request, set_to_status=set_to_status, hf_repo=hf_repo, local_dir=local_dir)
+            return
+        except Exception:
             time.sleep(60)
     return