Adding TOKEN when calling snapshot_download, to handle private directories.

scripts/create_request_file.py

@@ -7,6 +7,7 @@ from datetime import datetime, timezone
 import click
 from colorama import Fore
 from huggingface_hub import HfApi, snapshot_download
+from src.envs import TOKEN
 
 EVAL_REQUESTS_PATH = "eval-queue"
 QUEUE_REPO = "open-llm-leaderboard/requests"
@@ -36,7 +37,7 @@ def get_model_size(model_info, precision: str):
 def main():
     api = HfApi()
     current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-    snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH, repo_type="dataset")
+    snapshot_download(repo_id=QUEUE_REPO, revision="main", token=TOKEN, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset")
 
     model_name = click.prompt("Enter model name")
     revision = click.prompt("Enter revision", default="main")

src/backend/manage_requests.py

@@ -71,7 +71,7 @@ def get_eval_requests(job_status: list, local_dir: str, hf_repo: str) -> list[Ev
     Returns:
         `list[EvalRequest]`: a list of model info dicts.
     """
-    snapshot_download(repo_id=hf_repo, revision="main", local_dir=local_dir, repo_type="dataset", max_workers=60)
+    snapshot_download(repo_id=hf_repo, revision="main", token=TOKEN, local_dir=local_dir, repo_type="dataset", max_workers=60)
     json_files = glob.glob(f"{local_dir}/**/*.json", recursive=True)
 
     eval_requests = []
@@ -97,7 +97,7 @@ def check_completed_evals(
     local_dir_results: str,
 ):
     """Checks if the currently running evals are completed, if yes, update their status on the hub."""
-    snapshot_download(repo_id=hf_repo_results, revision="main", local_dir=local_dir_results, repo_type="dataset", max_workers=60)
+    snapshot_download(repo_id=hf_repo_results, revision="main", token=TOKEN, local_dir=local_dir_results, repo_type="dataset", max_workers=60)
 
     running_evals = get_eval_requests(checked_status, hf_repo=hf_repo, local_dir=local_dir)