File size: 2,899 Bytes
14e4843 d6d7ec6 14e4843 d6d7ec6 14e4843 d6d7ec6 14e4843 d6d7ec6 14e4843 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 |
#!/usr/bin/env python
from huggingface_hub import snapshot_download
from src.leaderboard.read_evals import get_raw_eval_results
from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, RESULTS_REPO
from src.backend.run_eval_suite import run_evaluation
from src.backend.manage_requests import check_completed_evals, get_eval_requests, set_eval_request
from src.backend.sort_queue import sort_models_by_priority
from src.backend.envs import Tasks, EVAL_REQUESTS_PATH_BACKEND, EVAL_RESULTS_PATH_BACKEND, DEVICE, LIMIT, Task
from src.leaderboard.read_evals import get_raw_eval_results
from src.backend.manage_requests import EvalRequest
from src.leaderboard.read_evals import EvalResult
snapshot_download(
repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
)
snapshot_download(
repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
)
PENDING_STATUS = "PENDING"
RUNNING_STATUS = "RUNNING"
FINISHED_STATUS = "FINISHED"
FAILED_STATUS = "FAILED"
TASKS_HARNESS = [task.value for task in Tasks]
current_finished_status = [FINISHED_STATUS]
def request_to_result_name(request: EvalRequest) -> str:
org_and_model = request.model.split("/", 1)
if len(org_and_model) == 1:
model = org_and_model[0]
res = f"{model}_{request.precision}"
else:
org = org_and_model[0]
model = org_and_model[1]
res = f"{org}_{model}_{request.precision}"
return res
# Get all eval request that are FINISHED, if you want to run other evals, change this parameter
eval_requests: list[EvalRequest] = get_eval_requests(
job_status=current_finished_status, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND
)
# Sort the evals by priority (first submitted first run)
eval_requests: list[EvalRequest] = sort_models_by_priority(api=API, models=eval_requests)
eval_results: list[EvalResult] = get_raw_eval_results(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH)
result_name_to_request = {request_to_result_name(r): r for r in eval_requests}
result_name_to_result = {r.eval_name: r for r in eval_results}
print("Requests", sorted(result_name_to_request.keys()))
print("Results", sorted(result_name_to_result.keys()))
for eval_request in eval_requests:
result_name: str = request_to_result_name(eval_request)
# Check the corresponding result
eval_result: EvalResult = result_name_to_result[result_name]
# Iterate over tasks and, if we do not have results for a task, run the relevant evaluations
for task in TASKS_HARNESS:
task_name = task.benchmark
if task_name not in eval_result.results:
print("RUN THIS ONE!", result_name, task_name)
raw_data = get_raw_eval_results(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH)
all_data_json = [v.to_dict() for v in raw_data if v.is_complete()]
breakpoint()
|