File size: 3,757 Bytes
14e4843 d6d7ec6 14e4843 d6d7ec6 14e4843 d6d7ec6 14e4843 d6d7ec6 14e4843 d6d7ec6 14e4843 d6d7ec6 14e4843 d6d7ec6 14e4843 d6d7ec6 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 |
import os
import json
import glob
from tqdm import tqdm
from huggingface_hub import HfApi, snapshot_download
from src.backend.manage_requests import EvalRequest
from src.backend.envs import EVAL_REQUESTS_PATH_BACKEND_SYNC
from src.envs import QUEUE_REPO, API
from src.envs import EVAL_REQUESTS_PATH_OPEN_LLM, QUEUE_REPO_OPEN_LLM
from src.utils import my_snapshot_download
def my_set_eval_request(api, json_filepath, hf_repo, local_dir):
for i in range(10):
try:
set_eval_request(api=api, json_filepath=json_filepath, hf_repo=hf_repo, local_dir=local_dir)
return
except Exception:
time.sleep(60)
return
def set_eval_request(api: HfApi, json_filepath: str, hf_repo: str, local_dir: str):
"""Updates a given eval request with its new status on the hub (running, completed, failed, ...)"""
with open(json_filepath) as fp:
data = json.load(fp)
with open(json_filepath, "w") as f:
f.write(json.dumps(data))
api.upload_file(
path_or_fileobj=json_filepath,
path_in_repo=json_filepath.replace(local_dir, ""),
repo_id=hf_repo,
repo_type="dataset",
)
def get_request_file_for_model(data, requests_path):
model_name = data["model"]
precision = data["precision"]
"""Selects the correct request file for a given model. Only keeps runs tagged as FINISHED and RUNNING"""
request_files = os.path.join(
requests_path,
f"{model_name}_eval_request_*.json",
)
request_files = glob.glob(request_files)
# Select correct request file (precision)
request_file = ""
request_files = sorted(request_files, reverse=True)
for tmp_request_file in request_files:
with open(tmp_request_file, "r") as f:
req_content = json.load(f)
if req_content["precision"] == precision.split(".")[-1]:
request_file = tmp_request_file
return request_file
def update_model_type(data, requests_path):
open_llm_request_file = get_request_file_for_model(data, requests_path)
try:
with open(open_llm_request_file, "r") as f:
open_llm_request = json.load(f)
data["model_type"] = open_llm_request["model_type"]
return True, data
except:
return False, data
def read_and_write_json_files(directory, requests_path_open_llm):
# Walk through the directory
for subdir, dirs, files in tqdm(os.walk(directory), desc="updating model type according to open llm leaderboard"):
for file in files:
# Check if the file is a JSON file
if file.endswith(".json"):
file_path = os.path.join(subdir, file)
# Open and read the JSON file
with open(file_path, "r") as json_file:
data = json.load(json_file)
sucess, data = update_model_type(data, requests_path_open_llm)
if sucess:
with open(file_path, "w") as json_file:
json.dump(data, json_file)
my_set_eval_request(
api=API, json_filepath=file_path, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND_SYNC
)
if __name__ == "__main__":
my_snapshot_download(
repo_id=QUEUE_REPO_OPEN_LLM,
revision="main",
local_dir=EVAL_REQUESTS_PATH_OPEN_LLM,
repo_type="dataset",
max_workers=60,
)
my_snapshot_download(
repo_id=QUEUE_REPO,
revision="main",
local_dir=EVAL_REQUESTS_PATH_BACKEND_SYNC,
repo_type="dataset",
max_workers=60,
)
read_and_write_json_files(EVAL_REQUESTS_PATH_BACKEND_SYNC, EVAL_REQUESTS_PATH_OPEN_LLM)
|