pingnie commited on
Commit
7993384
1 Parent(s): b142b2c

add sync code in cli

Browse files
Files changed (2) hide show
  1. cli/sync-open-llm-cli.py +91 -0
  2. src/backend/envs.py +1 -0
cli/sync-open-llm-cli.py ADDED
@@ -0,0 +1,91 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import json
3
+ import glob
4
+
5
+ from tqdm import tqdm
6
+ from huggingface_hub import HfApi, snapshot_download
7
+ from src.backend.manage_requests import EvalRequest
8
+ from src.backend.envs import EVAL_REQUESTS_PATH_BACKEND_SYNC
9
+ from src.envs import QUEUE_REPO, API
10
+ from src.envs import EVAL_REQUESTS_PATH_OPEN_LLM, QUEUE_REPO_OPEN_LLM
11
+ from src.utils import my_snapshot_download
12
+
13
+ def my_set_eval_request(api, json_filepath, hf_repo, local_dir):
14
+ for i in range(10):
15
+ try:
16
+ set_eval_request(api=api, json_filepath=json_filepath, hf_repo=hf_repo, local_dir=local_dir)
17
+ return
18
+ except Exception:
19
+ time.sleep(60)
20
+ return
21
+
22
+
23
+ def set_eval_request(api: HfApi, json_filepath: str, hf_repo: str, local_dir: str):
24
+ """Updates a given eval request with its new status on the hub (running, completed, failed, ...)"""
25
+
26
+ with open(json_filepath) as fp:
27
+ data = json.load(fp)
28
+
29
+ with open(json_filepath, "w") as f:
30
+ f.write(json.dumps(data))
31
+
32
+ api.upload_file(path_or_fileobj=json_filepath, path_in_repo=json_filepath.replace(local_dir, ""),
33
+ repo_id=hf_repo, repo_type="dataset")
34
+
35
+
36
+ def get_request_file_for_model(data, requests_path):
37
+ model_name = data["model"]
38
+ precision = data["precision"]
39
+ """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED and RUNNING"""
40
+ request_files = os.path.join(
41
+ requests_path,
42
+ f"{model_name}_eval_request_*.json",
43
+ )
44
+ request_files = glob.glob(request_files)
45
+
46
+ # Select correct request file (precision)
47
+ request_file = ""
48
+ request_files = sorted(request_files, reverse=True)
49
+
50
+ for tmp_request_file in request_files:
51
+ with open(tmp_request_file, "r") as f:
52
+ req_content = json.load(f)
53
+ if req_content["precision"] == precision.split(".")[-1]:
54
+ request_file = tmp_request_file
55
+ return request_file
56
+
57
+ def update_model_type(data, requests_path):
58
+ open_llm_request_file = get_request_file_for_model(data, requests_path)
59
+
60
+ try:
61
+ with open(open_llm_request_file, "r") as f:
62
+ open_llm_request = json.load(f)
63
+ data["model_type"] = open_llm_request["model_type"]
64
+ return True, data
65
+ except:
66
+ return False, data
67
+
68
+
69
+ def read_and_write_json_files(directory, requests_path_open_llm):
70
+ # Walk through the directory
71
+ for subdir, dirs, files in tqdm(os.walk(directory), desc="updating model type according to open llm leaderboard"):
72
+ for file in files:
73
+ # Check if the file is a JSON file
74
+ if file.endswith('.json'):
75
+ file_path = os.path.join(subdir, file)
76
+ # Open and read the JSON file
77
+ with open(file_path, 'r') as json_file:
78
+ data = json.load(json_file)
79
+ sucess, data = update_model_type(data, requests_path_open_llm)
80
+ if sucess:
81
+ with open(file_path, 'w') as json_file:
82
+ json.dump(data, json_file)
83
+ my_set_eval_request(api=API, json_filepath=file_path, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND_SYNC)
84
+
85
+
86
+
87
+
88
+ if __name__ == "__main__":
89
+ my_snapshot_download(repo_id=QUEUE_REPO_OPEN_LLM, revision="main", local_dir=EVAL_REQUESTS_PATH_OPEN_LLM, repo_type="dataset", max_workers=60)
90
+ my_snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND_SYNC, repo_type="dataset", max_workers=60)
91
+ read_and_write_json_files(EVAL_REQUESTS_PATH_BACKEND_SYNC, EVAL_REQUESTS_PATH_OPEN_LLM)
src/backend/envs.py CHANGED
@@ -59,6 +59,7 @@ class Tasks(Enum):
59
 
60
 
61
  EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
 
62
  EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
63
 
64
  DEVICE = "cuda" if torch.cuda.is_available() else 'cpu'
 
59
 
60
 
61
  EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
62
+ EVAL_REQUESTS_PATH_BACKEND_SYNC = os.path.join(CACHE_PATH, "eval-queue-bk-sync")
63
  EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
64
 
65
  DEVICE = "cuda" if torch.cuda.is_available() else 'cpu'