open_pt_llm_leaderboard

Running on CPU Upgrade

open_pt_llm_leaderboard / upload_initial_queue.py

fix eval_name for non main revision models

4717ca8 29 days ago

No virus

1.57 kB

	import os
	import json
	from src.envs import API, EVAL_REQUESTS_PATH, DYNAMIC_INFO_REPO, DYNAMIC_INFO_FILE_PATH, DYNAMIC_INFO_PATH, EVAL_RESULTS_PATH, H4_TOKEN, IS_PUBLIC, QUEUE_REPO, REPO_ID, RESULTS_REPO
	from huggingface_hub import snapshot_download
	from src.submission.submit import add_new_eval

	snapshot_download(
	repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
	)

	with open('initial_queue.jsonl', 'r', encoding='utf-8') as outfile:
	for line in outfile:
	if not line.startswith('//'):
	data = json.loads(line)
	model_id = f'{data["model"]}_eval_request_False_{data["precision"]}_{data["weight_type"]}'
	if data["revision"] != "main":
	model_id = f'{data["model"]}_eval_request_{data["revision"]}_False_{data["precision"]}_{data["weight_type"]}'
	model_path = os.path.join(EVAL_REQUESTS_PATH, model_id + '.json')
	if not os.path.exists(model_path):
	print(f"Adding new eval {model_id}")
	message = add_new_eval(
	model=data["model"],
	base_model=data["base_model"],
	revision=data["revision"],
	precision=data["precision"],
	private=False,
	weight_type=data["weight_type"],
	model_type=data["model_type"],
	main_language=data["main_language"],
	source="manual"
	)
	print(message)