File size: 4,235 Bytes
68ed9a2 1613f96 6bf4f4e 68ed9a2 6bf4f4e 0f68941 1613f96 68ed9a2 1613f96 7592671 6bf4f4e 1613f96 6bf4f4e 1613f96 6bf4f4e 1613f96 4c0c41f 1613f96 6bf4f4e 1613f96 6bf4f4e 7592671 1613f96 7592671 1613f96 7592671 1613f96 4c0c41f 1613f96 6bf4f4e 1613f96 68ed9a2 1613f96 5a0ccc4 1613f96 7592671 68ed9a2 1613f96 4c0c41f 1613f96 6bf4f4e 1613f96 7592671 1613f96 68ed9a2 7592671 1613f96 7592671 6bf4f4e 4c0c41f 7592671 1613f96 4c0c41f 7592671 1613f96 7592671 1613f96 9d43bec 1613f96 4c0c41f 1613f96 7592671 6bf4f4e 1613f96 4c0c41f 1613f96 0f68941 1613f96 7592671 1613f96 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 |
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# flake8: noqa E501
import json
import os
from datetime import datetime, timezone
from src.display.formatting import styled_error, styled_message, styled_warning
from src.envs import API, EVAL_REQUESTS_PATH, REQUESTS_REPO, TOKEN
from src.submission.check_validity import already_submitted_models, check_model_card, get_model_size, is_model_on_hub
from src.utils import get_request_hash, get_request_id
from src.submission.notify import notify
REQUESTED_MODELS = None
USERS_TO_SUBMISSION_DATES = None
def add_new_eval(
model_name: str,
# base_model: str,
revision: str,
# precision: str,
# weight_type: str,
model_type: str,
):
# Remove if added to the form
weight_type = 'Original'
precision = 'Unknown'
global REQUESTED_MODELS
global USERS_TO_SUBMISSION_DATES
if not REQUESTED_MODELS:
REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
precision = precision.split(" ")[0]
current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
if model_type is None or model_type == "":
return styled_error("Please, select a model type.")
# Does the model actually exist?
if revision == "":
revision = "main"
# Is the model on the hub?
# if weight_type in ["Delta", "Adapter"]:
# base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
# if not base_model_on_hub:
# return styled_error(f'Base model "{base_model}" {error}')
if weight_type != "Adapter":
model_on_hub, error, _ = is_model_on_hub(model_name=model_name, revision=revision, token=TOKEN, test_tokenizer=True)
if not model_on_hub:
return styled_error(f'Model "{model_name}" {error}')
# Is the model info correctly filled?
try:
model_info = API.model_info(repo_id=model_name, revision=revision)
except Exception:
return styled_error("Could not get your model information. Please, fill it up properly.")
model_size = get_model_size(
model_info=model_info,
precision=precision,
)
# Were the model card and license filled?
try:
license_title = model_info.cardData["license"]
except Exception:
license_title = "Unknown"
# return styled_error("Please, select a license for your model.")
is_model_card_ok, error_msg = check_model_card(model_name)
if not is_model_card_ok:
return styled_error(error_msg)
# Seems good, creating the eval
print("Adding new evaluation request...")
eval_entry = {
# "model": model,
# "base_model": base_model,
"revision": revision,
"precision": precision,
"weight_type": weight_type,
"status": "PENDING",
"submitted_time": current_time,
"model_type": model_type.split()[1], # remove the emoji
# "likes": model_info.likes,
"params": model_size,
"license": license_title,
# "private": False,
}
# Check for duplicate submission
request_id = get_request_id(model_name, revision, precision)
if request_id in REQUESTED_MODELS:
return styled_warning('This model has already been submitted.')
request_hash = get_request_hash(model_name, revision, precision)
print("Creating evaluation request file...")
OUT_DIR = f"{EVAL_REQUESTS_PATH}/{model_name}"
os.makedirs(OUT_DIR, exist_ok=True)
out_path = f"{OUT_DIR}/{request_hash}.json"
if os.path.exists(out_path):
os.remove(out_path)
with open(out_path, 'w') as f:
json.dump(
eval_entry,
f,
indent=4,
)
print("Uploading evaluation file...")
API.upload_file(
path_or_fileobj=out_path,
path_in_repo='{}/{}.json'.format(model_name, request_hash),
repo_id=REQUESTS_REPO,
repo_type="dataset",
commit_message=f"Add an evaluation request for {model_name}",
)
notify(model_name)
# Remove the local file
os.remove(out_path)
return styled_message(
"Your model has been submitted."
)
|