File size: 4,235 Bytes
68ed9a2
 
 
 
1613f96
 
 
 
 
6bf4f4e
68ed9a2
6bf4f4e
0f68941
1613f96
 
 
 
68ed9a2
1613f96
7592671
6bf4f4e
1613f96
6bf4f4e
 
1613f96
 
6bf4f4e
 
 
 
1613f96
 
 
 
 
 
 
 
 
4c0c41f
1613f96
 
 
 
 
 
6bf4f4e
 
 
 
1613f96
6bf4f4e
7592671
1613f96
7592671
1613f96
 
 
7592671
1613f96
4c0c41f
1613f96
6bf4f4e
 
 
 
1613f96
 
 
68ed9a2
1613f96
5a0ccc4
 
1613f96
7592671
68ed9a2
1613f96
 
 
4c0c41f
1613f96
 
6bf4f4e
 
1613f96
 
 
 
 
7592671
 
1613f96
68ed9a2
7592671
1613f96
 
 
7592671
6bf4f4e
4c0c41f
7592671
1613f96
4c0c41f
7592671
1613f96
7592671
 
 
 
1613f96
9d43bec
 
 
 
 
 
1613f96
4c0c41f
1613f96
 
7592671
6bf4f4e
1613f96
4c0c41f
1613f96
 
0f68941
 
1613f96
 
 
 
7592671
1613f96
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# flake8: noqa E501

import json
import os
from datetime import datetime, timezone

from src.display.formatting import styled_error, styled_message, styled_warning
from src.envs import API, EVAL_REQUESTS_PATH, REQUESTS_REPO, TOKEN
from src.submission.check_validity import already_submitted_models, check_model_card, get_model_size, is_model_on_hub
from src.utils import get_request_hash, get_request_id
from src.submission.notify import notify

REQUESTED_MODELS = None
USERS_TO_SUBMISSION_DATES = None


def add_new_eval(
    model_name: str,
    # base_model: str,
    revision: str,
    # precision: str,
    # weight_type: str,
    model_type: str,
):
    # Remove if added to the form
    weight_type = 'Original'
    precision = 'Unknown'

    global REQUESTED_MODELS
    global USERS_TO_SUBMISSION_DATES
    if not REQUESTED_MODELS:
        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)

    precision = precision.split(" ")[0]
    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")

    if model_type is None or model_type == "":
        return styled_error("Please, select a model type.")

    # Does the model actually exist?
    if revision == "":
        revision = "main"

    # Is the model on the hub?
    # if weight_type in ["Delta", "Adapter"]:
    #     base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
    #     if not base_model_on_hub:
    #         return styled_error(f'Base model "{base_model}" {error}')

    if weight_type != "Adapter":
        model_on_hub, error, _ = is_model_on_hub(model_name=model_name, revision=revision, token=TOKEN, test_tokenizer=True)
        if not model_on_hub:
            return styled_error(f'Model "{model_name}" {error}')

    # Is the model info correctly filled?
    try:
        model_info = API.model_info(repo_id=model_name, revision=revision)
    except Exception:
        return styled_error("Could not get your model information. Please, fill it up properly.")

    model_size = get_model_size(
        model_info=model_info,
        precision=precision,
    )

    # Were the model card and license filled?
    try:
        license_title = model_info.cardData["license"]
    except Exception:
        license_title = "Unknown"
        # return styled_error("Please, select a license for your model.")

    is_model_card_ok, error_msg = check_model_card(model_name)
    if not is_model_card_ok:
        return styled_error(error_msg)

    # Seems good, creating the eval
    print("Adding new evaluation request...")

    eval_entry = {
        # "model": model,
        # "base_model": base_model,
        "revision": revision,
        "precision": precision,
        "weight_type": weight_type,
        "status": "PENDING",
        "submitted_time": current_time,
        "model_type": model_type.split()[1],  # remove the emoji
        # "likes": model_info.likes,
        "params": model_size,
        "license": license_title,
        # "private": False,
    }

    # Check for duplicate submission
    request_id = get_request_id(model_name, revision, precision)
    if request_id in REQUESTED_MODELS:
        return styled_warning('This model has already been submitted.')
    request_hash = get_request_hash(model_name, revision, precision)

    print("Creating evaluation request file...")
    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{model_name}"
    os.makedirs(OUT_DIR, exist_ok=True)

    out_path = f"{OUT_DIR}/{request_hash}.json"
    if os.path.exists(out_path):
        os.remove(out_path)

    with open(out_path, 'w') as f:
        json.dump(
            eval_entry,
            f,
            indent=4,
        )

    print("Uploading evaluation file...")
    API.upload_file(
        path_or_fileobj=out_path,
        path_in_repo='{}/{}.json'.format(model_name, request_hash),
        repo_id=REQUESTS_REPO,
        repo_type="dataset",
        commit_message=f"Add an evaluation request for {model_name}",
    )

    notify(model_name)

    # Remove the local file
    os.remove(out_path)

    return styled_message(
        "Your model has been submitted."
    )