Spaces:
Running
Running
display message after eval submission
Browse files- .gitignore +1 -0
- app.py +41 -15
.gitignore
CHANGED
@@ -1,3 +1,4 @@
|
|
1 |
evals/
|
2 |
venv/
|
3 |
__pycache__/
|
|
|
|
1 |
evals/
|
2 |
venv/
|
3 |
__pycache__/
|
4 |
+
.env
|
app.py
CHANGED
@@ -145,27 +145,37 @@ def is_model_on_hub(model_name, revision) -> bool:
|
|
145 |
|
146 |
|
147 |
|
148 |
-
def add_new_eval(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
149 |
# check the model actually exists before adding the eval
|
150 |
if revision == "":
|
151 |
revision = "main"
|
152 |
if is_delta_weight and not is_model_on_hub(base_model, revision):
|
153 |
-
|
154 |
-
|
|
|
155 |
|
156 |
if not is_model_on_hub(model, revision):
|
157 |
-
|
158 |
-
|
|
|
|
|
159 |
print("adding new eval")
|
160 |
|
161 |
eval_entry = {
|
162 |
-
"model"
|
163 |
-
"base_model"
|
164 |
-
"revision"
|
165 |
-
"private"
|
166 |
-
"8bit_eval"
|
167 |
-
"is_delta_weight"
|
168 |
-
"status"
|
169 |
}
|
170 |
|
171 |
user_name = ""
|
@@ -174,7 +184,7 @@ def add_new_eval(model:str, base_model : str, revision:str, is_8_bit_eval: bool,
|
|
174 |
user_name = model.split("/")[0]
|
175 |
model_path = model.split("/")[1]
|
176 |
|
177 |
-
OUT_DIR=f"eval_requests/{user_name}"
|
178 |
os.makedirs(OUT_DIR, exist_ok=True)
|
179 |
out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{is_8_bit_eval}_{is_delta_weight}.json"
|
180 |
|
@@ -190,6 +200,9 @@ def add_new_eval(model:str, base_model : str, revision:str, is_8_bit_eval: bool,
|
|
190 |
token=H4_TOKEN,
|
191 |
repo_type="dataset",
|
192 |
)
|
|
|
|
|
|
|
193 |
|
194 |
|
195 |
def refresh():
|
@@ -247,8 +260,21 @@ We chose these benchmarks as they test a variety of reasoning and general knowle
|
|
247 |
|
248 |
with gr.Row():
|
249 |
submit_button = gr.Button("Submit Eval")
|
250 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
251 |
|
252 |
|
253 |
block.load(refresh, inputs=[], outputs=[leaderboard_table, eval_table])
|
254 |
-
block.launch()
|
|
|
145 |
|
146 |
|
147 |
|
148 |
+
def add_new_eval(
|
149 |
+
model: str,
|
150 |
+
base_model: str,
|
151 |
+
revision: str,
|
152 |
+
is_8_bit_eval: bool,
|
153 |
+
private: bool,
|
154 |
+
is_delta_weight: bool,
|
155 |
+
):
|
156 |
# check the model actually exists before adding the eval
|
157 |
if revision == "":
|
158 |
revision = "main"
|
159 |
if is_delta_weight and not is_model_on_hub(base_model, revision):
|
160 |
+
error_message = f"Base model \"{base_model}\" was not found on hub!"
|
161 |
+
print(error_message)
|
162 |
+
return f"<p style='color: red; font-size: 18px; text-align: center;'>{error_message}</p>"
|
163 |
|
164 |
if not is_model_on_hub(model, revision):
|
165 |
+
error_message = f"Model \"{model}\"was not found on hub!"
|
166 |
+
print(error_message)
|
167 |
+
return f"<p style='color: red; font-size: 18px; text-align: center;'>{error_message}</p>"
|
168 |
+
|
169 |
print("adding new eval")
|
170 |
|
171 |
eval_entry = {
|
172 |
+
"model": model,
|
173 |
+
"base_model": base_model,
|
174 |
+
"revision": revision,
|
175 |
+
"private": private,
|
176 |
+
"8bit_eval": is_8_bit_eval,
|
177 |
+
"is_delta_weight": is_delta_weight,
|
178 |
+
"status": "PENDING",
|
179 |
}
|
180 |
|
181 |
user_name = ""
|
|
|
184 |
user_name = model.split("/")[0]
|
185 |
model_path = model.split("/")[1]
|
186 |
|
187 |
+
OUT_DIR = f"eval_requests/{user_name}"
|
188 |
os.makedirs(OUT_DIR, exist_ok=True)
|
189 |
out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{is_8_bit_eval}_{is_delta_weight}.json"
|
190 |
|
|
|
200 |
token=H4_TOKEN,
|
201 |
repo_type="dataset",
|
202 |
)
|
203 |
+
|
204 |
+
success_message = "Your request has been submitted to the evaluation queue!"
|
205 |
+
return f"<p style='color: green; font-size: 18px; text-align: center;'>{success_message}</p>"
|
206 |
|
207 |
|
208 |
def refresh():
|
|
|
260 |
|
261 |
with gr.Row():
|
262 |
submit_button = gr.Button("Submit Eval")
|
263 |
+
with gr.Row():
|
264 |
+
submission_result = gr.Markdown()
|
265 |
+
submit_button.click(
|
266 |
+
add_new_eval,
|
267 |
+
[
|
268 |
+
model_name_textbox,
|
269 |
+
base_model_name_textbox,
|
270 |
+
revision_name_textbox,
|
271 |
+
is_8bit_toggle,
|
272 |
+
private,
|
273 |
+
is_delta_weight,
|
274 |
+
],
|
275 |
+
submission_result
|
276 |
+
)
|
277 |
|
278 |
|
279 |
block.load(refresh, inputs=[], outputs=[leaderboard_table, eval_table])
|
280 |
+
block.launch()
|