refactor: Update model handling and utility functions

- Removed BrainDAO from model type dropdown in app.py
- Commented out citation accordion in app.py
- Adjusted job scheduler interval from 1800 to 3600 seconds in app.py
- Updated utility functions in src/utils.py for model name extraction from file paths
- Refactored model name retrieval in src/leaderboard/read_evals.py, src/populate.py, and src/submission/submit.py
- Removed model likes from submission data in src/submission/submit.py
- Adjusted model type storage in submission data to exclude emoji

app.py

@@ -170,7 +170,7 @@ with demo:
                     model_name_textbox = gr.Textbox(label="Model name")
                     revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
                     model_type = gr.Dropdown(
-                        choices=[t.to_str(" ") for t in ModelType if t != ModelType.Unknown],
+                        choices=[t.to_str(" ") for t in ModelType if t not in [ModelType.Unknown, ModelType.BrainDAO]],
                         label="Model type",
                         multiselect=False,
                         value=None,
@@ -207,18 +207,18 @@ with demo:
                 submission_result,
             )
 
-    with gr.Row():
-        with gr.Accordion("📙 Citation", open=False):
-            citation_button = gr.Textbox(
-                value=CITATION_BUTTON_TEXT,
-                label=CITATION_BUTTON_LABEL,
-                lines=20,
-                elem_id="citation-button",
-                show_copy_button=True,
-            )
+    # with gr.Row():
+    #     with gr.Accordion("📙 Citation", open=False):
+    #         citation_button = gr.Textbox(
+    #             value=CITATION_BUTTON_TEXT,
+    #             label=CITATION_BUTTON_LABEL,
+    #             lines=20,
+    #             elem_id="citation-button",
+    #             show_copy_button=True,
+    #         )
 
 scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
+scheduler.add_job(restart_space, "interval", seconds=3600)
 scheduler.start()
 demo.queue(default_concurrency_limit=40).launch(
     server_name="0.0.0.0",

src/leaderboard/read_evals.py

@@ -13,7 +13,7 @@ import numpy as np
 from src.display.formatting import make_clickable_model
 from src.display.utils import AutoEvalColumn, ModelType, Precision, Tasks, WeightType
 from src.submission.check_validity import is_model_on_hub
-from src.utils import get_model_name, get_org_and_model_names, get_request_hash
+from src.utils import get_model_name_from_filepath, get_org_and_model_names_from_filepath, get_request_hash
 
 
 @dataclass
@@ -42,14 +42,14 @@ class EvalResult:
         with open(json_filepath) as fp:
             data = json.load(fp)
 
-        org, model = get_org_and_model_names(json_filepath)
+        org, model = get_org_and_model_names_from_filepath(json_filepath)
         config = data.get("config")
 
         # Precision
         precision = Precision.from_str(config.get("model_dtype"))
 
         result_key = f"{org}_{model}_{precision.value.name}"
-        model_name = get_model_name(json_filepath)
+        model_name = get_model_name_from_filepath(json_filepath)
 
         still_on_hub, _, model_config = is_model_on_hub(
             model_name,

src/populate.py

@@ -10,7 +10,7 @@ import pandas as pd
 from src.display.formatting import has_no_nan_values, make_clickable_model
 from src.display.utils import AutoEvalColumn, EvalQueueColumn
 from src.leaderboard.read_evals import get_raw_eval_results
-from src.utils import get_model_name
+from src.utils import get_model_name_from_filepath
 
 
 def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
@@ -43,7 +43,7 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
 #                 continue
 
 #             # data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-#             model_name = get_model_name(file_path)
+#             model_name = get_model_name_from_filepath(file_path)
 #             data[EvalQueueColumn.model.name] = make_clickable_model(model_name)
 
 #             data[EvalQueueColumn.revision.name] = data.get("revision", "main")
@@ -66,7 +66,7 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
 #                     continue
 
 #                 # data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-#                 model_name = get_model_name(file_path)
+#                 model_name = get_model_name_from_filepath(file_path)
 #                 data[EvalQueueColumn.model.name] = make_clickable_model(model_name)
 
 #                 data[EvalQueueColumn.revision.name] = data.get("revision", "main")
@@ -92,7 +92,7 @@ def get_evaluation_requests_df(save_path: str, cols: list) -> list[pd.DataFrame]
             print(f"Error reading or decoding {file_path}: {e}")
             return None
 
-        model_name = get_model_name(file_path)
+        model_name = get_model_name_from_filepath(file_path)
         # data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
         data[EvalQueueColumn.model.name] = make_clickable_model(model_name)
         data[EvalQueueColumn.revision.name] = data.get("revision", "main")

src/submission/submit.py

@@ -16,7 +16,7 @@ USERS_TO_SUBMISSION_DATES = None
 
 
 def add_new_eval(
-    model: str,
+    model_name: str,
     # base_model: str,
     revision: str,
     # precision: str,
@@ -32,12 +32,6 @@ def add_new_eval(
     if not REQUESTED_MODELS:
         REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
 
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
     precision = precision.split(" ")[0]
     current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
 
@@ -55,13 +49,13 @@ def add_new_eval(
     #         return styled_error(f'Base model "{base_model}" {error}')
 
     if weight_type != "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
+        model_on_hub, error, _ = is_model_on_hub(model_name=model_name, revision=revision, token=TOKEN, test_tokenizer=True)
         if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
+            return styled_error(f'Model "{model_name}" {error}')
 
     # Is the model info correctly filled?
     try:
-        model_info = API.model_info(repo_id=model, revision=revision)
+        model_info = API.model_info(repo_id=model_name, revision=revision)
     except Exception:
         return styled_error("Could not get your model information. Please fill it up properly.")
 
@@ -76,7 +70,7 @@ def add_new_eval(
     except Exception:
         return styled_error("Please select a license for your model")
 
-    is_model_card_ok, error_msg = check_model_card(model)
+    is_model_card_ok, error_msg = check_model_card(model_name)
     if not is_model_card_ok:
         return styled_error(error_msg)
 
@@ -91,23 +85,26 @@ def add_new_eval(
         "weight_type": weight_type,
         "status": "PENDING",
         "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
+        "model_type": model_type.split()[1],  # remove the emoji
+        # "likes": model_info.likes,
         "params": model_size,
         "license": license_title,
-        "private": False,
+        # "private": False,
     }
 
     # Check for duplicate submission
-    request_id = get_request_id(model, revision, precision)
+    request_id = get_request_id(model_name, revision, precision)
     if request_id in REQUESTED_MODELS:
         return styled_warning("This model has been already submitted.")
-    request_hash = get_request_hash(model, revision, precision)
+    request_hash = get_request_hash(model_name, revision, precision)
 
     print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{model_name}"
     os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}/{request_hash}.json"
+
+    out_path = f"{OUT_DIR}/{request_hash}.json"
+    if os.path.exists(out_path):
+        os.remove(out_path)
 
     with open(out_path, "w") as f:
         f.write(json.dumps(eval_entry))
@@ -115,15 +112,15 @@ def add_new_eval(
     print("Uploading eval file")
     API.upload_file(
         path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-requests/")[1],
+        path_in_repo='{}/{}.json'.format(model_name, request_hash),
         repo_id=REQUESTS_REPO,
         repo_type="dataset",
-        commit_message=f"Add {model} to eval requests",
+        commit_message=f"Add {model_name} to eval requests",
     )
 
     # Remove the local file
     os.remove(out_path)
 
     return styled_message(
-        "Your request has been submitted to the evaluation requests!\nPlease wait for up to an hour for the model to show in the PENDING list."
+        "Your model has been submitted."
     )

src/utils.py

@@ -4,15 +4,15 @@
 import hashlib
 
 
-def get_org_and_model_names(file_path: str) -> str:
+def get_org_and_model_names_from_filepath(file_path: str) -> str:
     org, model = file_path.split("/")[-3:-1]
     model = model.removesuffix(".json")
     model = model.split('_request_')[0]
     return org, model
 
 
-def get_model_name(file_path: str) -> str:
-    org, model = get_org_and_model_names(file_path)
+def get_model_name_from_filepath(file_path: str) -> str:
+    org, model = get_org_and_model_names_from_filepath(file_path)
     return f"{org}/{model}"