better checkboxes, better filtering

app.py

@@ -99,13 +99,10 @@ def update_table(
     type_query: list,
     precision_query: str,
     size_query: list,
-    show_deleted: bool,
-    show_merges: bool,
-    show_moe: bool,
-    show_flagged: bool,
+    hide_models: list,
     query: str,
 ):
-    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted, show_merges, show_moe, show_flagged)
+    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, hide_models)
     filtered_df = filter_queries(query, filtered_df)
     df = select_columns(filtered_df, columns)
     return df
@@ -153,21 +150,21 @@ def filter_queries(query: str, filtered_df: pd.DataFrame):
 
 
 def filter_models(
-    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool, show_merges: bool, show_moe:bool, show_flagged: bool
+    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, hide_models: list
 ) -> pd.DataFrame:
     # Show all models
-    if show_deleted:
-        filtered_df = df
-    else:  # Show only still on the hub models
+    if "Private or deleted" in hide_models:
         filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
+    else:
+        filtered_df = df
 
-    if not show_merges:
+    if "Merges and moerges" in hide_models:
         filtered_df = filtered_df[filtered_df[AutoEvalColumn.merged.name] == False]
 
-    if not show_moe:
+    if "MoE" in hide_models:
         filtered_df = filtered_df[filtered_df[AutoEvalColumn.moe.name] == False]
 
-    if not show_flagged:
+    if "Flagged" in hide_models:
         filtered_df = filtered_df[filtered_df[AutoEvalColumn.flagged.name] == False]
 
     type_emoji = [t[0] for t in type_query]
@@ -186,10 +183,7 @@ leaderboard_df = filter_models(
     type_query=[t.to_str(" : ") for t in ModelType], 
     size_query=list(NUMERIC_INTERVALS.keys()), 
     precision_query=[i.value.name for i in Precision],
-    show_deleted=False, 
-    show_merges=False, 
-    show_moe=True,
-    show_flagged=False
+    hide_models=[True, True, True, False], # Deleted, merges, flagged, MoEs
 )
 
 demo = gr.Blocks(css=custom_css)
@@ -224,17 +218,11 @@ with demo:
                             interactive=True,
                         )
                     with gr.Row():
-                        deleted_models_visibility = gr.Checkbox(
-                            value=False, label="Show private/deleted models", interactive=True
-                        )
-                        merged_models_visibility = gr.Checkbox(
-                            value=False, label="Show merges", interactive=True
-                        )
-                        moe_models_visibility = gr.Checkbox(
-                            value=True, label="Show MoE", interactive=True
-                        )
-                        flagged_models_visibility = gr.Checkbox(
-                            value=False, label="Show flagged models", interactive=True
+                        hide_models = gr.CheckboxGroup(
+                            label="Hide models",
+                            choices = ["Private or deleted", "Merges and moerges", "Flagged", "MoE"],
+                            value=["Private or deleted", "Merges and moerges", "Flagged"],
+                            interactive=True
                         )
                 with gr.Column(min_width=320):
                     #with gr.Box(elem_id="box-filter"):
@@ -289,10 +277,7 @@ with demo:
                     filter_columns_type,
                     filter_columns_precision,
                     filter_columns_size,
-                    deleted_models_visibility,
-                    merged_models_visibility,
-                    moe_models_visibility,
-                    flagged_models_visibility,
+                    hide_models,
                     search_bar,
                 ],
                 leaderboard_table,
@@ -308,10 +293,7 @@ with demo:
                     filter_columns_type,
                     filter_columns_precision,
                     filter_columns_size,
-                    deleted_models_visibility,
-                    merged_models_visibility,
-                    moe_models_visibility,
-                    flagged_models_visibility,
+                    hide_models,
                     search_bar,
                 ],
                 leaderboard_table,
@@ -319,7 +301,7 @@ with demo:
             # Check query parameter once at startup and update search bar + hidden component
             demo.load(load_query, inputs=[], outputs=[search_bar, hidden_search_bar])
             
-            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility, merged_models_visibility, moe_models_visibility, flagged_models_visibility]:
+            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, hide_models]:
                 selector.change(
                     update_table,
                     [
@@ -328,10 +310,7 @@ with demo:
                         filter_columns_type,
                         filter_columns_precision,
                         filter_columns_size,
-                        deleted_models_visibility,
-                        merged_models_visibility,
-                        moe_models_visibility,
-                        flagged_models_visibility,
+                        hide_models,
                         search_bar,
                     ],
                     leaderboard_table,

src/scripts/update_all_request_files.py

@@ -3,7 +3,7 @@ from huggingface_hub import ModelCard
 
 import json
 import time
-from src.submission.check_validity import is_model_on_hub, check_model_card
+from src.submission.check_validity import is_model_on_hub, check_model_card, get_model_tags
 from src.envs import DYNAMIC_INFO_REPO, DYNAMIC_INFO_PATH, DYNAMIC_INFO_FILE_PATH, API, H4_TOKEN
 
 def update_models(file_path, models):
@@ -35,37 +35,16 @@ def update_models(file_path, models):
             # If the model doesn't have a model card or a license, we consider it's deleted
             if still_on_hub:
                 try:
-                    if check_model_card(model_id)[0] is False:
+                    status, msg, model_card = check_model_card(model_id)
+                    if status is False:
                         still_on_hub = False
                 except Exception:
+                    model_card = None
                     still_on_hub = False
             data['still_on_hub'] = still_on_hub
 
-            #  Check if the model is a merge
-            is_merge_from_metadata = False
-            is_moe_from_metadata = False
             if still_on_hub:
-                model_card = ModelCard.load(model_id)
-
-                # Storing the model metadata
-                tags = []
-                if model_card.data.tags:
-                    is_merge_from_metadata = "merge" in model_card.data.tags
-                    is_moe_from_metadata = "moe" in model_card.data.tags
-                merge_keywords = ["mergekit", "merged model", "merge model", "merging"]
-                # If the model is a merge but not saying it in the metadata, we flag it
-                is_merge_from_model_card = any(keyword in model_card.text.lower() for keyword in merge_keywords)
-                if is_merge_from_model_card or is_merge_from_metadata:
-                    tags.append("merge")
-                    if not is_merge_from_metadata:
-                        tags.append("flagged:undisclosed_merge")
-                moe_keywords = ["moe", "mixture of experts", "mixtral"]
-                is_moe_from_model_card = any(keyword in model_card.text.lower() for keyword in moe_keywords)
-                is_moe_from_name = "moe" in model_id.lower().replace("/", "-").replace("_", "-").split("-")
-                if is_moe_from_model_card or is_moe_from_name or is_moe_from_metadata:
-                    tags.append("moe")
-                    if not is_moe_from_metadata:
-                        tags.append("flagged:undisclosed_moe")
+                tags = get_model_tags(model_card, model_id)
 
             data["tags"] = tags
 

src/submission/check_validity.py

@@ -19,7 +19,7 @@ def check_model_card(repo_id: str) -> tuple[bool, str]:
     try:
         card = ModelCard.load(repo_id)
     except huggingface_hub.utils.EntryNotFoundError:
-        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
+        return False, "Please add a model card to your model to explain how you trained/fine-tuned it.", None
 
     # Enforce license metadata
     if card.data.license is None:
@@ -27,13 +27,13 @@ def check_model_card(repo_id: str) -> tuple[bool, str]:
             return False, (
                 "License not found. Please add a license to your model card using the `license` metadata or a"
                 " `license_name`/`license_link` pair."
-            )
+            ), None
 
     # Enforce card content
     if len(card.text) < 200:
-        return False, "Please add a description to your model card, it is too short."
+        return False, "Please add a description to your model card, it is too short.", None
 
-    return True, ""
+    return True, "", card
 
 
 def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str, AutoConfig]:
@@ -133,3 +133,32 @@ def already_submitted_models(requested_models_dir: str) -> set[str]:
                     users_to_submission_dates[organisation].append(info["submitted_time"])
 
     return set(file_names), users_to_submission_dates
+
+def get_model_tags(model_card, model: str):
+    is_merge_from_metadata = False
+    is_moe_from_metadata = False
+
+    tags = []
+    if model_card is None:
+        return tags
+    if model_card.data.tags:
+        is_merge_from_metadata = "merge" in model_card.data.tags
+        is_moe_from_metadata = "moe" in model_card.data.tags
+    merge_keywords = ["mergekit", "merged model", "merge model", "merging"]
+    # If the model is a merge but not saying it in the metadata, we flag it
+    is_merge_from_model_card = any(keyword in model_card.text.lower() for keyword in merge_keywords)
+    if is_merge_from_model_card or is_merge_from_metadata:
+        tags.append("merge")
+        if not is_merge_from_metadata:
+            tags.append("flagged:undisclosed_merge")
+    moe_keywords = ["moe", "mixture of experts", "mixtral"]
+    is_moe_from_model_card = any(keyword in model_card.text.lower() for keyword in moe_keywords)
+    is_moe_from_name = "moe" in model.lower().replace("/", "-").replace("_", "-").split("-")
+    if is_moe_from_model_card or is_moe_from_name or is_moe_from_metadata:
+        tags.append("moe")
+        # We no longer tag undisclosed MoEs
+        #if not is_moe_from_metadata:
+        #    tags.append("flagged:undisclosed_moe")
+
+
+    return tags

src/submission/submit.py

@@ -13,6 +13,7 @@ from src.submission.check_validity import (
     get_model_size,
     is_model_on_hub,
     user_submission_permission,
+    get_model_tags
 )
 
 REQUESTED_MODELS = None
@@ -96,34 +97,11 @@ def add_new_eval(
     except Exception:
         return styled_error("Please select a license for your model")
 
-    modelcard_OK, error_msg = check_model_card(model)
+    modelcard_OK, error_msg, model_card = check_model_card(model)
     if not modelcard_OK:
         return styled_error(error_msg)
     
-    is_merge_from_metadata = False
-    is_moe_from_metadata = False
-    model_card = ModelCard.load(model)
-
-    # Storing the model tags
-    tags = []
-    if model_card.data.tags:
-        is_merge_from_metadata = "merge" in model_card.data.tags
-        is_moe_from_metadata = "moe" in model_card.data.tags
-    merge_keywords = ["mergekit", "merged model", "merge model", "merging"]
-    # If the model is a merge but not saying it in the metadata, we flag it
-    is_merge_from_model_card = any(keyword in model_card.text.lower() for keyword in merge_keywords)
-    if is_merge_from_model_card or is_merge_from_metadata:
-        tags.append("merge")
-        if not is_merge_from_metadata:
-            tags.append("flagged:undisclosed_merge")
-    moe_keywords = ["moe", "mixture of experts", "mixtral"]
-    is_moe_from_model_card = any(keyword in model_card.text.lower() for keyword in moe_keywords)
-    is_moe_from_name = "moe" in model.lower().replace("/", "-").replace("_", "-").split("-")
-    if is_moe_from_model_card or is_moe_from_name or is_moe_from_metadata:
-        tags.append("moe")
-        if not is_moe_from_metadata:
-            tags.append("flagged:undisclosed_moe")
-
+    tags = get_model_tags(model_card, model)
 
     # Seems good, creating the eval
     print("Adding new eval")