simplified display, added an extra config repo to carry dynamic information

app.py

@@ -30,6 +30,7 @@ from src.display.utils import (
 from src.envs import API, EVAL_REQUESTS_PATH, DYNAMIC_INFO_REPO, DYNAMIC_INFO_FILE_PATH, DYNAMIC_INFO_PATH, EVAL_RESULTS_PATH, H4_TOKEN, IS_PUBLIC, QUEUE_REPO, REPO_ID, RESULTS_REPO
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
+from src.scripts.update_all_request_files import update_dynamic_files
 from src.tools.collections import update_collections
 from src.tools.plots import (
     create_metric_plot_obj,
@@ -100,10 +101,11 @@ def update_table(
     size_query: list,
     show_deleted: bool,
     show_merges: bool,
+    show_moe: bool,
     show_flagged: bool,
     query: str,
 ):
-    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted, show_merges, show_flagged)
+    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted, show_merges, show_moe, show_flagged)
     filtered_df = filter_queries(query, filtered_df)
     df = select_columns(filtered_df, columns)
     return df
@@ -119,13 +121,13 @@ def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
 
 
 def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
-    always_here_cols = [
-        AutoEvalColumn.model_type_symbol.name,
-        AutoEvalColumn.model.name,
-    ]
+    always_here_cols = [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
+    dummy_col = [AutoEvalColumn.dummy.name]
+        #AutoEvalColumn.model_type_symbol.name,
+        #AutoEvalColumn.model.name,
     # We use COLS to maintain sorting
     filtered_df = df[
-        always_here_cols + [c for c in COLS if c in df.columns and c in columns] + [AutoEvalColumn.dummy.name]
+        always_here_cols + [c for c in COLS if c in df.columns and c in columns] + dummy_col
     ]
     return filtered_df
 
@@ -151,7 +153,7 @@ def filter_queries(query: str, filtered_df: pd.DataFrame):
 
 
 def filter_models(
-    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool, show_merges: bool, show_flagged: bool
+    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool, show_merges: bool, show_moe:bool, show_flagged: bool
 ) -> pd.DataFrame:
     # Show all models
     if show_deleted:
@@ -162,6 +164,9 @@ def filter_models(
     if not show_merges:
         filtered_df = filtered_df[filtered_df[AutoEvalColumn.merged.name] == False]
 
+    if not show_moe:
+        filtered_df = filtered_df[filtered_df[AutoEvalColumn.moe.name] == False]
+
     if not show_flagged:
         filtered_df = filtered_df[filtered_df[AutoEvalColumn.flagged.name] == False]
 
@@ -176,7 +181,16 @@ def filter_models(
 
     return filtered_df
 
-leaderboard_df = filter_models(leaderboard_df, [t.to_str(" : ") for t in ModelType], list(NUMERIC_INTERVALS.keys()), [i.value.name for i in Precision], False, False, False)
+leaderboard_df = filter_models(
+    df=leaderboard_df, 
+    type_query=[t.to_str(" : ") for t in ModelType], 
+    size_query=list(NUMERIC_INTERVALS.keys()), 
+    precision_query=[i.value.name for i in Precision],
+    show_deleted=False, 
+    show_merges=False, 
+    show_moe=True,
+    show_flagged=False
+)
 
 demo = gr.Blocks(css=custom_css)
 with demo:
@@ -216,6 +230,9 @@ with demo:
                         merged_models_visibility = gr.Checkbox(
                             value=False, label="Show merges", interactive=True
                         )
+                        moe_models_visibility = gr.Checkbox(
+                            value=True, label="Show MoE", interactive=True
+                        )
                         flagged_models_visibility = gr.Checkbox(
                             value=False, label="Show flagged models", interactive=True
                         )
@@ -274,6 +291,7 @@ with demo:
                     filter_columns_size,
                     deleted_models_visibility,
                     merged_models_visibility,
+                    moe_models_visibility,
                     flagged_models_visibility,
                     search_bar,
                 ],
@@ -292,6 +310,7 @@ with demo:
                     filter_columns_size,
                     deleted_models_visibility,
                     merged_models_visibility,
+                    moe_models_visibility,
                     flagged_models_visibility,
                     search_bar,
                 ],
@@ -300,7 +319,7 @@ with demo:
             # Check query parameter once at startup and update search bar + hidden component
             demo.load(load_query, inputs=[], outputs=[search_bar, hidden_search_bar])
             
-            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility, merged_models_visibility, flagged_models_visibility]:
+            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility, merged_models_visibility, moe_models_visibility, flagged_models_visibility]:
                 selector.change(
                     update_table,
                     [
@@ -311,6 +330,7 @@ with demo:
                         filter_columns_size,
                         deleted_models_visibility,
                         merged_models_visibility,
+                        moe_models_visibility,
                         flagged_models_visibility,
                         search_bar,
                     ],
@@ -439,6 +459,7 @@ with demo:
 
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=10800)
+scheduler.add_job(update_dynamic_files, "interval", seconds=10000) # taking about 3 min
 scheduler.start()
 
 demo.queue(default_concurrency_limit=40).launch()

src/display/utils.py

@@ -50,9 +50,10 @@ auto_eval_column_dict.append(["merged", ColumnContent, ColumnContent("Merged", "
 auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
 auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
 auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False, hidden=True)])
 auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
-auto_eval_column_dict.append(["flagged", ColumnContent, ColumnContent("Flagged", "bool", False, False)])
+auto_eval_column_dict.append(["flagged", ColumnContent, ColumnContent("Flagged", "bool", False, hidden=True)])
+auto_eval_column_dict.append(["moe", ColumnContent, ColumnContent("MoE", "bool", False, hidden=True)])
 # Dummy column for the search bar (hidden by the custom CSS)
 auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
 
@@ -108,6 +109,7 @@ human_baseline_row = {
     AutoEvalColumn.gsm8k.name: 100,
     AutoEvalColumn.dummy.name: "human_baseline",
     AutoEvalColumn.model_type.name: "",
+    AutoEvalColumn.flagged.name: False,
 }
 
 @dataclass
@@ -168,10 +170,8 @@ class Precision(Enum):
 
 
 # Column selection
-COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
-TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
-COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
-TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
+COLS = [c.name for c in fields(AutoEvalColumn)]
+TYPES = [c.type for c in fields(AutoEvalColumn)]
 
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]

src/leaderboard/read_evals.py

@@ -30,7 +30,7 @@ class EvalResult:
     likes: int = 0
     num_params: int = 0
     date: str = "" # submission date of request file
-    still_on_hub: bool = False
+    still_on_hub: bool = True
     is_merge: bool = False
     flagged: bool = False
     tags: list = None
@@ -106,12 +106,12 @@ class EvalResult:
         try:
             with open(request_file, "r") as f:
                 request = json.load(f)
-            self.model_type = ModelType.from_str(request.get("model_type", ""))
+            self.model_type = ModelType.from_str(request.get("model_type", "Unknown"))
             self.weight_type = WeightType[request.get("weight_type", "Original")]
             self.num_params = request.get("params", 0)
             self.date = request.get("submitted_time", "")
-            self.architecture = request["architectures"]
-        except Exception:
+            self.architecture = request.get("architectures", "Unknown")
+        except Exception as e:
             print(f"Could not find request file for {self.org}/{self.model}")
 
     def update_with_dynamic_file_dict(self, file_dict):
@@ -119,7 +119,6 @@ class EvalResult:
         self.likes = file_dict.get("likes", 0)
         self.still_on_hub = file_dict["still_on_hub"]
         self.flagged = any("flagged" in tag for tag in file_dict["tags"])
-        self.is_merge = "merge" in file_dict["tags"]
         self.tags = file_dict["tags"]
         
 
@@ -130,7 +129,6 @@ class EvalResult:
             "eval_name": self.eval_name,  # not a column, just a save name,
             AutoEvalColumn.precision.name: self.precision.value.name,
             AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.merged.name: self.is_merge,
             AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
             AutoEvalColumn.weight_type.name: self.weight_type.value.name,
             AutoEvalColumn.architecture.name: self.architecture,
@@ -142,6 +140,8 @@ class EvalResult:
             AutoEvalColumn.likes.name: self.likes,
             AutoEvalColumn.params.name: self.num_params,
             AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+            AutoEvalColumn.merged.name: "merge" in self.tags if self.tags else False,
+            AutoEvalColumn.moe.name: ("moe" in self.tags if self.tags else False) or "moe" in self.full_model.lower(),
             AutoEvalColumn.flagged.name: self.flagged
         }
 
@@ -199,7 +199,8 @@ def get_raw_eval_results(results_path: str, requests_path: str, dynamic_path: st
         # Creation of result
         eval_result = EvalResult.init_from_json_file(model_result_filepath)
         eval_result.update_with_request_file(requests_path)
-        eval_result.update_with_dynamic_file_dict(dynamic_data[eval_result.full_model])
+        if eval_result.full_model in dynamic_data:
+            eval_result.update_with_dynamic_file_dict(dynamic_data[eval_result.full_model])
 
         # Store results of same eval together
         eval_name = eval_result.eval_name

src/scripts/update_all_request_files.py

@@ -1,31 +1,10 @@
-from huggingface_hub import HfApi, ModelFilter, snapshot_download
+from huggingface_hub import ModelFilter, snapshot_download
 from huggingface_hub import ModelCard
 
 import json
-import os
 import time
-import shutil
-from src.submission.check_validity import is_model_on_hub, check_model_card, get_model_size
-from src.envs import DYNAMIC_INFO_REPO, DYNAMIC_INFO_FILE_PATH, API
-
-HF_TOKEN = os.environ.get("HF_TOKEN", None)
-
-TMP_FOLDER = "tmp_requests"
-snapshot_download(
-    repo_id=DYNAMIC_INFO_REPO, local_dir=TMP_FOLDER, repo_type="dataset", tqdm_class=None, etag_timeout=30
-)
-
-# Get models
-start = time.time()
-
-models = list(API.list_models(
-    filter=ModelFilter(task="text-generation"),
-    full=False,
-    cardData=True,
-    fetch_config=True,
-))
-
-print(f"Downloaded list of models in {time.time() - start:.2f} seconds")
+from src.submission.check_validity import is_model_on_hub, check_model_card
+from src.envs import DYNAMIC_INFO_REPO, DYNAMIC_INFO_PATH, DYNAMIC_INFO_FILE_PATH, API
 
 def update_models(file_path, models):
     """
@@ -80,18 +59,37 @@ def update_models(file_path, models):
     with open(file_path, 'w') as f:
         json.dump(model_infos, f, indent=2)
 
-start = time.time()
+def update_dynamic_files():
+    """ This will only update metadata for models already linked in the repo, not add missing ones.
+    """
+    snapshot_download(
+        repo_id=DYNAMIC_INFO_REPO, local_dir=DYNAMIC_INFO_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
+    )
+
+    print("UPDATE_DYNAMIC: Loaded snapshot")
+    # Get models
+    start = time.time()
+
+    models = list(API.list_models(
+        filter=ModelFilter(task="text-generation"),
+        full=False,
+        cardData=True,
+        fetch_config=True,
+    ))
+
+    print(f"UPDATE_DYNAMIC: Downloaded list of models in {time.time() - start:.2f} seconds")
 
-updated_ids = update_models(DYNAMIC_INFO_FILE_PATH, models)
+    start = time.time()
 
-print(f"updated in {time.time() - start:.2f} seconds, updated ids: {len(updated_ids)}")
+    update_models(DYNAMIC_INFO_FILE_PATH, models)
 
-API.upload_file(
-    path_or_fileobj=DYNAMIC_INFO_FILE_PATH,
-    path_in_repo=DYNAMIC_INFO_FILE_PATH.split("/")[-1],
-    repo_id=DYNAMIC_INFO_REPO,
-    repo_type="dataset",
-    commit_message=f"Daily request file update.",
-)
+    print(f"UPDATE_DYNAMIC: updated in {time.time() - start:.2f} seconds")
 
-shutil.rmtree(TMP_FOLDER)
+    API.upload_file(
+        path_or_fileobj=DYNAMIC_INFO_FILE_PATH,
+        path_in_repo=DYNAMIC_INFO_FILE_PATH.split("/")[-1],
+        repo_id=DYNAMIC_INFO_REPO,
+        repo_type="dataset",
+        commit_message=f"Daily request file update.",
+    )
+    print(f"UPDATE_DYNAMIC: pushed to hub")

src/submission/check_validity.py

@@ -52,7 +52,7 @@ def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_rem
                 return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
         return True, None, config
 
-    except ValueError:
+    except ValueError as e:
         return (
             False,
             "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",