dummy column refactoring (#688)

- collection update only happens on full initialization (d131b6cd68b918bb71c70060116d55dbd1d7e4be)
- removed dummy column (ecacc0f636e01ce3fa984e61cab8f4b0f5670af6)
- enhanced naming of dummy column (bab5ced191e1edf26b96473d5d42f51b0bd19784)

app.py

@@ -82,10 +82,12 @@ def download_dataset(repo_id, local_dir, repo_type="dataset", max_attempts=3):
 def init_space(full_init: bool = True):
     """Initializes the application space, loading only necessary data."""
     if full_init:
+        # These downloads only occur on full initialization
         download_dataset(QUEUE_REPO, EVAL_REQUESTS_PATH)
         download_dataset(DYNAMIC_INFO_REPO, DYNAMIC_INFO_PATH)
         download_dataset(RESULTS_REPO, EVAL_RESULTS_PATH)
 
+    # Always retrieve the leaderboard DataFrame
     raw_data, original_df = get_leaderboard_df(
         results_path=EVAL_RESULTS_PATH,
         requests_path=EVAL_REQUESTS_PATH,
@@ -93,14 +95,18 @@ def init_space(full_init: bool = True):
         cols=COLS,
         benchmark_cols=BENCHMARK_COLS,
     )
-    update_collections(original_df)
+
+    if full_init:
+        # Collection update only happens on full initialization
+        update_collections(original_df)
+
     leaderboard_df = original_df.copy()
     
+    # Evaluation queue DataFrame retrieval is independent of initialization detail level
     eval_queue_dfs = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
     return leaderboard_df, raw_data, original_df, eval_queue_dfs
 
-
 # Convert the environment variable "LEADERBOARD_FULL_INIT" to a boolean value, defaulting to True if the variable is not set.
 # This controls whether a full initialization should be performed.
 do_full_init = os.getenv("LEADERBOARD_FULL_INIT", "True") == "True"
@@ -148,23 +154,17 @@ def load_query(request: gr.Request):  # triggered only once at startup => read q
 
 
 def search_model(df: pd.DataFrame, query: str) -> pd.DataFrame:
-    return df[(df[AutoEvalColumn.dummy.name].str.contains(query, case=False, na=False))]
-
+    return df[(df[AutoEvalColumn.fullname.name].str.contains(query, case=False, na=False))]
 
 def search_license(df: pd.DataFrame, query: str) -> pd.DataFrame:
     return df[df[AutoEvalColumn.license.name].str.contains(query, case=False, na=False)]
 
-
 def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
     always_here_cols = [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
-    dummy_col = [AutoEvalColumn.dummy.name]
-    # AutoEvalColumn.model_type_symbol.name,
-    # AutoEvalColumn.model.name,
-    # We use COLS to maintain sorting
+    dummy_col = [AutoEvalColumn.fullname.name]
     filtered_df = df[always_here_cols + [c for c in COLS if c in df.columns and c in columns] + dummy_col]
     return filtered_df
 
-
 def filter_queries(query: str, df: pd.DataFrame):
     tmp_result_df = []
 
@@ -323,14 +323,13 @@ with demo:
                 value=leaderboard_df[
                     [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
                     + shown_columns.value
-                    + [AutoEvalColumn.dummy.name]
+                    + [AutoEvalColumn.fullname.name]
                 ],
                 headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
                 datatype=TYPES,
                 elem_id="leaderboard-table",
                 interactive=False,
                 visible=True,
-                # column_widths=["2%", "33%"]
             )
 
             # Dummy leaderboard for handling the case when the user uses backspace key

src/display/css_html_js.py

@@ -1,4 +1,5 @@
 custom_css = """
+
 /* Hides the final AutoEvalColumn */
 #llm-benchmark-tab-table table td:last-child,
 #llm-benchmark-tab-table table th:last-child {
@@ -44,7 +45,7 @@ table th:first-child {
     background: none;
     border: none;
 }
- 
+
 #search-bar {
     padding: 0px;
 }
@@ -94,4 +95,4 @@ get_window_url_params = """
         url_params = Object.fromEntries(params);
         return url_params;
     }
-    """
+    """

src/display/utils.py

@@ -71,12 +71,13 @@ auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sh
 auto_eval_column_dict.append(["flagged", ColumnContent, ColumnContent("Flagged", "bool", False, hidden=True)])
 auto_eval_column_dict.append(["moe", ColumnContent, ColumnContent("MoE", "bool", False, hidden=True)])
 # Dummy column for the search bar (hidden by the custom CSS)
-auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
+auto_eval_column_dict.append(["fullname", ColumnContent, ColumnContent("fullname", "str", False, dummy=True)])
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
 
 
+
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
     model = ColumnContent("model", "markdown", True)
@@ -99,7 +100,7 @@ baseline_row = {
     AutoEvalColumn.truthfulqa.name: 25.0,
     AutoEvalColumn.winogrande.name: 50.0,
     AutoEvalColumn.gsm8k.name: 0.21,
-    AutoEvalColumn.dummy.name: "baseline",
+    AutoEvalColumn.fullname.name: "baseline",
     AutoEvalColumn.model_type.name: "",
     AutoEvalColumn.flagged.name: False,
 }
@@ -124,7 +125,7 @@ human_baseline_row = {
     AutoEvalColumn.truthfulqa.name: 94.0,
     AutoEvalColumn.winogrande.name: 94.0,
     AutoEvalColumn.gsm8k.name: 100,
-    AutoEvalColumn.dummy.name: "human_baseline",
+    AutoEvalColumn.fullname.name: "human_baseline",
     AutoEvalColumn.model_type.name: "",
     AutoEvalColumn.flagged.name: False,
 }

src/leaderboard/filter_models.py

@@ -130,14 +130,17 @@ DO_NOT_SUBMIT_MODELS = [
 
 
 def flag_models(leaderboard_data: list[dict]):
+    """Flags models based on external criteria or flagged status."""
     for model_data in leaderboard_data:
         # Merges and moes are flagged automatically
         if model_data[AutoEvalColumn.flagged.name]:
             flag_key = "merged"
         else:
-            flag_key = model_data["model_name_for_query"]
+            flag_key = model_data[AutoEvalColumn.fullname.name]
 
+        print(f"model check: {flag_key}")
         if flag_key in FLAGGED_MODELS:
+            print(f"Flagged model: {flag_key}")
             issue_num = FLAGGED_MODELS[flag_key].split("/")[-1]
             issue_link = model_hyperlink(
                 FLAGGED_MODELS[flag_key],
@@ -152,11 +155,13 @@ def flag_models(leaderboard_data: list[dict]):
 
 
 def remove_forbidden_models(leaderboard_data: list[dict]):
+    """Removes models from the leaderboard based on the DO_NOT_SUBMIT list."""
     indices_to_remove = []
     for ix, model in enumerate(leaderboard_data):
-        if model["model_name_for_query"] in DO_NOT_SUBMIT_MODELS:
+        if model[AutoEvalColumn.fullname.name] in DO_NOT_SUBMIT_MODELS:
             indices_to_remove.append(ix)
 
+    # Remove the models from the list
     for ix in reversed(indices_to_remove):
         leaderboard_data.pop(ix)
     return leaderboard_data
@@ -165,3 +170,4 @@ def remove_forbidden_models(leaderboard_data: list[dict]):
 def filter_models_flags(leaderboard_data: list[dict]):
     leaderboard_data = remove_forbidden_models(leaderboard_data)
     flag_models(leaderboard_data)
+

src/leaderboard/read_evals.py

@@ -133,7 +133,7 @@ class EvalResult:
             AutoEvalColumn.weight_type.name: self.weight_type.value.name,
             AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            AutoEvalColumn.dummy.name: self.full_model,
+            AutoEvalColumn.fullname.name: self.full_model,
             AutoEvalColumn.revision.name: self.revision,
             AutoEvalColumn.average.name: average,
             AutoEvalColumn.license.name: self.license,

src/tools/collections.py

@@ -60,7 +60,7 @@ def update_collections(df: DataFrame):
         for size, interval in intervals.items():
             filtered_df = _filter_by_type_and_size(df, model_type, interval)
             best_models = list(
-                filtered_df.sort_values(AutoEvalColumn.average.name, ascending=False)[AutoEvalColumn.dummy.name][:10]
+                filtered_df.sort_values(AutoEvalColumn.average.name, ascending=False)[AutoEvalColumn.fullname.name][:10]
             )
             print(model_type.value.symbol, size, best_models)
             _add_models_to_collection(collection, best_models, model_type, size)
@@ -73,4 +73,4 @@ def update_collections(df: DataFrame):
         try:
             delete_collection_item(collection_slug=PATH_TO_COLLECTION, item_object_id=item_id, token=H4_TOKEN)
         except HfHubHTTPError:
-            continue
+            continue