fix

app.py

@@ -109,7 +109,7 @@ def select_columns(df, columns):
     return filtered_df
 
 
-def filter_items(df, leaderboard_table, query):
+def filter_types(df, leaderboard_table, query):
     if query == "all":
         return df[leaderboard_table.columns]
     else:
@@ -118,6 +118,16 @@ def filter_items(df, leaderboard_table, query):
     return filtered_df[leaderboard_table.columns]
 
 
+def filter_direct_complete(df, leaderboard_table, query):
+    if query == "all":
+        return df[leaderboard_table.columns]
+
+    if query == "chat template":
+        return df[~df["direct_complete"]][leaderboard_table.columns]
+    else:
+        return df[df["direct_complete"]][leaderboard_table.columns]
+
+
 def search_table(df, leaderboard_table, query):
     filtered_df = df[(df["model"].str.contains(query, case=False))]
     return filtered_df[leaderboard_table.columns]
@@ -174,13 +184,18 @@ with demo:
                                 show_label=False,
                                 elem_id="search-bar",
                             )
-                            filter_columns = gr.Radio(
+                            filter_types_columns = gr.Radio(
                                 label="⏚ Filter model types",
-                                choices=["all", "🟢 base", "🔶 instruction-tuned", "EXT external-evaluation"],
+                                choices=["all", "🟢 base", "🔶 instruction-tuned"], #, "EXT external-evaluation"],
                                 value="all",
                                 elem_id="filter-columns",
                             )
-
+                            filter_prompting_columns = gr.Radio(
+                                label="⏚ Filter prompting",
+                                choices=["all", "chat template", "direct complete"],
+                                value="all",
+                                elem_id="filter-direct-complete",
+                            )
                     leaderboard_df = gr.components.Dataframe(
                         value=df[
                             [
@@ -210,9 +225,14 @@ with demo:
                         [hidden_leaderboard_df, leaderboard_df, search_bar],
                         leaderboard_df,
                     )
-                    filter_columns.change(
-                        filter_items,
-                        [hidden_leaderboard_df, leaderboard_df, filter_columns],
+                    filter_types_columns.change(
+                        filter_types,
+                        [hidden_leaderboard_df, leaderboard_df, filter_types_columns],
+                        leaderboard_df,
+                    )
+                    filter_prompting_columns.change(
+                        filter_direct_complete,
+                        [hidden_leaderboard_df, leaderboard_df, filter_prompting_columns],
                         leaderboard_df,
                     )
                     shown_columns.change(
@@ -229,7 +249,6 @@ with demo:
                     - `complete` and `instruct` represent the calibrated Pass@1 score on the BigCodeBench benchmark variants.
                     - `elo_mle` represents the task-level Bootstrap of Maximum Likelihood Elo rating on `BigCodeBench-Complete`, which starts from 1000 and is boostrapped 500 times.
                     - `size` is the amount of activated model weight during inference.
-                    - Some instruction-tuned models are marked with 🟢 symbol, as they miss the chat templates in their tokenizer configurations.
                     - Model providers have the responsibility to avoid data contamination. Models trained on close data can be affected by contamination.
                     - For more details check the 📝 About section.
                     - Models with a 🔴 symbol represent external evaluation submission, this means that we didn't verify the results, you can find the author's submission under `Submission PR` field from `See All Columns` tab.

src/utils.py

@@ -24,12 +24,11 @@ def fields(raw_class):
 class AutoEvalColumn:  # Auto evals column
     model_type_symbol = ColumnContent("type", "str", True)
     model = ColumnContent("model", "markdown", True)
-    size = ColumnContent("size", "number", False)
     complete_score = ColumnContent("complete", "number", True)
     instruct_score = ColumnContent("instruct", "number", True)
     elo_mle = ColumnContent("elo_mle", "number", True)
     dummy = ColumnContent("model", "str", True)
-    link = ColumnContent("link", "str", False)
+    size = ColumnContent("size", "number", False)
 
 
 def model_hyperlink(link, model_name):