Nathan Habib commited on
Commit
a69dfa9
1 Parent(s): 2762eff

Adds a way to hide flagged models

Browse files

- Adds flagged models
- makes it so that by default we do not show flagged or removed models

app.py CHANGED
@@ -78,9 +78,10 @@ def update_table(
78
  precision_query: str,
79
  size_query: list,
80
  show_deleted: bool,
 
81
  query: str,
82
  ):
83
- filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
84
  filtered_df = filter_queries(query, filtered_df)
85
  df = select_columns(filtered_df, columns)
86
  return df
@@ -128,7 +129,7 @@ def filter_queries(query: str, filtered_df: pd.DataFrame):
128
 
129
 
130
  def filter_models(
131
- df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
132
  ) -> pd.DataFrame:
133
  # Show all models
134
  if show_deleted:
@@ -136,6 +137,9 @@ def filter_models(
136
  else: # Show only still on the hub models
137
  filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
138
 
 
 
 
139
  type_emoji = [t[0] for t in type_query]
140
  filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
141
  filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
@@ -147,6 +151,7 @@ def filter_models(
147
 
148
  return filtered_df
149
 
 
150
 
151
  demo = gr.Blocks(css=custom_css)
152
  with demo:
@@ -183,6 +188,9 @@ with demo:
183
  deleted_models_visibility = gr.Checkbox(
184
  value=False, label="Show private/deleted models", interactive=True
185
  )
 
 
 
186
  with gr.Column(min_width=320):
187
  #with gr.Box(elem_id="box-filter"):
188
  filter_columns_type = gr.CheckboxGroup(
@@ -237,6 +245,7 @@ with demo:
237
  filter_columns_precision,
238
  filter_columns_size,
239
  deleted_models_visibility,
 
240
  search_bar,
241
  ],
242
  leaderboard_table,
@@ -253,6 +262,7 @@ with demo:
253
  filter_columns_precision,
254
  filter_columns_size,
255
  deleted_models_visibility,
 
256
  search_bar,
257
  ],
258
  leaderboard_table,
@@ -260,7 +270,7 @@ with demo:
260
  # Check query parameter once at startup and update search bar + hidden component
261
  demo.load(load_query, inputs=[], outputs=[search_bar, hidden_search_bar])
262
 
263
- for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility]:
264
  selector.change(
265
  update_table,
266
  [
@@ -270,6 +280,7 @@ with demo:
270
  filter_columns_precision,
271
  filter_columns_size,
272
  deleted_models_visibility,
 
273
  search_bar,
274
  ],
275
  leaderboard_table,
 
78
  precision_query: str,
79
  size_query: list,
80
  show_deleted: bool,
81
+ show_flagged: bool,
82
  query: str,
83
  ):
84
+ filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted, show_flagged)
85
  filtered_df = filter_queries(query, filtered_df)
86
  df = select_columns(filtered_df, columns)
87
  return df
 
129
 
130
 
131
  def filter_models(
132
+ df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool, show_flagged: bool
133
  ) -> pd.DataFrame:
134
  # Show all models
135
  if show_deleted:
 
137
  else: # Show only still on the hub models
138
  filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
139
 
140
+ if not show_flagged:
141
+ filtered_df = filtered_df[filtered_df[AutoEvalColumn.flagged.name] == False]
142
+
143
  type_emoji = [t[0] for t in type_query]
144
  filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
145
  filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
 
151
 
152
  return filtered_df
153
 
154
+ leaderboard_df = filter_models(leaderboard_df, [t.to_str(" : ") for t in ModelType], list(NUMERIC_INTERVALS.keys()), [i.value.name for i in Precision], False, False)
155
 
156
  demo = gr.Blocks(css=custom_css)
157
  with demo:
 
188
  deleted_models_visibility = gr.Checkbox(
189
  value=False, label="Show private/deleted models", interactive=True
190
  )
191
+ flagged_models_visibility = gr.Checkbox(
192
+ value=False, label="Show flagged models", interactive=True
193
+ )
194
  with gr.Column(min_width=320):
195
  #with gr.Box(elem_id="box-filter"):
196
  filter_columns_type = gr.CheckboxGroup(
 
245
  filter_columns_precision,
246
  filter_columns_size,
247
  deleted_models_visibility,
248
+ flagged_models_visibility,
249
  search_bar,
250
  ],
251
  leaderboard_table,
 
262
  filter_columns_precision,
263
  filter_columns_size,
264
  deleted_models_visibility,
265
+ flagged_models_visibility,
266
  search_bar,
267
  ],
268
  leaderboard_table,
 
270
  # Check query parameter once at startup and update search bar + hidden component
271
  demo.load(load_query, inputs=[], outputs=[search_bar, hidden_search_bar])
272
 
273
+ for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility, flagged_models_visibility]:
274
  selector.change(
275
  update_table,
276
  [
 
280
  filter_columns_precision,
281
  filter_columns_size,
282
  deleted_models_visibility,
283
+ flagged_models_visibility,
284
  search_bar,
285
  ],
286
  leaderboard_table,
src/display/utils.py CHANGED
@@ -51,6 +51,7 @@ auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B
51
  auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
52
  auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
53
  auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
 
54
  # Dummy column for the search bar (hidden by the custom CSS)
55
  auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
56
 
@@ -80,6 +81,7 @@ baseline_row = {
80
  AutoEvalColumn.gsm8k.name: 0.21,
81
  AutoEvalColumn.dummy.name: "baseline",
82
  AutoEvalColumn.model_type.name: "",
 
83
  }
84
 
85
  # Average ⬆️ human baseline is 0.897 (source: averaging human baselines below)
 
51
  auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
52
  auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
53
  auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
54
+ auto_eval_column_dict.append(["flagged", ColumnContent, ColumnContent("Flagged", "bool", False, False)])
55
  # Dummy column for the search bar (hidden by the custom CSS)
56
  auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
57
 
 
81
  AutoEvalColumn.gsm8k.name: 0.21,
82
  AutoEvalColumn.dummy.name: "baseline",
83
  AutoEvalColumn.model_type.name: "",
84
+ AutoEvalColumn.flagged.name: False,
85
  }
86
 
87
  # Average ⬆️ human baseline is 0.897 (source: averaging human baselines below)
src/leaderboard/filter_models.py CHANGED
@@ -1,3 +1,4 @@
 
1
  from src.display.formatting import model_hyperlink
2
  from src.display.utils import AutoEvalColumn
3
 
@@ -14,6 +15,17 @@ FLAGGED_MODELS = {
14
  "AIDC-ai-business/Marcoroni-13B": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/287",
15
  "AIDC-ai-business/Marcoroni-7B": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/287",
16
  "fblgit/una-xaberius-34b-v1beta": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/444",
 
 
 
 
 
 
 
 
 
 
 
17
  }
18
 
19
  # Models which have been requested by orgs to not be submitted on the leaderboard
@@ -36,6 +48,9 @@ def flag_models(leaderboard_data: list[dict]):
36
  model_data[
37
  AutoEvalColumn.model.name
38
  ] = f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}"
 
 
 
39
 
40
 
41
  def remove_forbidden_models(leaderboard_data: list[dict]):
 
1
+ from pprint import pprint
2
  from src.display.formatting import model_hyperlink
3
  from src.display.utils import AutoEvalColumn
4
 
 
15
  "AIDC-ai-business/Marcoroni-13B": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/287",
16
  "AIDC-ai-business/Marcoroni-7B": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/287",
17
  "fblgit/una-xaberius-34b-v1beta": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/444",
18
+ "jan-hq/trinity-v1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
19
+ "rwitz2/go-bruins-v2.1.1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
20
+ "rwitz2/go-bruins-v2.1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
21
+ "GreenNode/GreenNodeLM-v3olet-7B": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
22
+ "GreenNode/GreenNodeLM-7B-v4leo": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
23
+ "GreenNode/LeoScorpius-GreenNode-7B-v1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
24
+ "viethq188/LeoScorpius-7B-Chat-DPO": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
25
+ "GreenNode/GreenNodeLM-7B-v2leo": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
26
+ "janai-hq/trinity-v1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
27
+ "ignos/LeoScorpius-GreenNode-Alpaca-7B-v1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
28
+ "fblgit/una-cybertron-7b-v3-OMA": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
29
  }
30
 
31
  # Models which have been requested by orgs to not be submitted on the leaderboard
 
48
  model_data[
49
  AutoEvalColumn.model.name
50
  ] = f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}"
51
+ model_data[AutoEvalColumn.flagged.name] = True
52
+ else:
53
+ model_data[AutoEvalColumn.flagged.name] = False
54
 
55
 
56
  def remove_forbidden_models(leaderboard_data: list[dict]):
src/populate.py CHANGED
@@ -1,5 +1,6 @@
1
  import json
2
  import os
 
3
 
4
  import pandas as pd
5
 
 
1
  import json
2
  import os
3
+ from pprint import pprint
4
 
5
  import pandas as pd
6