Spaces:
Running
Running
dummy column refactoring (#688)
Browse files- collection update only happens on full initialization (d131b6cd68b918bb71c70060116d55dbd1d7e4be)
- removed dummy column (ecacc0f636e01ce3fa984e61cab8f4b0f5670af6)
- enhanced naming of dummy column (bab5ced191e1edf26b96473d5d42f51b0bd19784)
- app.py +11 -12
- src/display/css_html_js.py +3 -2
- src/display/utils.py +4 -3
- src/leaderboard/filter_models.py +8 -2
- src/leaderboard/read_evals.py +1 -1
- src/tools/collections.py +2 -2
app.py
CHANGED
@@ -82,10 +82,12 @@ def download_dataset(repo_id, local_dir, repo_type="dataset", max_attempts=3):
|
|
82 |
def init_space(full_init: bool = True):
|
83 |
"""Initializes the application space, loading only necessary data."""
|
84 |
if full_init:
|
|
|
85 |
download_dataset(QUEUE_REPO, EVAL_REQUESTS_PATH)
|
86 |
download_dataset(DYNAMIC_INFO_REPO, DYNAMIC_INFO_PATH)
|
87 |
download_dataset(RESULTS_REPO, EVAL_RESULTS_PATH)
|
88 |
|
|
|
89 |
raw_data, original_df = get_leaderboard_df(
|
90 |
results_path=EVAL_RESULTS_PATH,
|
91 |
requests_path=EVAL_REQUESTS_PATH,
|
@@ -93,14 +95,18 @@ def init_space(full_init: bool = True):
|
|
93 |
cols=COLS,
|
94 |
benchmark_cols=BENCHMARK_COLS,
|
95 |
)
|
96 |
-
|
|
|
|
|
|
|
|
|
97 |
leaderboard_df = original_df.copy()
|
98 |
|
|
|
99 |
eval_queue_dfs = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
|
100 |
|
101 |
return leaderboard_df, raw_data, original_df, eval_queue_dfs
|
102 |
|
103 |
-
|
104 |
# Convert the environment variable "LEADERBOARD_FULL_INIT" to a boolean value, defaulting to True if the variable is not set.
|
105 |
# This controls whether a full initialization should be performed.
|
106 |
do_full_init = os.getenv("LEADERBOARD_FULL_INIT", "True") == "True"
|
@@ -148,23 +154,17 @@ def load_query(request: gr.Request): # triggered only once at startup => read q
|
|
148 |
|
149 |
|
150 |
def search_model(df: pd.DataFrame, query: str) -> pd.DataFrame:
|
151 |
-
return df[(df[AutoEvalColumn.
|
152 |
-
|
153 |
|
154 |
def search_license(df: pd.DataFrame, query: str) -> pd.DataFrame:
|
155 |
return df[df[AutoEvalColumn.license.name].str.contains(query, case=False, na=False)]
|
156 |
|
157 |
-
|
158 |
def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
|
159 |
always_here_cols = [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
|
160 |
-
dummy_col = [AutoEvalColumn.
|
161 |
-
# AutoEvalColumn.model_type_symbol.name,
|
162 |
-
# AutoEvalColumn.model.name,
|
163 |
-
# We use COLS to maintain sorting
|
164 |
filtered_df = df[always_here_cols + [c for c in COLS if c in df.columns and c in columns] + dummy_col]
|
165 |
return filtered_df
|
166 |
|
167 |
-
|
168 |
def filter_queries(query: str, df: pd.DataFrame):
|
169 |
tmp_result_df = []
|
170 |
|
@@ -323,14 +323,13 @@ with demo:
|
|
323 |
value=leaderboard_df[
|
324 |
[c.name for c in fields(AutoEvalColumn) if c.never_hidden]
|
325 |
+ shown_columns.value
|
326 |
-
+ [AutoEvalColumn.
|
327 |
],
|
328 |
headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
|
329 |
datatype=TYPES,
|
330 |
elem_id="leaderboard-table",
|
331 |
interactive=False,
|
332 |
visible=True,
|
333 |
-
# column_widths=["2%", "33%"]
|
334 |
)
|
335 |
|
336 |
# Dummy leaderboard for handling the case when the user uses backspace key
|
|
|
82 |
def init_space(full_init: bool = True):
|
83 |
"""Initializes the application space, loading only necessary data."""
|
84 |
if full_init:
|
85 |
+
# These downloads only occur on full initialization
|
86 |
download_dataset(QUEUE_REPO, EVAL_REQUESTS_PATH)
|
87 |
download_dataset(DYNAMIC_INFO_REPO, DYNAMIC_INFO_PATH)
|
88 |
download_dataset(RESULTS_REPO, EVAL_RESULTS_PATH)
|
89 |
|
90 |
+
# Always retrieve the leaderboard DataFrame
|
91 |
raw_data, original_df = get_leaderboard_df(
|
92 |
results_path=EVAL_RESULTS_PATH,
|
93 |
requests_path=EVAL_REQUESTS_PATH,
|
|
|
95 |
cols=COLS,
|
96 |
benchmark_cols=BENCHMARK_COLS,
|
97 |
)
|
98 |
+
|
99 |
+
if full_init:
|
100 |
+
# Collection update only happens on full initialization
|
101 |
+
update_collections(original_df)
|
102 |
+
|
103 |
leaderboard_df = original_df.copy()
|
104 |
|
105 |
+
# Evaluation queue DataFrame retrieval is independent of initialization detail level
|
106 |
eval_queue_dfs = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
|
107 |
|
108 |
return leaderboard_df, raw_data, original_df, eval_queue_dfs
|
109 |
|
|
|
110 |
# Convert the environment variable "LEADERBOARD_FULL_INIT" to a boolean value, defaulting to True if the variable is not set.
|
111 |
# This controls whether a full initialization should be performed.
|
112 |
do_full_init = os.getenv("LEADERBOARD_FULL_INIT", "True") == "True"
|
|
|
154 |
|
155 |
|
156 |
def search_model(df: pd.DataFrame, query: str) -> pd.DataFrame:
|
157 |
+
return df[(df[AutoEvalColumn.fullname.name].str.contains(query, case=False, na=False))]
|
|
|
158 |
|
159 |
def search_license(df: pd.DataFrame, query: str) -> pd.DataFrame:
|
160 |
return df[df[AutoEvalColumn.license.name].str.contains(query, case=False, na=False)]
|
161 |
|
|
|
162 |
def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
|
163 |
always_here_cols = [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
|
164 |
+
dummy_col = [AutoEvalColumn.fullname.name]
|
|
|
|
|
|
|
165 |
filtered_df = df[always_here_cols + [c for c in COLS if c in df.columns and c in columns] + dummy_col]
|
166 |
return filtered_df
|
167 |
|
|
|
168 |
def filter_queries(query: str, df: pd.DataFrame):
|
169 |
tmp_result_df = []
|
170 |
|
|
|
323 |
value=leaderboard_df[
|
324 |
[c.name for c in fields(AutoEvalColumn) if c.never_hidden]
|
325 |
+ shown_columns.value
|
326 |
+
+ [AutoEvalColumn.fullname.name]
|
327 |
],
|
328 |
headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
|
329 |
datatype=TYPES,
|
330 |
elem_id="leaderboard-table",
|
331 |
interactive=False,
|
332 |
visible=True,
|
|
|
333 |
)
|
334 |
|
335 |
# Dummy leaderboard for handling the case when the user uses backspace key
|
src/display/css_html_js.py
CHANGED
@@ -1,4 +1,5 @@
|
|
1 |
custom_css = """
|
|
|
2 |
/* Hides the final AutoEvalColumn */
|
3 |
#llm-benchmark-tab-table table td:last-child,
|
4 |
#llm-benchmark-tab-table table th:last-child {
|
@@ -44,7 +45,7 @@ table th:first-child {
|
|
44 |
background: none;
|
45 |
border: none;
|
46 |
}
|
47 |
-
|
48 |
#search-bar {
|
49 |
padding: 0px;
|
50 |
}
|
@@ -94,4 +95,4 @@ get_window_url_params = """
|
|
94 |
url_params = Object.fromEntries(params);
|
95 |
return url_params;
|
96 |
}
|
97 |
-
"""
|
|
|
1 |
custom_css = """
|
2 |
+
|
3 |
/* Hides the final AutoEvalColumn */
|
4 |
#llm-benchmark-tab-table table td:last-child,
|
5 |
#llm-benchmark-tab-table table th:last-child {
|
|
|
45 |
background: none;
|
46 |
border: none;
|
47 |
}
|
48 |
+
|
49 |
#search-bar {
|
50 |
padding: 0px;
|
51 |
}
|
|
|
95 |
url_params = Object.fromEntries(params);
|
96 |
return url_params;
|
97 |
}
|
98 |
+
"""
|
src/display/utils.py
CHANGED
@@ -71,12 +71,13 @@ auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sh
|
|
71 |
auto_eval_column_dict.append(["flagged", ColumnContent, ColumnContent("Flagged", "bool", False, hidden=True)])
|
72 |
auto_eval_column_dict.append(["moe", ColumnContent, ColumnContent("MoE", "bool", False, hidden=True)])
|
73 |
# Dummy column for the search bar (hidden by the custom CSS)
|
74 |
-
auto_eval_column_dict.append(["
|
75 |
|
76 |
# We use make dataclass to dynamically fill the scores from Tasks
|
77 |
AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
|
78 |
|
79 |
|
|
|
80 |
@dataclass(frozen=True)
|
81 |
class EvalQueueColumn: # Queue column
|
82 |
model = ColumnContent("model", "markdown", True)
|
@@ -99,7 +100,7 @@ baseline_row = {
|
|
99 |
AutoEvalColumn.truthfulqa.name: 25.0,
|
100 |
AutoEvalColumn.winogrande.name: 50.0,
|
101 |
AutoEvalColumn.gsm8k.name: 0.21,
|
102 |
-
AutoEvalColumn.
|
103 |
AutoEvalColumn.model_type.name: "",
|
104 |
AutoEvalColumn.flagged.name: False,
|
105 |
}
|
@@ -124,7 +125,7 @@ human_baseline_row = {
|
|
124 |
AutoEvalColumn.truthfulqa.name: 94.0,
|
125 |
AutoEvalColumn.winogrande.name: 94.0,
|
126 |
AutoEvalColumn.gsm8k.name: 100,
|
127 |
-
AutoEvalColumn.
|
128 |
AutoEvalColumn.model_type.name: "",
|
129 |
AutoEvalColumn.flagged.name: False,
|
130 |
}
|
|
|
71 |
auto_eval_column_dict.append(["flagged", ColumnContent, ColumnContent("Flagged", "bool", False, hidden=True)])
|
72 |
auto_eval_column_dict.append(["moe", ColumnContent, ColumnContent("MoE", "bool", False, hidden=True)])
|
73 |
# Dummy column for the search bar (hidden by the custom CSS)
|
74 |
+
auto_eval_column_dict.append(["fullname", ColumnContent, ColumnContent("fullname", "str", False, dummy=True)])
|
75 |
|
76 |
# We use make dataclass to dynamically fill the scores from Tasks
|
77 |
AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
|
78 |
|
79 |
|
80 |
+
|
81 |
@dataclass(frozen=True)
|
82 |
class EvalQueueColumn: # Queue column
|
83 |
model = ColumnContent("model", "markdown", True)
|
|
|
100 |
AutoEvalColumn.truthfulqa.name: 25.0,
|
101 |
AutoEvalColumn.winogrande.name: 50.0,
|
102 |
AutoEvalColumn.gsm8k.name: 0.21,
|
103 |
+
AutoEvalColumn.fullname.name: "baseline",
|
104 |
AutoEvalColumn.model_type.name: "",
|
105 |
AutoEvalColumn.flagged.name: False,
|
106 |
}
|
|
|
125 |
AutoEvalColumn.truthfulqa.name: 94.0,
|
126 |
AutoEvalColumn.winogrande.name: 94.0,
|
127 |
AutoEvalColumn.gsm8k.name: 100,
|
128 |
+
AutoEvalColumn.fullname.name: "human_baseline",
|
129 |
AutoEvalColumn.model_type.name: "",
|
130 |
AutoEvalColumn.flagged.name: False,
|
131 |
}
|
src/leaderboard/filter_models.py
CHANGED
@@ -130,14 +130,17 @@ DO_NOT_SUBMIT_MODELS = [
|
|
130 |
|
131 |
|
132 |
def flag_models(leaderboard_data: list[dict]):
|
|
|
133 |
for model_data in leaderboard_data:
|
134 |
# Merges and moes are flagged automatically
|
135 |
if model_data[AutoEvalColumn.flagged.name]:
|
136 |
flag_key = "merged"
|
137 |
else:
|
138 |
-
flag_key = model_data[
|
139 |
|
|
|
140 |
if flag_key in FLAGGED_MODELS:
|
|
|
141 |
issue_num = FLAGGED_MODELS[flag_key].split("/")[-1]
|
142 |
issue_link = model_hyperlink(
|
143 |
FLAGGED_MODELS[flag_key],
|
@@ -152,11 +155,13 @@ def flag_models(leaderboard_data: list[dict]):
|
|
152 |
|
153 |
|
154 |
def remove_forbidden_models(leaderboard_data: list[dict]):
|
|
|
155 |
indices_to_remove = []
|
156 |
for ix, model in enumerate(leaderboard_data):
|
157 |
-
if model[
|
158 |
indices_to_remove.append(ix)
|
159 |
|
|
|
160 |
for ix in reversed(indices_to_remove):
|
161 |
leaderboard_data.pop(ix)
|
162 |
return leaderboard_data
|
@@ -165,3 +170,4 @@ def remove_forbidden_models(leaderboard_data: list[dict]):
|
|
165 |
def filter_models_flags(leaderboard_data: list[dict]):
|
166 |
leaderboard_data = remove_forbidden_models(leaderboard_data)
|
167 |
flag_models(leaderboard_data)
|
|
|
|
130 |
|
131 |
|
132 |
def flag_models(leaderboard_data: list[dict]):
|
133 |
+
"""Flags models based on external criteria or flagged status."""
|
134 |
for model_data in leaderboard_data:
|
135 |
# Merges and moes are flagged automatically
|
136 |
if model_data[AutoEvalColumn.flagged.name]:
|
137 |
flag_key = "merged"
|
138 |
else:
|
139 |
+
flag_key = model_data[AutoEvalColumn.fullname.name]
|
140 |
|
141 |
+
print(f"model check: {flag_key}")
|
142 |
if flag_key in FLAGGED_MODELS:
|
143 |
+
print(f"Flagged model: {flag_key}")
|
144 |
issue_num = FLAGGED_MODELS[flag_key].split("/")[-1]
|
145 |
issue_link = model_hyperlink(
|
146 |
FLAGGED_MODELS[flag_key],
|
|
|
155 |
|
156 |
|
157 |
def remove_forbidden_models(leaderboard_data: list[dict]):
|
158 |
+
"""Removes models from the leaderboard based on the DO_NOT_SUBMIT list."""
|
159 |
indices_to_remove = []
|
160 |
for ix, model in enumerate(leaderboard_data):
|
161 |
+
if model[AutoEvalColumn.fullname.name] in DO_NOT_SUBMIT_MODELS:
|
162 |
indices_to_remove.append(ix)
|
163 |
|
164 |
+
# Remove the models from the list
|
165 |
for ix in reversed(indices_to_remove):
|
166 |
leaderboard_data.pop(ix)
|
167 |
return leaderboard_data
|
|
|
170 |
def filter_models_flags(leaderboard_data: list[dict]):
|
171 |
leaderboard_data = remove_forbidden_models(leaderboard_data)
|
172 |
flag_models(leaderboard_data)
|
173 |
+
|
src/leaderboard/read_evals.py
CHANGED
@@ -133,7 +133,7 @@ class EvalResult:
|
|
133 |
AutoEvalColumn.weight_type.name: self.weight_type.value.name,
|
134 |
AutoEvalColumn.architecture.name: self.architecture,
|
135 |
AutoEvalColumn.model.name: make_clickable_model(self.full_model),
|
136 |
-
AutoEvalColumn.
|
137 |
AutoEvalColumn.revision.name: self.revision,
|
138 |
AutoEvalColumn.average.name: average,
|
139 |
AutoEvalColumn.license.name: self.license,
|
|
|
133 |
AutoEvalColumn.weight_type.name: self.weight_type.value.name,
|
134 |
AutoEvalColumn.architecture.name: self.architecture,
|
135 |
AutoEvalColumn.model.name: make_clickable_model(self.full_model),
|
136 |
+
AutoEvalColumn.fullname.name: self.full_model,
|
137 |
AutoEvalColumn.revision.name: self.revision,
|
138 |
AutoEvalColumn.average.name: average,
|
139 |
AutoEvalColumn.license.name: self.license,
|
src/tools/collections.py
CHANGED
@@ -60,7 +60,7 @@ def update_collections(df: DataFrame):
|
|
60 |
for size, interval in intervals.items():
|
61 |
filtered_df = _filter_by_type_and_size(df, model_type, interval)
|
62 |
best_models = list(
|
63 |
-
filtered_df.sort_values(AutoEvalColumn.average.name, ascending=False)[AutoEvalColumn.
|
64 |
)
|
65 |
print(model_type.value.symbol, size, best_models)
|
66 |
_add_models_to_collection(collection, best_models, model_type, size)
|
@@ -73,4 +73,4 @@ def update_collections(df: DataFrame):
|
|
73 |
try:
|
74 |
delete_collection_item(collection_slug=PATH_TO_COLLECTION, item_object_id=item_id, token=H4_TOKEN)
|
75 |
except HfHubHTTPError:
|
76 |
-
continue
|
|
|
60 |
for size, interval in intervals.items():
|
61 |
filtered_df = _filter_by_type_and_size(df, model_type, interval)
|
62 |
best_models = list(
|
63 |
+
filtered_df.sort_values(AutoEvalColumn.average.name, ascending=False)[AutoEvalColumn.fullname.name][:10]
|
64 |
)
|
65 |
print(model_type.value.symbol, size, best_models)
|
66 |
_add_models_to_collection(collection, best_models, model_type, size)
|
|
|
73 |
try:
|
74 |
delete_collection_item(collection_slug=PATH_TO_COLLECTION, item_object_id=item_id, token=H4_TOKEN)
|
75 |
except HfHubHTTPError:
|
76 |
+
continue
|