sanchit-gandhi HF staff commited on
Commit
ac0e546
1 Parent(s): 7b110c0

add fleurs

Browse files
Files changed (1) hide show
  1. app.py +37 -38
app.py CHANGED
@@ -1,7 +1,7 @@
1
  import pandas as pd
2
  import streamlit as st
3
  from huggingface_hub import HfApi
4
- from utils import ascending_metrics, metric_ranges, LANGUAGES
5
  import numpy as np
6
  from st_aggrid import AgGrid, GridOptionsBuilder, JsCode
7
  from os.path import exists
@@ -130,66 +130,69 @@ dataframe = get_data_wrapper()
130
 
131
  st.markdown("# 🤗 Whisper Event: Final Leaderboard")
132
 
 
133
  query_params = st.experimental_get_query_params()
134
  if "first_query_params" not in st.session_state:
135
  st.session_state.first_query_params = query_params
136
  first_query_params = st.session_state.first_query_params
137
 
138
- default_config = first_query_params.get("config", [None])[0]
139
- default_metric = first_query_params.get("metric", [None])[0]
140
-
141
  only_verified_results = False
142
  task = "automatic-speech-recognition"
143
- dataset = "mozilla-foundation/common_voice_11_0"
 
144
  split = "test"
 
 
145
 
146
- dataframe = dataframe[dataframe.only_verified == only_verified_results]
147
-
148
- current_query_params = {"dataset": [dataset], "only_verified": [int(only_verified_results)], "task": [task],
149
- "split": [split]}
150
-
151
- st.experimental_set_query_params(**current_query_params)
152
-
153
- dataset_df = dataframe[dataframe.dataset == dataset]
154
- dataset_df = dataset_df[dataset_df.split == split]
155
-
156
- dataset_df = dataset_df.dropna(axis="columns", how="all")
157
-
158
  dataset = st.sidebar.selectbox(
159
  "Dataset",
160
- [dataset],
161
- index=0,
162
  )
 
163
 
 
 
 
 
 
 
 
164
  selectable_configs = list(set(dataset_df["config"]))
165
  selectable_configs.sort(key=lambda name: name.lower())
166
- selectable_configs.remove("-unspecified-")
167
- selectable_configs = [config for config in selectable_configs if config in LANGUAGES]
168
 
169
- visual_configs = [f"{config}: {LANGUAGES[config]}" for config in selectable_configs]
 
 
 
 
 
 
 
 
170
 
171
  config = st.sidebar.selectbox(
172
  "Language",
173
  visual_configs,
174
- index=0,
175
  help="Filter the results on the current leaderboard by language."
176
  )
 
177
 
178
- config = config.split(":")[0]
179
-
180
- dataset_df = dataset_df[dataset_df.config == config]
181
-
182
  split = st.sidebar.selectbox(
183
  "Split",
184
  [split],
185
  index=0,
 
186
  )
187
 
188
- not_selectable_metrics = ["model_id", "dataset", "split", "config", "pipeline_tag", "only_verified"]
189
- # also ignore irrelevant ASR metrics
190
- not_selectable_metrics.extend(["wer_without_norm", "mer"])
191
 
192
- selectable_metrics = list(filter(lambda column: column not in not_selectable_metrics, dataset_df.columns))
193
 
194
  dataset_df = dataset_df.filter(["model_id"] + (["dataset"] if dataset == "-any-" else []) + selectable_metrics)
195
  dataset_df = dataset_df.dropna(thresh=2) # Want at least two non-na values (one for model_id and one for a metric).
@@ -201,12 +204,8 @@ sorting_metric = st.sidebar.radio(
201
  help="Select the metric to sort the leaderboard by. Click on the metric name in the leaderboard to reverse the sorting order."
202
  )
203
 
204
- current_query_params.update({"metric": [sorting_metric]})
205
-
206
- st.experimental_set_query_params(**current_query_params)
207
-
208
  st.markdown(
209
- f"This is the leaderboard for {LANGUAGES[config]} ({config})."
210
  )
211
 
212
  st.markdown(
@@ -214,7 +213,7 @@ st.markdown(
214
  )
215
 
216
  st.markdown(
217
- "Want to beat the leaderboard? Don't see your model here? Simply ..."
218
  )
219
 
220
  # Make the default metric appear right after model names and dataset names
@@ -237,7 +236,7 @@ gb.configure_column(
237
  )
238
 
239
  for name in selectable_metrics:
240
- gb.configure_column(name, type=["numericColumn","numberColumnFilter","customNumericFormat"], precision=4, aggFunc='sum')
241
 
242
  gb.configure_column(
243
  sorting_metric,
 
1
  import pandas as pd
2
  import streamlit as st
3
  from huggingface_hub import HfApi
4
+ from utils import ascending_metrics, metric_ranges, CV11_LANGUAGES, FLEURS_LANGUAGES
5
  import numpy as np
6
  from st_aggrid import AgGrid, GridOptionsBuilder, JsCode
7
  from os.path import exists
 
130
 
131
  st.markdown("# 🤗 Whisper Event: Final Leaderboard")
132
 
133
+ # query params are used to refine the browser URL as more options are selected
134
  query_params = st.experimental_get_query_params()
135
  if "first_query_params" not in st.session_state:
136
  st.session_state.first_query_params = query_params
137
  first_query_params = st.session_state.first_query_params
138
 
139
+ # define the scope of the leaderboard
 
 
140
  only_verified_results = False
141
  task = "automatic-speech-recognition"
142
+ selectable_datasets = ["mozilla-foundation/common_voice_11_0", "google/fleurs"]
143
+ dataset_mapping = {"mozilla-foundation/common_voice_11_0": "Common Voice 11", "google/fleurs": "FLEURS"} # get a 'pretty' name for our datasets
144
  split = "test"
145
+ selectable_metrics = ["wer", "cer"]
146
+ default_metric = selectable_metrics[0]
147
 
148
+ # select dataset from list provided
 
 
 
 
 
 
 
 
 
 
 
149
  dataset = st.sidebar.selectbox(
150
  "Dataset",
151
+ selectable_datasets,
152
+ help="Select a dataset to see the leaderboard!"
153
  )
154
+ dataset_name = dataset_mapping[dataset]
155
 
156
+ # slice dataframe to entries of interest
157
+ dataframe = dataframe[dataframe.only_verified == only_verified_results]
158
+ dataset_df = dataframe[dataframe.dataset == dataset]
159
+ dataset_df = dataset_df[dataset_df.split == split] # hardcoded to "test"
160
+ dataset_df = dataset_df.dropna(axis="columns", how="all")
161
+
162
+ # get potential dataset configs (languages)
163
  selectable_configs = list(set(dataset_df["config"]))
164
  selectable_configs.sort(key=lambda name: name.lower())
 
 
165
 
166
+ if "-unspecified-" in selectable_configs:
167
+ selectable_configs.remove("-unspecified-")
168
+
169
+ if dataset == "mozilla-foundation/common_voice_11_0":
170
+ selectable_configs = [config for config in selectable_configs if config in CV11_LANGUAGES]
171
+ visual_configs = [f"{config}: {CV11_LANGUAGES[config]}" for config in selectable_configs]
172
+ elif dataset == "google/fleurs":
173
+ selectable_configs = [config for config in selectable_configs if config in FLEURS_LANGUAGES]
174
+ visual_configs = [f"{config}: {FLEURS_LANGUAGES[config]}" for config in selectable_configs]
175
 
176
  config = st.sidebar.selectbox(
177
  "Language",
178
  visual_configs,
 
179
  help="Filter the results on the current leaderboard by language."
180
  )
181
+ config, language = config.split(":")
182
 
183
+ # just for show -> we've fixed the split to "test"
 
 
 
184
  split = st.sidebar.selectbox(
185
  "Split",
186
  [split],
187
  index=0,
188
+ help="View the results for the `test` split for evaluation performance.",
189
  )
190
 
191
+ # update browser URL with selections
192
+ current_query_params = {"dataset": [dataset], "config": [config], "split": split}
193
+ st.experimental_set_query_params(**current_query_params)
194
 
195
+ dataset_df = dataset_df[dataset_df.config == config]
196
 
197
  dataset_df = dataset_df.filter(["model_id"] + (["dataset"] if dataset == "-any-" else []) + selectable_metrics)
198
  dataset_df = dataset_df.dropna(thresh=2) # Want at least two non-na values (one for model_id and one for a metric).
 
204
  help="Select the metric to sort the leaderboard by. Click on the metric name in the leaderboard to reverse the sorting order."
205
  )
206
 
 
 
 
 
207
  st.markdown(
208
+ f"This is the leaderboard for {dataset_name} {language} ({config})."
209
  )
210
 
211
  st.markdown(
 
213
  )
214
 
215
  st.markdown(
216
+ "Want to beat the leaderboard? Don't see your model here? Ensure..."
217
  )
218
 
219
  # Make the default metric appear right after model names and dataset names
 
236
  )
237
 
238
  for name in selectable_metrics:
239
+ gb.configure_column(name, type=["numericColumn", "numberColumnFilter", "customNumericFormat"], precision=4, aggFunc='sum')
240
 
241
  gb.configure_column(
242
  sorting_metric,