Spaces:
Runtime error
Runtime error
import pandas as pd | |
import streamlit as st | |
from huggingface_hub import HfApi | |
from utils import ascending_metrics, metric_ranges, LANGUAGES | |
import numpy as np | |
from st_aggrid import AgGrid, GridOptionsBuilder, JsCode, ColumnsAutoSizeMode | |
from os.path import exists | |
import threading | |
st.set_page_config(layout="wide") | |
def get_model_infos(): | |
api = HfApi() | |
model_infos = api.list_models(filter="model-index", cardData=True) | |
return model_infos | |
def parse_metric_value(value): | |
if isinstance(value, str): | |
"".join(value.split("%")) | |
try: | |
value = float(value) | |
except: # noqa: E722 | |
value = None | |
elif isinstance(value, list): | |
if len(value) > 0: | |
value = value[0] | |
else: | |
value = None | |
value = round(value, 4) if isinstance(value, float) else None | |
return value | |
def parse_metrics_rows(meta, only_verified=False): | |
if not isinstance(meta["model-index"], list) or len(meta["model-index"]) == 0 or "results" not in meta["model-index"][0]: | |
return None | |
for result in meta["model-index"][0]["results"]: | |
if not isinstance(result, dict) or "dataset" not in result or "metrics" not in result or "type" not in result["dataset"]: | |
continue | |
dataset = result["dataset"]["type"] | |
if dataset == "": | |
continue | |
row = {"dataset": dataset, "split": "-unspecified-", "config": "-unspecified-"} | |
if "split" in result["dataset"]: | |
row["split"] = result["dataset"]["split"] | |
if "config" in result["dataset"]: | |
row["config"] = result["dataset"]["config"] | |
no_results = True | |
incorrect_results = False | |
for metric in result["metrics"]: | |
name = metric["type"].lower().strip() | |
if name in ("model_id", "dataset", "split", "config", "pipeline_tag", "only_verified"): | |
# Metrics are not allowed to be named "dataset", "split", "config", "pipeline_tag" | |
continue | |
value = parse_metric_value(metric.get("value", None)) | |
if value is None: | |
continue | |
if name in row: | |
new_metric_better = value < row[name] if name in ascending_metrics else value > row[name] | |
if name not in row or new_metric_better: | |
# overwrite the metric if the new value is better. | |
if only_verified: | |
if "verified" in metric and metric["verified"]: | |
no_results = False | |
row[name] = value | |
if name in metric_ranges: | |
if value < metric_ranges[name][0] or value > metric_ranges[name][1]: | |
incorrect_results = True | |
else: | |
no_results = False | |
row[name] = value | |
if name in metric_ranges: | |
if value < metric_ranges[name][0] or value > metric_ranges[name][1]: | |
incorrect_results = True | |
if no_results or incorrect_results: | |
continue | |
yield row | |
def get_data_wrapper(): | |
def get_data(dataframe=None, verified_dataframe=None): | |
data = [] | |
verified_data = [] | |
print("getting model infos") | |
model_infos = get_model_infos() | |
print("got model infos") | |
for model_info in model_infos: | |
meta = model_info.cardData | |
if meta is None: | |
continue | |
for row in parse_metrics_rows(meta): | |
if row is None: | |
continue | |
row["model_id"] = model_info.id | |
row["pipeline_tag"] = model_info.pipeline_tag | |
row["only_verified"] = False | |
data.append(row) | |
for row in parse_metrics_rows(meta, only_verified=True): | |
if row is None: | |
continue | |
row["model_id"] = model_info.id | |
row["pipeline_tag"] = model_info.pipeline_tag | |
row["only_verified"] = True | |
data.append(row) | |
dataframe = pd.DataFrame.from_records(data) | |
dataframe.to_pickle("cache.pkl") | |
if exists("cache.pkl"): | |
# If we have saved the results previously, call an asynchronous process | |
# to fetch the results and update the saved file. Don't make users wait | |
# while we fetch the new results. Instead, display the old results for | |
# now. The new results should be loaded when this method | |
# is called again. | |
dataframe = pd.read_pickle("cache.pkl") | |
t = threading.Thread(name="get_data procs", target=get_data) | |
t.start() | |
else: | |
# We have to make the users wait during the first startup of this app. | |
get_data() | |
dataframe = pd.read_pickle("cache.pkl") | |
return dataframe | |
dataframe = get_data_wrapper() | |
st.markdown("# ๐ค Whisper Event: Final Leaderboard") | |
query_params = st.experimental_get_query_params() | |
if "first_query_params" not in st.session_state: | |
st.session_state.first_query_params = query_params | |
first_query_params = st.session_state.first_query_params | |
default_config = first_query_params.get("config", [None])[0] | |
default_metric = first_query_params.get("metric", [None])[0] | |
only_verified_results = False | |
task = "automatic-speech-recognition" | |
dataset = "mozilla-foundation/common_voice_11_0" | |
split = "test" | |
dataframe = dataframe[dataframe.only_verified == only_verified_results] | |
current_query_params = {"dataset": [dataset], "only_verified": [int(only_verified_results)], "task": [task], | |
"split": [split]} | |
st.experimental_set_query_params(**current_query_params) | |
dataset_df = dataframe[dataframe.dataset == dataset] | |
dataset_df = dataset_df[dataset_df.split == split] | |
dataset_df = dataset_df.dropna(axis="columns", how="all") | |
selectable_metrics = ["wer", "cer"] | |
sorting_metric = "wer" | |
selectable_configs = list(set(dataset_df["config"])) | |
selectable_configs.sort(key=lambda name: name.lower()) | |
selectable_configs.remove("-unspecified-") | |
selectable_configs = [config for config in selectable_configs if config in LANGUAGES] | |
visual_configs = [f"{config}: {LANGUAGES[config]}" for config in selectable_configs] | |
dataset_df = dataset_df[["config", "model_id"] + selectable_metrics] | |
all_ds = [] | |
for config in selectable_configs: | |
dataset_df_ds = dataset_df[dataset_df.config == config] | |
dataset_df_ds = dataset_df_ds.dropna(thresh=2) # Want at least two non-na values (one for model_id and one for a metric) | |
# Make the default metric appear right after model names and dataset names | |
cols = dataset_df_ds.columns.tolist() | |
cols.remove(sorting_metric) | |
sorting_metric_index = 1 if dataset != "-any-" else 2 | |
cols = cols[:sorting_metric_index] + [sorting_metric] + cols[sorting_metric_index:] | |
dataset_df_ds = dataset_df_ds[cols] | |
# Sort the leaderboard, giving the sorting metric highest priority and then ordering by other metrics in the case of equal values. | |
dataset_df_ds = dataset_df_ds.sort_values(by=cols[sorting_metric_index:], ascending=[metric in ascending_metrics for metric in cols[sorting_metric_index:]]) | |
dataset_df_ds = dataset_df_ds.replace(np.nan, '-') | |
all_ds.append(dataset_df_ds.iloc[0]) | |
all_ds = pd.DataFrame(all_ds, columns=["config", "model_id", "wer", "cer"]) | |
language_names = [LANGUAGES[config] for config in selectable_configs] | |
all_ds.insert(1, "language", language_names) | |
# Make the leaderboard | |
gb = GridOptionsBuilder.from_dataframe(all_ds) | |
gb.configure_default_column(sortable=False) | |
gb.configure_column( | |
"model_id", | |
cellRenderer=JsCode('''function(params) {return '<a target="_blank" href="https://huggingface.co/'+params.value+'">'+params.value+'</a>'}'''), | |
) | |
for name in selectable_metrics: | |
gb.configure_column(name, type=["numericColumn","numberColumnFilter","customNumericFormat"], precision=2, aggFunc='sum') | |
gb.configure_column( | |
sorting_metric, | |
sortable=True, | |
#cellStyle=JsCode('''function(params) { return {'backgroundColor': '#FFD21E'}}''') | |
) | |
go = gb.build() | |
fit_columns = len(all_ds.columns) < 10 | |
AgGrid(all_ds, gridOptions=go, height=28*len(all_ds) + (35 if fit_columns else 41), allow_unsafe_jscode=True, enable_enterprise_modules=False, columns_auto_size_mode=ColumnsAutoSizeMode.FIT_CONTENTS) | |