import pandas as pd import streamlit as st from huggingface_hub import HfApi from utils import ascending_metrics, metric_ranges, CV11_LANGUAGES, FLEURS_LANGUAGES, MLS_LANGUAGES import numpy as np from st_aggrid import AgGrid, GridOptionsBuilder, JsCode from os.path import exists import threading st.set_page_config(layout="wide") def get_model_infos(): api = HfApi() model_infos = api.list_models(filter="model-index", cardData=True) return model_infos def parse_metric_value(value): if isinstance(value, str): "".join(value.split("%")) try: value = float(value) except: # noqa: E722 value = None elif isinstance(value, list): if len(value) > 0: value = value[0] else: value = None value = round(value, 4) if isinstance(value, float) else None return value def parse_metrics_rows(meta, only_verified=False): if not isinstance(meta["model-index"], list) or len(meta["model-index"]) == 0 or "results" not in meta["model-index"][0]: return None for result in meta["model-index"][0]["results"]: if not isinstance(result, dict) or "dataset" not in result or "metrics" not in result or "type" not in result["dataset"]: continue dataset = result["dataset"]["type"] if dataset == "": continue row = {"dataset": dataset, "split": "-unspecified-", "config": "-unspecified-"} if "split" in result["dataset"]: row["split"] = result["dataset"]["split"] if "config" in result["dataset"]: row["config"] = result["dataset"]["config"] no_results = True incorrect_results = False for metric in result["metrics"]: name = metric["type"].lower().strip() if name in ("model_id", "dataset", "split", "config", "pipeline_tag", "only_verified"): # Metrics are not allowed to be named "dataset", "split", "config", "pipeline_tag" continue value = parse_metric_value(metric.get("value", None)) if value is None: continue if name in row: new_metric_better = value < row[name] if name in ascending_metrics else value > row[name] if name not in row or new_metric_better: # overwrite the metric if the new value is better. if only_verified: if "verified" in metric and metric["verified"]: no_results = False row[name] = value if name in metric_ranges: if value < metric_ranges[name][0] or value > metric_ranges[name][1]: incorrect_results = True else: no_results = False row[name] = value if name in metric_ranges: if value < metric_ranges[name][0] or value > metric_ranges[name][1]: incorrect_results = True if no_results or incorrect_results: continue yield row @st.cache(ttl=0) def get_data_wrapper(): def get_data(dataframe=None, verified_dataframe=None): data = [] verified_data = [] print("getting model infos") model_infos = get_model_infos() print("got model infos") for model_info in model_infos: meta = model_info.cardData if meta is None: continue for row in parse_metrics_rows(meta): if row is None: continue row["model_id"] = model_info.id row["pipeline_tag"] = model_info.pipeline_tag row["only_verified"] = False data.append(row) for row in parse_metrics_rows(meta, only_verified=True): if row is None: continue row["model_id"] = model_info.id row["pipeline_tag"] = model_info.pipeline_tag row["only_verified"] = True data.append(row) dataframe = pd.DataFrame.from_records(data) dataframe.to_pickle("cache.pkl") if exists("cache.pkl"): # If we have saved the results previously, call an asynchronous process # to fetch the results and update the saved file. Don't make users wait # while we fetch the new results. Instead, display the old results for # now. The new results should be loaded when this method # is called again. dataframe = pd.read_pickle("cache.pkl") t = threading.Thread(name="get_data procs", target=get_data) t.start() else: # We have to make the users wait during the first startup of this app. get_data() dataframe = pd.read_pickle("cache.pkl") return dataframe dataframe = get_data_wrapper() st.markdown("# 🤗 Whisper Event: Final Leaderboard") # query params are used to refine the browser URL as more options are selected query_params = st.experimental_get_query_params() if "first_query_params" not in st.session_state: st.session_state.first_query_params = query_params first_query_params = st.session_state.first_query_params # define the scope of the leaderboard only_verified_results = False task = "automatic-speech-recognition" selectable_datasets = ["mozilla-foundation/common_voice_11_0", "google/fleurs", "facebook/multilingual_librispeech"] dataset_mapping = {"mozilla-foundation/common_voice_11_0": "Common Voice 11", "google/fleurs": "FLEURS", "facebook/multilingual_librispeech": "MLS"} # get a 'pretty' name for our datasets split = "test" selectable_metrics = ["wer", "cer"] default_metric = selectable_metrics[0] # select dataset from list provided dataset = st.sidebar.selectbox( "Dataset", selectable_datasets, help="Select a dataset to see the leaderboard!" ) dataset_name = dataset_mapping[dataset] # slice dataframe to entries of interest dataframe = dataframe[dataframe.only_verified == only_verified_results] dataset_df = dataframe[dataframe.dataset == dataset] dataset_df = dataset_df[dataset_df.split == split] # hardcoded to "test" dataset_df = dataset_df.dropna(axis="columns", how="all") # get potential dataset configs (languages) selectable_configs = list(set(dataset_df["config"])) selectable_configs.sort(key=lambda name: name.lower()) if "-unspecified-" in selectable_configs: selectable_configs.remove("-unspecified-") if dataset == "mozilla-foundation/common_voice_11_0": selectable_configs = [config for config in selectable_configs if config in CV11_LANGUAGES] visual_configs = [f"{config}: {CV11_LANGUAGES[config]}" for config in selectable_configs] elif dataset == "google/fleurs": selectable_configs = [config for config in selectable_configs if config in FLEURS_LANGUAGES] visual_configs = [f"{config}: {FLEURS_LANGUAGES[config]}" for config in selectable_configs] elif dataset == "facebook/multilingual_librispeech": selectable_configs = [config for config in selectable_configs if config in MLS_LANGUAGES] visual_configs = [f"{config}: {MLS_LANGUAGES[config]}" for config in selectable_configs] config = st.sidebar.selectbox( "Language", visual_configs, help="Filter the results on the current leaderboard by language." ) config, language = config.split(":") # just for show -> we've fixed the split to "test" split = st.sidebar.selectbox( "Split", [split], index=0, help="View the results for the `test` split for evaluation performance.", ) # update browser URL with selections current_query_params = {"dataset": [dataset], "config": [config], "split": split} st.experimental_set_query_params(**current_query_params) dataset_df = dataset_df[dataset_df.config == config] dataset_df = dataset_df.filter(["model_id"] + (["dataset"] if dataset == "-any-" else []) + selectable_metrics) dataset_df = dataset_df.dropna(thresh=2) # Want at least two non-na values (one for model_id and one for a metric). sorting_metric = st.sidebar.radio( "Sorting Metric", selectable_metrics, index=selectable_metrics.index(default_metric) if default_metric in selectable_metrics else 0, help="Select the metric to sort the leaderboard by. Click on the metric name in the leaderboard to reverse the sorting order." ) st.markdown( f"This is the leaderboard for {dataset_name} {language} ({config})." ) st.markdown( "Please click on the model's name to be redirected to its model card." ) st.markdown( "Want to beat the leaderboard? Don't see your model here? Ensure..." ) # Make the default metric appear right after model names and dataset names cols = dataset_df.columns.tolist() cols.remove(sorting_metric) sorting_metric_index = 1 if dataset != "-any-" else 2 cols = cols[:sorting_metric_index] + [sorting_metric] + cols[sorting_metric_index:] dataset_df = dataset_df[cols] # Sort the leaderboard, giving the sorting metric highest priority and then ordering by other metrics in the case of equal values. dataset_df = dataset_df.sort_values(by=cols[sorting_metric_index:], ascending=[metric in ascending_metrics for metric in cols[sorting_metric_index:]]) dataset_df = dataset_df.replace(np.nan, '-') # Make the leaderboard gb = GridOptionsBuilder.from_dataframe(dataset_df) gb.configure_default_column(sortable=False) gb.configure_column( "model_id", cellRenderer=JsCode('''function(params) {return ''+params.value+''}'''), ) for name in selectable_metrics: gb.configure_column(name, type=["numericColumn", "numberColumnFilter", "customNumericFormat"], precision=2, aggFunc='sum') gb.configure_column( sorting_metric, sortable=True, cellStyle=JsCode('''function(params) { return {'backgroundColor': '#FFD21E'}}''') ) go = gb.build() fit_columns = len(dataset_df.columns) < 10 AgGrid(dataset_df, gridOptions=go, height=28*len(dataset_df) + (35 if fit_columns else 41), allow_unsafe_jscode=True, fit_columns_on_grid_load=fit_columns, enable_enterprise_modules=False)