Duplicate from sanchit-gandhi/leaderboards

Co-authored-by: Sanchit Gandhi <sanchit-gandhi@users.noreply.huggingface.co>

README.md

@@ -0,0 +1,15 @@
+---
+title: Leaderboards
+emoji: 📈
+colorFrom: red
+colorTo: yellow
+sdk: streamlit
+sdk_version: 1.10.0
+app_file: app.py
+pinned: false
+license: apache-2.0
+duplicated_from: sanchit-gandhi/leaderboards
+---
+
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces#reference

app.py

@@ -0,0 +1,250 @@
+import pandas as pd
+import streamlit as st
+from huggingface_hub import HfApi
+from utils import ascending_metrics, metric_ranges, LANGUAGES
+import numpy as np
+from st_aggrid import AgGrid, GridOptionsBuilder, JsCode
+from os.path import exists
+import threading
+
+st.set_page_config(layout="wide")
+
+
+def get_model_infos():
+    api = HfApi()
+    model_infos = api.list_models(filter="model-index", cardData=True)
+    return model_infos
+
+
+def parse_metric_value(value):
+    if isinstance(value, str):
+        "".join(value.split("%"))
+        try:
+            value = float(value)
+        except:  # noqa: E722
+            value = None
+    elif isinstance(value, list):
+        if len(value) > 0:
+            value = value[0]
+        else:
+            value = None
+    value = round(value, 4) if isinstance(value, float) else None
+    return value
+
+
+def parse_metrics_rows(meta, only_verified=False):
+    if not isinstance(meta["model-index"], list) or len(meta["model-index"]) == 0 or "results" not in meta["model-index"][0]:
+        return None
+    for result in meta["model-index"][0]["results"]:
+        if not isinstance(result, dict) or "dataset" not in result or "metrics" not in result or "type" not in result["dataset"]:
+            continue
+        dataset = result["dataset"]["type"]
+        if dataset == "":
+            continue
+        row = {"dataset": dataset, "split": "-unspecified-", "config": "-unspecified-"}
+        if "split" in result["dataset"]:
+            row["split"] = result["dataset"]["split"]
+        if "config" in result["dataset"]:
+            row["config"] = result["dataset"]["config"]
+        no_results = True
+        incorrect_results = False
+        for metric in result["metrics"]:
+            name = metric["type"].lower().strip()
+
+            if name in ("model_id", "dataset", "split", "config", "pipeline_tag", "only_verified"):
+                # Metrics are not allowed to be named "dataset", "split", "config", "pipeline_tag"
+                continue
+            value = parse_metric_value(metric.get("value", None))
+            if value is None:
+                continue
+            if name in row:
+                new_metric_better = value < row[name] if name in ascending_metrics else value > row[name]
+            if name not in row or new_metric_better:
+                # overwrite the metric if the new value is better.
+
+                if only_verified:
+                    if "verified" in metric and metric["verified"]:
+                        no_results = False
+                        row[name] = value
+                        if name in metric_ranges:
+                            if value < metric_ranges[name][0] or value > metric_ranges[name][1]:
+                                incorrect_results = True
+                else:
+                    no_results = False
+                    row[name] = value
+                    if name in metric_ranges:
+                        if value < metric_ranges[name][0] or value > metric_ranges[name][1]:
+                            incorrect_results = True
+        if no_results or incorrect_results:
+            continue
+        yield row
+
+
+@st.cache(ttl=0)
+def get_data_wrapper():
+    def get_data(dataframe=None, verified_dataframe=None):
+        data = []
+        verified_data = []
+        print("getting model infos")
+        model_infos = get_model_infos()
+        print("got model infos")
+        for model_info in model_infos:
+            meta = model_info.cardData
+            if meta is None:
+                continue
+            for row in parse_metrics_rows(meta):
+                if row is None:
+                    continue
+                row["model_id"] = model_info.id
+                row["pipeline_tag"] = model_info.pipeline_tag
+                row["only_verified"] = False
+                data.append(row)
+            for row in parse_metrics_rows(meta, only_verified=True):
+                if row is None:
+                    continue
+                row["model_id"] = model_info.id
+                row["pipeline_tag"] = model_info.pipeline_tag
+                row["only_verified"] = True
+                data.append(row)
+        dataframe = pd.DataFrame.from_records(data)
+        dataframe.to_pickle("cache.pkl")
+
+    if exists("cache.pkl"):
+        # If we have saved the results previously, call an asynchronous process
+        # to fetch the results and update the saved file. Don't make users wait
+        # while we fetch the new results. Instead, display the old results for
+        # now. The new results should be loaded when this method
+        # is called again.
+        dataframe = pd.read_pickle("cache.pkl")
+        t = threading.Thread(name="get_data procs", target=get_data)
+        t.start()
+    else:
+        # We have to make the users wait during the first startup of this app.
+        get_data()
+        dataframe = pd.read_pickle("cache.pkl")
+
+    return dataframe
+
+
+dataframe = get_data_wrapper()
+
+st.markdown("# 🤗 Whisper Event: Final Leaderboard")
+
+query_params = st.experimental_get_query_params()
+if "first_query_params" not in st.session_state:
+    st.session_state.first_query_params = query_params
+first_query_params = st.session_state.first_query_params
+
+default_config = first_query_params.get("config", [None])[0]
+default_metric = first_query_params.get("metric", [None])[0]
+
+only_verified_results = False
+task = "automatic-speech-recognition"
+dataset = "mozilla-foundation/common_voice_11_0"
+split = "test"
+
+dataframe = dataframe[dataframe.only_verified == only_verified_results]
+
+current_query_params = {"dataset": [dataset], "only_verified": [int(only_verified_results)], "task": [task],
+                        "split": [split]}
+
+st.experimental_set_query_params(**current_query_params)
+
+dataset_df = dataframe[dataframe.dataset == dataset]
+dataset_df = dataset_df[dataset_df.split == split]
+
+dataset_df = dataset_df.dropna(axis="columns", how="all")
+
+dataset = st.sidebar.selectbox(
+    "Dataset",
+    [dataset],
+    index=0,
+)
+
+selectable_configs = list(set(dataset_df["config"]))
+selectable_configs.sort(key=lambda name: name.lower())
+selectable_configs.remove("-unspecified-")
+selectable_configs = [config for config in selectable_configs if config in LANGUAGES]
+
+visual_configs = [f"{config}: {LANGUAGES[config]}" for config in selectable_configs]
+
+config = st.sidebar.selectbox(
+    "Language",
+    visual_configs,
+    index=0,
+    help="Filter the results on the current leaderboard by language."
+)
+
+config = config.split(":")[0]
+
+dataset_df = dataset_df[dataset_df.config == config]
+
+split = st.sidebar.selectbox(
+    "Split",
+    [split],
+    index=0,
+)
+
+not_selectable_metrics = ["model_id", "dataset", "split", "config", "pipeline_tag", "only_verified"]
+# also ignore irrelevant ASR metrics
+not_selectable_metrics.extend(["wer_without_norm", "mer"])
+
+selectable_metrics = list(filter(lambda column: column not in not_selectable_metrics, dataset_df.columns))
+
+dataset_df = dataset_df.filter(["model_id"] + (["dataset"] if dataset == "-any-" else []) + selectable_metrics)
+dataset_df = dataset_df.dropna(thresh=2)  # Want at least two non-na values (one for model_id and one for a metric).
+
+sorting_metric = st.sidebar.radio(
+    "Sorting Metric",
+    selectable_metrics,
+    index=selectable_metrics.index(default_metric) if default_metric in selectable_metrics else 0,
+    help="Select the metric to sort the leaderboard by. Click on the metric name in the leaderboard to reverse the sorting order."
+)
+
+current_query_params.update({"metric": [sorting_metric]})
+
+st.experimental_set_query_params(**current_query_params)
+
+st.markdown(
+    f"This is the leaderboard for {LANGUAGES[config]} ({config})."
+)
+
+st.markdown(
+    "Please click on the model's name to be redirected to its model card."
+)
+
+st.markdown(
+    "Want to beat the leaderboard? Don't see your model here? Simply ..."
+)
+
+# Make the default metric appear right after model names and dataset names
+cols = dataset_df.columns.tolist()
+cols.remove(sorting_metric)
+sorting_metric_index = 1 if dataset != "-any-" else 2
+cols = cols[:sorting_metric_index] + [sorting_metric] + cols[sorting_metric_index:]
+dataset_df = dataset_df[cols]
+
+# Sort the leaderboard, giving the sorting metric highest priority and then ordering by other metrics in the case of equal values.
+dataset_df = dataset_df.sort_values(by=cols[sorting_metric_index:], ascending=[metric in ascending_metrics for metric in cols[sorting_metric_index:]])
+dataset_df = dataset_df.replace(np.nan, '-')
+
+# Make the leaderboard
+gb = GridOptionsBuilder.from_dataframe(dataset_df)
+gb.configure_default_column(sortable=False)
+gb.configure_column(
+    "model_id",
+    cellRenderer=JsCode('''function(params) {return '<a target="_blank" href="https://huggingface.co/'+params.value+'">'+params.value+'</a>'}'''),
+)
+
+for name in selectable_metrics:
+    gb.configure_column(name, type=["numericColumn","numberColumnFilter","customNumericFormat"], precision=4, aggFunc='sum')
+
+gb.configure_column(
+    sorting_metric,
+    sortable=True,
+    cellStyle=JsCode('''function(params) { return {'backgroundColor': '#FFD21E'}}''')
+)
+
+go = gb.build()
+fit_columns = len(dataset_df.columns) < 10
+AgGrid(dataset_df, gridOptions=go, height=28*len(dataset_df) + (35 if fit_columns else 41), allow_unsafe_jscode=True, fit_columns_on_grid_load=fit_columns, enable_enterprise_modules=False)

requirements.txt

@@ -0,0 +1,4 @@
+pandas==1.5.1
+huggingface_hub==0.11.1
+numpy==1.23.4
+streamlit-aggrid==0.3.3

utils.py

@@ -0,0 +1,37 @@
+ascending_metrics = {
+    "wer",
+    "cer",
+    "loss",
+    "mae",
+    "mahalanobis",
+    "mse",
+    "perplexity",
+    "ter",
+}
+
+metric_ranges = {
+    "accuracy": (0,1),
+    "precision": (0,1),
+    "recall": (0,1),
+    "macro f1": (0,1),
+    "micro f1": (0,1),
+    "pearson": (-1, 1),
+    "matthews_correlation": (-1, 1),
+    "spearmanr": (-1, 1),
+    "google_bleu": (0, 1),
+    "precision@10": (0, 1),
+    "mae": (0, 1),
+    "mauve": (0, 1),
+    "frontier_integral": (0, 1),
+    "mean_iou": (0, 1),
+    "mean_accuracy": (0, 1),
+    "overall_accuracy": (0, 1),
+    "meteor": (0, 1),
+    "mse": (0, 1),
+    "perplexity": (0, float("inf")),
+    "rogue1": (0, 1),
+    "rogue2": (0, 1),
+    "sari": (0, 100),
+}
+
+LANGUAGES = {'ab': 'Abkhaz', 'ace': 'Acehnese', 'ady': 'Adyghe', 'af': 'Afrikaans', 'am': 'Amharic', 'an': 'Aragonese', 'ar': 'Arabic', 'arn': 'Mapudungun', 'as': 'Assamese', 'ast': 'Asturian', 'az': 'Azerbaijani', 'ba': 'Bashkir', 'bas': 'Basaa', 'be': 'Belarusian', 'bg': 'Bulgarian', 'bn': 'Bengali', 'br': 'Breton', 'bs': 'Bosnian', 'bxr': 'Buryat', 'ca': 'Catalan', 'cak': 'Kaqchikel', 'ckb': 'Central Kurdish', 'cnh': 'Hakha Chin', 'co': 'Corsican', 'cs': 'Czech', 'cv': 'Chuvash', 'cy': 'Welsh', 'da': 'Danish', 'de': 'German', 'dsb': 'Sorbian, Lower', 'dv': 'Dhivehi', 'dyu': 'Dioula', 'el': 'Greek', 'en': 'English', 'eo': 'Esperanto', 'es': 'Spanish', 'et': 'Estonian', 'eu': 'Basque', 'fa': 'Persian', 'ff': 'Fulah', 'fi': 'Finnish', 'fo': 'Faroese', 'fr': 'French', 'fy-NL': 'Frisian', 'ga-IE': 'Irish', 'gl': 'Galician', 'gn': 'Guarani', 'gom': 'Goan Konkani', 'ha': 'Hausa', 'he': 'Hebrew', 'hi': 'Hindi', 'hil': 'Hiligaynon', 'hr': 'Croatian', 'hsb': 'Sorbian, Upper', 'ht': 'Haitian', 'hu': 'Hungarian', 'hy-AM': 'Armenian', 'hyw': 'Armenian Western', 'ia': 'Interlingua', 'id': 'Indonesian', 'ie': 'Interlingue', 'ig': 'Igbo', 'is': 'Icelandic', 'it': 'Italian', 'izh': 'Izhorian', 'ja': 'Japanese', 'jbo': 'Lojban', 'ka': 'Georgian', 'kaa': 'Karakalpak', 'kab': 'Kabyle', 'kbd': 'Kabardian', 'ki': 'Kikuyu', 'kk': 'Kazakh', 'km': 'Khmer', 'kmr': 'Kurmanji Kurdish', 'kn': 'Kannada', 'knn': 'Konkani (Devanagari)', 'ko': 'Korean', 'kpv': 'Komi-Zyrian', 'kw': 'Cornish', 'ky': 'Kyrgyz', 'lb': 'Luxembourgish', 'lg': 'Luganda', 'lij': 'Ligurian', 'ln': 'Lingala', 'lo': 'Lao', 'lt': 'Lithuanian', 'lv': 'Latvian', 'mai': 'Maithili', 'mdf': 'Moksha', 'mg': 'Malagasy', 'mhr': 'Meadow Mari', 'mk': 'Macedonian', 'ml': 'Malayalam', 'mn': 'Mongolian', 'mni': 'Meetei Lon', 'mos': 'Mossi', 'mr': 'Marathi', 'mrj': 'Hill Mari', 'ms': 'Malay', 'mt': 'Maltese', 'my': 'Burmese', 'myv': 'Erzya', 'nan-tw': 'Taiwanese (Minnan)', 'nb-NO': 'Norwegian Bokmål', 'nd': 'IsiNdebele (North)', 'ne-NP': 'Nepali', 'nia': 'Nias', 'nl': 'Dutch', 'nn-NO': 'Norwegian Nynorsk', 'nr': 'IsiNdebele (South)', 'nso': 'Northern Sotho', 'nyn': 'Runyankole', 'oc': 'Occitan', 'om': 'Afaan Ormoo', 'or': 'Odia', 'pa-IN': 'Punjabi', 'pap-AW': 'Papiamento (Aruba)', 'pl': 'Polish', 'ps': 'Pashto', 'pt': 'Portuguese', 'quc': "K'iche'", 'quy': 'Quechua Chanka', 'rm-sursilv': 'Romansh Sursilvan', 'rm-vallader': 'Romansh Vallader', 'ro': 'Romanian', 'ru': 'Russian', 'rw': 'Kinyarwanda', 'sah': 'Sakha', 'sat': 'Santali (Ol Chiki)', 'sc': 'Sardinian', 'scn': 'Sicilian', 'sdh': 'Southern Kurdish', 'shi': 'Shilha', 'si': 'Sinhala', 'sk': 'Slovak', 'skr': 'Saraiki', 'sl': 'Slovenian', 'snk': 'Soninke', 'so': 'Somali', 'sq': 'Albanian', 'sr': 'Serbian', 'ss': 'Siswati', 'st': 'Southern Sotho', 'sv-SE': 'Swedish', 'sw': 'Swahili', 'syr': 'Syriac', 'ta': 'Tamil', 'te': 'Telugu', 'tg': 'Tajik', 'th': 'Thai', 'ti': 'Tigrinya', 'tig': 'Tigre', 'tk': 'Turkmen', 'tl': 'Tagalog', 'tn': 'Setswana', 'tok': 'Toki Pona', 'tr': 'Turkish', 'ts': 'Xitsonga', 'tt': 'Tatar', 'tw': 'Twi', 'ty': 'Tahitian', 'uby': 'Ubykh', 'udm': 'Udmurt', 'ug': 'Uyghur', 'uk': 'Ukrainian', 'ur': 'Urdu', 'uz': 'Uzbek', 've': 'Tshivenda', 'vec': 'Venetian', 'vi': 'Vietnamese', 'vot': 'Votic', 'xh': 'Xhosa', 'yi': 'Yiddish', 'yo': 'Yoruba', 'yue': 'Cantonese', 'zgh': 'Tamazight', 'zh-CN': 'Chinese (China)', 'zh-HK': 'Chinese (Hong Kong)', 'zh-TW': 'Chinese (Taiwan)', 'zu': 'Zulu'}