initial commit

README.md

@@ -1,12 +1,14 @@
 ---
 title: Open ASR Leaderboard
-emoji: 🐢
-colorFrom: green
-colorTo: indigo
+emoji: 🏆
+colorFrom: red
+colorTo: blue
 sdk: gradio
-sdk_version: 4.44.0
+sdk_version: 4.41.0
 app_file: app.py
-pinned: false
+pinned: true
+tags:
+  - leaderboard
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,127 @@
+import gradio as gr
+import pandas as pd
+import json
+from constants import BANNER, INTRODUCTION_TEXT, CITATION_TEXT, METRICS_TAB_TEXT, DIR_OUTPUT_REQUESTS
+from init import is_model_on_hub, upload_file, load_all_info_from_dataset_hub
+from utils_display import AutoEvalColumn, fields, make_clickable_model, styled_error, styled_message
+from datetime import datetime, timezone
+from utils_display import make_best_bold
+
+LAST_UPDATED = "Sep 11th 2024"
+
+column_names = {
+    "MODEL": "Model",
+    "Avg. WER": "Average WER ⬇️ ",    
+    "Avg. RTFx": "RTFx ⬆️ ",
+    "AMI WER": "AMI",
+    "Earnings22 WER": "Earnings22",
+    "Gigaspeech WER": "Gigaspeech",
+    "LS Clean WER": "LS Clean",
+    "LS Other WER": "LS Other",
+    "SPGISpeech WER": "SPGISpeech",
+}
+
+original_df = pd.read_csv("data.csv")
+requested_models = []
+
+# Formats the columns
+def formatter(x):
+    if type(x) is str:
+        x = x
+    else: 
+        x = round(x, 2)
+    return x
+
+def format_df(df:pd.DataFrame):
+    for col in df.columns:
+        if col == "model":
+            df[col] = df[col].apply(lambda x: x.replace(x, make_clickable_model(x)))
+        else:
+            df[col] = make_best_bold(df[col], col)
+    return df
+
+original_df = format_df(original_df)
+original_df.rename(columns=column_names, inplace=True)
+original_df.sort_values(by='Average WER ⬇️ ', inplace=True)
+
+COLS = [c.name for c in fields(AutoEvalColumn)]
+TYPES = [c.type for c in fields(AutoEvalColumn)]
+
+def request_model(model_text, chbcoco2017):
+    # ... (keep the existing request_model function as is)
+    pass
+
+def update_table(column_selection):
+    original_df = pd.read_csv("data.csv")
+    
+    if column_selection == "All Columns":
+        new_df = original_df
+    elif column_selection == "Main Metrics":
+        new_df = original_df[["model", "Average WER ⬇️ ", "RTFx ⬆️ "]]
+    elif column_selection == "Narrated":
+        new_df = original_df[["model", "Average WER ⬇️ ", "RTFx ⬆️ ", "LS Clean", "LS Other", "Gigaspeech"]]
+        new_df["Average WER ⬇️ "] = new_df[["LS Clean", "LS Other", "Gigaspeech"]].mean(axis=1).round(2)
+    elif column_selection == "Oratory":
+        new_df = original_df[["model", "Average WER ⬇️ ", "RTFx ⬆️ ", "Tedlium", "SPGISpeech", "Earnings22"]]
+        new_df["Average WER ⬇️ "] = new_df[["Tedlium", "SPGISpeech", "Earnings22"]].mean(axis=1).round(2)
+    elif column_selection == "Spontaneous":
+        new_df = original_df[["model", "Average WER ⬇️ ", "RTFx ⬆️ ", "Gigaspeech", "SPGISpeech", "Earnings22", "AMI"]]
+        new_df["Average WER ⬇️ "] = new_df[["Gigaspeech", "SPGISpeech", "Earnings22", "AMI"]].mean(axis=1).round(2)
+
+
+    new_df = new_df.sort_values(by='Average WER ⬇️ ', ascending=True)
+    new_df = format_df(new_df)
+
+    return new_df
+
+with gr.Blocks() as demo:
+    gr.HTML(BANNER, elem_id="banner")
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🏅 Leaderboard", elem_id="od-benchmark-tab-table", id=0):
+            leaderboard_table = gr.components.Dataframe(
+                value=original_df,
+                datatype=TYPES,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+                height=500,
+            )
+            column_radio = gr.Radio(
+                ["All Columns", "Main Metrics", "Narrated", "Oratory", "Spontaneous"],
+                label="Select columns to display",
+                value="All Columns"
+            )
+        column_radio.change(update_table, inputs=[column_radio], outputs=[leaderboard_table])
+
+        with gr.TabItem("📈 Metrics", elem_id="od-benchmark-tab-table", id=1):
+            gr.Markdown(METRICS_TAB_TEXT, elem_classes="markdown-text")
+
+        with gr.TabItem("✉️✨ Request a model here!", elem_id="od-benchmark-tab-table", id=2):
+            with gr.Column():
+                gr.Markdown("# ✉️✨ Request results for a new model here!", elem_classes="markdown-text")
+            with gr.Column():
+                gr.Markdown("Select a dataset:", elem_classes="markdown-text")
+                with gr.Column():
+                    model_name_textbox = gr.Textbox(label="Model name (user_name/model_name)")
+                    chb_coco2017 = gr.Checkbox(label="COCO validation 2017 dataset", visible=False, value=True, interactive=False)
+                with gr.Column():
+                    mdw_submission_result = gr.Markdown()
+                    btn_submitt = gr.Button(value="🚀 Request")
+                    btn_submitt.click(request_model, 
+                                      [model_name_textbox, chb_coco2017], 
+                                      mdw_submission_result)
+
+    gr.Markdown(f"Last updated on **{LAST_UPDATED}**", elem_classes="markdown-text")
+    
+    with gr.Row():
+        with gr.Accordion("📙 Citation", open=False):
+            gr.Textbox(
+                value=CITATION_TEXT, lines=7,
+                label="Copy the BibTeX snippet to cite this source",
+                elem_id="citation-button",
+                show_copy_button=True,
+            )
+
+demo.launch()

constants.py

@@ -0,0 +1,115 @@
+from pathlib import Path
+
+# Directory where request by models are stored
+DIR_OUTPUT_REQUESTS = Path("requested_models")
+EVAL_REQUESTS_PATH = Path("eval_requests")
+
+##########################
+# Text definitions       #
+##########################
+
+banner_url = "https://huggingface.co/datasets/reach-vb/random-images/resolve/main/asr_leaderboard.png"
+BANNER = f'<div style="display: flex; justify-content: space-around;"><img src="{banner_url}" alt="Banner" style="width: 40vw; min-width: 300px; max-width: 600px;"> </div>'
+
+TITLE = "<html> <head> <style> h1 {text-align: center;} </style> </head> <body> <h1> 🤗 Open Automatic Speech Recognition Leaderboard </b> </body> </html>"
+
+INTRODUCTION_TEXT = "📐 The 🤗 Open ASR Leaderboard ranks and evaluates speech recognition models \
+    on the Hugging Face Hub. \
+    \nWe report the Average [WER](https://huggingface.co/spaces/evaluate-metric/wer) (⬇️ lower the better) and [RTFx](https://github.com/NVIDIA/DeepLearningExamples/blob/master/Kaldi/SpeechRecognition/README.md#metrics) (⬆️ higher the better). Models are ranked based on their Average WER, from lowest to highest. Check the 📈 Metrics tab to understand how the models are evaluated. \
+    \nIf you want results for a model that is not listed here, you can submit a request for it to be included ✉️✨. \
+    \nThe leaderboard currently focuses on English speech recognition, and will be expanded to multilingual evaluation in later versions."
+
+CITATION_TEXT = """@misc{open-asr-leaderboard,
+	title        = {Open Automatic Speech Recognition Leaderboard},
+	author       = {Srivastav, Vaibhav and Majumdar, Somshubra and Koluguri, Nithin and Moumen, Adel and Gandhi, Sanchit and others},
+	year         = 2023,
+	publisher    = {Hugging Face},
+	howpublished = "\\url{https://huggingface.co/spaces/hf-audio/open_asr_leaderboard}"
+}
+"""
+
+METRICS_TAB_TEXT = """
+Here you will find details about the speech recognition metrics and datasets reported in our leaderboard.
+
+## Metrics
+
+Models are evaluated jointly using the Word Error Rate (WER) and Inverse Real Time Factor (RTFx) metrics. The WER metric
+is used to assess the accuracy of a system, and the RTFx the inference speed. Models are ranked in the leaderboard based 
+on their WER, lowest to highest.
+
+Crucially, the WER and RTFx values are computed for the same inference run using a single script. The implication of this is two-fold:
+1. The WER and RTFx values are coupled: for a given WER, one can expect to achieve the corresponding RTFx. This allows the proposer to trade-off lower WER for higher RTFx should they wish.
+2. The WER and RTFx values are averaged over all audios in the benchmark (in the order of thousands of audios).
+
+For details on reproducing the benchmark numbers, refer to the [Open ASR GitHub repository](https://github.com/huggingface/open_asr_leaderboard#evaluate-a-model).
+
+### Word Error Rate (WER)
+
+Word Error Rate is used to measure the **accuracy** of automatic speech recognition systems. It calculates the percentage 
+of words in the system's output that differ from the reference (correct) transcript. **A lower WER value indicates higher accuracy**.
+
+Take the following example:
+
+| Reference:  | the | cat | sat     | on  | the | mat |
+|-------------|-----|-----|---------|-----|-----|-----|
+| Prediction: | the | cat | **sit** | on  | the |     |  |
+| Label:      | ✅   | ✅   | S       | ✅   | ✅   | D   |
+
+Here, we have:
+* 1 substitution ("sit" instead of "sat")
+* 0 insertions
+* 1 deletion ("mat" is missing)
+
+This gives 2 errors in total. To get our word error rate, we divide the total number of errors (substitutions + insertions + deletions) by the total number of words in our
+reference (N), which for this example is 6:
+
+```
+WER = (S + I + D) / N = (1 + 0 + 1) / 6 = 0.333
+```
+
+Giving a WER of 0.33, or 33%. For a fair comparison, we calculate **zero-shot** (i.e. pre-trained models only) *normalised WER* for all the model checkpoints, meaning punctuation and casing is removed from the references and predictions. You can find the evaluation code on our [Github repository](https://github.com/huggingface/open_asr_leaderboard). To read more about how the WER is computed, refer to the [Audio Transformers Course](https://huggingface.co/learn/audio-course/chapter5/evaluation).
+
+### Inverse Real Time Factor (RTFx)
+
+Inverse Real Time Factor is a measure of  the **latency** of automatic speech recognition systems, i.e. how long it takes an 
+model to process a given amount of speech. It is defined as:
+```
+RTFx = (number of seconds of audio inferred) / (compute time in seconds)
+``` 
+
+Therefore, and RTFx of 1 means a system processes speech as fast as it's spoken, while an RTFx of 2 means it takes half the time. 
+Thus, **a higher RTFx value indicates lower latency**.
+
+## How to reproduce our results
+
+The ASR Leaderboard will be a continued effort to benchmark open source/access speech recognition models where possible. 
+Along with the Leaderboard we're open-sourcing the codebase used for running these evaluations.
+For more details head over to our repo at: https://github.com/huggingface/open_asr_leaderboard 
+
+P.S. We'd love to know which other models you'd like us to benchmark next. Contributions are more than welcome! ♥️
+
+## Benchmark datasets
+
+Evaluating Speech Recognition systems is a hard problem. We use the multi-dataset benchmarking strategy proposed in the 
+[ESB paper](https://arxiv.org/abs/2210.13352) to obtain robust evaluation scores for each model.
+
+ESB is a benchmark for evaluating the performance of a single automatic speech recognition (ASR) system across a broad 
+set of speech datasets. It comprises eight English speech recognition datasets, capturing a broad range of domains, 
+acoustic conditions, speaker styles, and transcription requirements. As such, it gives a better indication of how 
+a model is likely to perform on downstream ASR compared to evaluating it on one dataset alone.
+
+The ESB score is calculated as a macro-average of the WER scores across the ESB datasets. The models in the leaderboard
+are ranked based on their average WER scores, from lowest to highest.
+
+| Dataset                                                                                 | Domain                      | Speaking Style        | Train (h) | Dev (h) | Test (h) | Transcriptions     | License         |
+|-----------------------------------------------------------------------------------------|-----------------------------|-----------------------|-----------|---------|----------|--------------------|-----------------|
+| [LibriSpeech](https://huggingface.co/datasets/librispeech_asr)                          | Audiobook                   | Narrated              | 960       | 11      | 11       | Normalised         | CC-BY-4.0       |
+| [VoxPopuli](https://huggingface.co/datasets/facebook/voxpopuli)                         | European Parliament         | Oratory               | 523       | 5       | 5        | Punctuated         | CC0             |
+| [TED-LIUM](https://huggingface.co/datasets/LIUM/tedlium)                                | TED talks                   | Oratory               | 454       | 2       | 3        | Normalised         | CC-BY-NC-ND 3.0 |
+| [GigaSpeech](https://huggingface.co/datasets/speechcolab/gigaspeech)                    | Audiobook, podcast, YouTube | Narrated, spontaneous | 2500      | 12      | 40       | Punctuated         | apache-2.0      |
+| [SPGISpeech](https://huggingface.co/datasets/kensho/spgispeech)                         | Financial meetings          | Oratory, spontaneous  | 4900      | 100     | 100      | Punctuated & Cased | User Agreement  |
+| [Earnings-22](https://huggingface.co/datasets/revdotcom/earnings22)                     | Financial meetings          | Oratory, spontaneous  | 105       | 5       | 5        | Punctuated & Cased | CC-BY-SA-4.0    |
+| [AMI](https://huggingface.co/datasets/edinburghcstr/ami)                                | Meetings                    | Spontaneous           | 78        | 9       | 9        | Punctuated & Cased | CC-BY-4.0       |
+
+For more details on the individual datasets and how models are evaluated to give the ESB score, refer to the [ESB paper](https://arxiv.org/abs/2210.13352).
+"""

data.csv

@@ -0,0 +1,35 @@
+model,Average WER ⬇️ ,RTFx ⬆️ ,AMI,Earnings22,Gigaspeech,LS Clean,LS Other,SPGISpeech,Tedlium
+nvidia/canary-1b,6.5,235.34,13.9,12.19,10.12,1.48,2.93,2.06,3.56
+nvidia/parakeet-tdt-1.1b,7.01,2390.61,15.87,14.49,9.52,1.4,2.6,3.16,3.59
+nvidia/parakeet-rnnt-1.1b,7.12,2053.15,17.01,13.94,9.89,1.45,2.5,2.93,3.83
+nvidia/parakeet-ctc-1.1b,7.4,2728.52,15.67,13.75,10.28,1.83,3.51,4.02,3.57
+openai/whisper-large-v3,7.44,145.51,15.95,11.29,10.02,2.01,3.91,2.94,3.86
+nvidia/parakeet-rnnt-0.6b,7.5,2815.72,17.4,14.66,10.01,1.62,3.02,3.32,3.85
+distil-whisper/distil-large-v3,7.52,214.42,15.16,11.79,10.08,2.54,5.19,3.27,3.86
+nvidia/parakeet-ctc-0.6b,7.69,4281.53,16.46,14.26,10.39,1.88,3.8,3.89,3.77
+openai/whisper-large-v2,7.83,144.45,16.74,12.05,10.67,2.83,5.14,3.87,3.9
+distil-whisper/distil-large-v2,7.92,202.95,14.67,12.19,10.32,2.94,6.84,3.3,4.87
+openai/whisper-large,7.94,143.76,16.73,12.91,10.76,2.73,5.54,3.2,3.91
+openai/whisper-medium.en,8.09,182.13,16.68,12.63,11.03,3.02,5.85,3.33,4.12
+nvidia/stt_en_conformer_ctc_large,8.32,4295.01,15.95,15.99,11.6,2.05,4.15,5.57,4.41
+distil-whisper/distil-small.en,8.57,331.89,16.16,13.15,10.87,3.48,7.73,3.82,4.54
+openai/whisper-small.en,8.59,268.91,17.93,12.97,11.35,3.05,7.25,3.6,4.07
+distil-whisper/distil-medium.en,8.77,279.73,16.12,12.99,11.3,3.69,8.35,3.83,4.84
+nvidia/stt_en_fastconformer_ctc_large,8.96,6399.25,18.61,18.81,12.17,1.93,4.04,5.06,4.76
+nvidia/stt_en_fastconformer_transducer_large,9.06,4097.43,19.09,19.41,12.31,1.8,3.97,4.97,4.46
+stt_en_conformer_transducer_small,10.26,3714.36,20.28,18.13,13.7,2.77,6.47,6.63,6.21
+openai/whisper-base.en,10.32,320.67,21.13,15.09,12.83,4.25,10.35,4.26,4.87
+nvidia/stt_en_conformer_ctc_small,11.16,5686.9,20.43,18.84,14.46,3.59,7.92,7.8,7.16
+openai/whisper-tiny.en,12.81,348.12,24.24,19.12,14.08,5.66,15.45,5.93,5.97
+speechbrain/asr-wav2vec2-librispeech,14.35,451.18,32.05,28.52,16.92,1.77,3.83,10.39,7.58
+facebook/wav2vec2-large-960h-lv60-self,21.27,509.32,36.77,31.68,23.94,11.13,12.42,17.94,14.88
+facebook/mms-1b-all,22.54,230.79,42.02,31.17,26.44,12.63,15.99,16.95,17.48
+facebook/hubert-xlarge-ls960-ft,22.55,361.32,39.11,36.13,24.74,11.3,12.22,18.58,15.83
+facebook/hubert-large-ls960-ft,22.69,495.86,39.72,35.24,25.01,11.35,12.75,18.86,15.92
+facebook/wav2vec2-large-robust-ft-libri-960h,22.93,503.81,37.75,36.22,25.12,11.84,13.76,19.03,16.46
+facebook/data2vec-audio-large-960h,23.21,470.15,40.51,37.82,24.8,11.4,12.94,18.49,15.86
+facebook/wav2vec2-conformer-rope-large-960h-ft,23.28,607.87,42.47,37.52,25.0,11.34,12.54,18.87,15.92
+facebook/wav2vec2-conformer-rel-pos-large-960h-ft,23.29,522.46,42.39,38.33,24.96,11.2,12.44,18.85,15.77
+facebook/wav2vec2-large-960h,26.77,516.58,42.66,43.75,27.74,12.81,15.46,22.82,18.85
+facebook/data2vec-audio-base-960h,28.3,648.14,47.27,49.56,29.78,12.13,15.48,25.46,19.49
+facebook/wav2vec2-base-960h,29.4,686.0,45.56,48.47,30.85,12.53,16.72,27.56,21.05

init.py

@@ -0,0 +1,93 @@
+import os 
+from constants import EVAL_REQUESTS_PATH
+from pathlib import Path
+from huggingface_hub import HfApi, Repository
+
+TOKEN_HUB = os.environ.get("TOKEN_HUB", None)
+QUEUE_REPO = os.environ.get("QUEUE_REPO", None)
+QUEUE_PATH = os.environ.get("QUEUE_PATH", None)
+
+hf_api = HfApi(
+    endpoint="https://huggingface.co", 
+    token=TOKEN_HUB, 
+)
+
+def load_all_info_from_dataset_hub():
+    eval_queue_repo = None
+    requested_models = None
+
+    passed = True
+    if TOKEN_HUB is None:
+        passed = False
+    else:
+        print("Pulling evaluation requests and results.")
+
+        eval_queue_repo = Repository(
+            local_dir=QUEUE_PATH,
+            clone_from=QUEUE_REPO,
+            use_auth_token=TOKEN_HUB,
+            repo_type="dataset",
+        )
+        eval_queue_repo.git_pull()
+        
+        # Local directory where dataset repo is cloned + folder with eval requests
+        directory = QUEUE_PATH / EVAL_REQUESTS_PATH
+        requested_models = get_all_requested_models(directory)
+        requested_models = [p.stem for p in requested_models]
+        # Local directory where dataset repo is cloned
+        csv_results = get_csv_with_results(QUEUE_PATH)
+        if csv_results is None:
+            passed = False
+    if not passed:
+        raise ValueError("No Hugging Face token provided. Skipping evaluation requests and results.")
+
+    return eval_queue_repo, requested_models, csv_results
+
+def upload_file(requested_model_name, path_or_fileobj):
+    dest_repo_file = Path(EVAL_REQUESTS_PATH) / path_or_fileobj.name
+    dest_repo_file = str(dest_repo_file)
+    hf_api.upload_file(
+            path_or_fileobj=path_or_fileobj,
+            path_in_repo=str(dest_repo_file),
+            repo_id=QUEUE_REPO,
+            token=TOKEN_HUB,
+            repo_type="dataset",
+            commit_message=f"Add {requested_model_name} to eval queue")
+
+def get_all_requested_models(directory):
+    directory = Path(directory)
+    all_requested_models = list(directory.glob("*.txt"))
+    return all_requested_models
+
+def get_csv_with_results(directory):
+    directory = Path(directory)
+    all_csv_files = list(directory.glob("*.csv"))
+    latest = [f for f in all_csv_files if f.stem.endswith("latest")]
+    if len(latest) != 1:
+        return None
+    return latest[0]
+
+def is_model_on_hub(model_name, revision="main") -> bool:
+
+    try:
+        model_name = model_name.replace(" ","")
+        author = model_name.split("/")[0]
+        model_id = model_name.split("/")[1]
+        if len(author) == 0 or len(model_id) == 0:
+            return False, "is not a valid model name. Please use the format `author/model_name`."
+    except Exception as e:
+        return False, "is not a valid model name. Please use the format `author/model_name`."
+
+    try:
+        models = list(hf_api.list_models(author=author, search=model_id))
+        matched = [model_name for m in models if m.modelId == model_name]
+        if len(matched) != 1:
+            return False, "was not found on the hub!"
+        else:
+            return True, None
+    except Exception as e:
+        print(f"Could not get the model from the hub.: {e}")
+        return False, "was not found on hub!"
+    
+
+

old_app.py

@@ -0,0 +1,159 @@
+import gradio as gr
+import pandas as pd
+import json
+from constants import BANNER, INTRODUCTION_TEXT, CITATION_TEXT, METRICS_TAB_TEXT, DIR_OUTPUT_REQUESTS
+from init import is_model_on_hub, upload_file, load_all_info_from_dataset_hub
+from utils_display import AutoEvalColumn, fields, make_clickable_model, styled_error, styled_message
+from datetime import datetime, timezone
+
+
+
+# imports for updated version
+from utils_display import make_best_bold
+
+
+
+
+
+LAST_UPDATED = "Sep 11th 2024"
+
+column_names = {
+    "MODEL": "Model",
+    "Avg. WER": "Average WER ⬇️",    
+    "Avg. RTFx": "RTFx ⬆️️",
+    "AMI WER": "AMI",
+    "Earnings22 WER": "Earnings22",
+    "Gigaspeech WER": "Gigaspeech",
+    "LS Clean WER": "LS Clean",
+    "LS Other WER": "LS Other",
+    "SPGISpeech WER": "SPGISpeech",
+    # "Tedlium WER": "Tedlium" - just changed for locally running version
+}
+
+# eval_queue_repo, requested_models, csv_results = load_all_info_from_dataset_hub()
+
+# if not csv_results.exists():
+#     raise Exception(f"CSV file {csv_results} does not exist locally")
+    
+# Get csv with data and parse columns
+original_df = pd.read_csv("data.csv")
+
+requested_models = []
+
+# Formats the columns
+def formatter(x):
+    if type(x) is str:
+        x = x
+    else: 
+        x = round(x, 2)
+    return x
+
+for col in original_df.columns:
+    if col == "model":
+        original_df[col] = original_df[col].apply(lambda x: x.replace(x, make_clickable_model(x)))
+    else:
+        original_df[col] = make_best_bold(original_df[col], col) # For numerical values
+
+
+original_df.rename(columns=column_names, inplace=True)
+original_df.sort_values(by='Average WER ⬇️', inplace=True)
+
+COLS = [c.name for c in fields(AutoEvalColumn)]
+TYPES = [c.type for c in fields(AutoEvalColumn)]
+
+
+def request_model(model_text, chbcoco2017):
+    
+    # Determine the selected checkboxes
+    dataset_selection = []
+    if chbcoco2017:
+        dataset_selection.append("ESB Datasets tests only")
+
+    if len(dataset_selection) == 0:
+        return styled_error("You need to select at least one dataset")
+        
+    base_model_on_hub, error_msg = is_model_on_hub(model_text)
+
+    if not base_model_on_hub:
+        return styled_error(f"Base model '{model_text}' {error_msg}")
+    
+    # Construct the output dictionary
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    required_datasets = ', '.join(dataset_selection)
+    eval_entry = {
+        "date": current_time,
+        "model": model_text,
+        "datasets_selected": required_datasets
+    }
+    
+    # Prepare file path 
+    DIR_OUTPUT_REQUESTS.mkdir(parents=True, exist_ok=True)
+    
+    fn_datasets = '@ '.join(dataset_selection)
+    filename = model_text.replace("/","@") + "@@" + fn_datasets 
+    if filename in requested_models:
+        return styled_error(f"A request for this model '{model_text}' and dataset(s) was already made.")
+    try:
+        filename_ext = filename + ".txt"
+        out_filepath = DIR_OUTPUT_REQUESTS / filename_ext
+
+        # Write the results to a text file
+        with open(out_filepath, "w") as f:
+            f.write(json.dumps(eval_entry))
+            
+        upload_file(filename, out_filepath)
+        
+        # Include file in the list of uploaded files
+        requested_models.append(filename)
+        
+        # Remove the local file
+        out_filepath.unlink()
+
+        return styled_message("🤗 Your request has been submitted and will be evaluated soon!</p>")
+    except Exception as e:
+        return styled_error(f"Error submitting request!")
+
+with gr.Blocks() as demo:
+    gr.HTML(BANNER, elem_id="banner")
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🏅 Leaderboard", elem_id="od-benchmark-tab-table", id=0):
+            leaderboard_table = gr.components.Dataframe(
+                value=original_df,
+                datatype=TYPES,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+                )
+
+        with gr.TabItem("📈 Metrics", elem_id="od-benchmark-tab-table", id=1):
+            gr.Markdown(METRICS_TAB_TEXT, elem_classes="markdown-text")
+
+        with gr.TabItem("✉️✨ Request a model here!", elem_id="od-benchmark-tab-table", id=2):
+            with gr.Column():
+                gr.Markdown("# ✉️✨ Request results for a new model here!", elem_classes="markdown-text")
+            with gr.Column():
+                gr.Markdown("Select a dataset:", elem_classes="markdown-text")
+                with gr.Column():
+                    model_name_textbox = gr.Textbox(label="Model name (user_name/model_name)")
+                    chb_coco2017 = gr.Checkbox(label="COCO validation 2017 dataset", visible=False, value=True, interactive=False)
+                with gr.Column():
+                    mdw_submission_result = gr.Markdown()
+                    btn_submitt = gr.Button(value="🚀 Request")
+                    btn_submitt.click(request_model, 
+                                      [model_name_textbox, chb_coco2017], 
+                                      mdw_submission_result)
+
+    gr.Markdown(f"Last updated on **{LAST_UPDATED}**", elem_classes="markdown-text")
+    
+    with gr.Row():
+        with gr.Accordion("📙 Citation", open=False):
+            gr.Textbox(
+                value=CITATION_TEXT, lines=7,
+                label="Copy the BibTeX snippet to cite this source",
+                elem_id="citation-button",
+                show_copy_button=True,
+            )
+
+demo.launch()

requirements.txt

@@ -0,0 +1,61 @@
+aiohttp==3.8.4
+aiosignal==1.3.1
+async-timeout==4.0.2
+attrs==23.1.0
+certifi==2023.7.22
+charset-normalizer==3.2.0
+cmake==3.26.4
+contourpy==1.1.0
+Cython==3.0.0
+datasets==2.13.1
+dill==0.3.6
+filelock==3.12.2
+fonttools==4.40.0
+frozenlist==1.4.0
+fsspec==2023.6.0
+huggingface-hub==0.16.4
+idna==3.4
+Jinja2==3.1.2
+kiwisolver==1.4.4
+lit==16.0.6
+MarkupSafe==2.1.3
+matplotlib==3.7.2
+mpmath==1.3.0
+multidict==6.0.4
+multiprocess==0.70.14
+networkx==3.1
+numpy==1.25.2
+nvidia-cublas-cu11==11.10.3.66
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+nvidia-cuda-runtime-cu11==11.7.99
+nvidia-cudnn-cu11==8.5.0.96
+nvidia-cufft-cu11==10.9.0.58
+nvidia-curand-cu11==10.2.10.91
+nvidia-cusolver-cu11==11.4.0.1
+nvidia-cusparse-cu11==11.7.4.91
+nvidia-nccl-cu11==2.14.3
+nvidia-nvtx-cu11==11.7.91
+packaging==23.1
+pandas==2.0.3
+Pillow==10.0.0
+pyarrow==12.0.1
+python-dateutil==2.8.2
+pytz==2023.3
+PyYAML==6.0.1
+regex==2023.6.3
+requests==2.31.0
+responses==0.18.0
+safetensors==0.3.1
+six==1.16.0
+sympy==1.12
+tokenizers==0.13.3
+torch==2.0.1
+torchvision==0.15.2
+tqdm==4.65.0
+triton==2.0.0
+typing_extensions==4.7.1
+tzdata==2023.3
+urllib3==2.0.4
+xxhash==3.2.0
+yarl==1.9.2

utils_display.py

@@ -0,0 +1,48 @@
+from dataclasses import dataclass
+
+# These classes are for user facing column names, to avoid having to change them 
+# all around the code when a modif is needed 
+@dataclass
+class ColumnContent:
+    name: str
+    type: str 
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+@dataclass(frozen=True)
+class AutoEvalColumn: # Auto evals column
+    model = ColumnContent("Model", "markdown")
+    avg_wer = ColumnContent("Average WER ⬇️ ", "markdown")    
+    rtf = ColumnContent("RTFx ⬆️ ", "markdown")
+    ami_wer = ColumnContent("AMI", "markdown")
+    e22_wer = ColumnContent("Earnings22", "markdown")
+    gs_wer = ColumnContent("Gigaspeech", "markdown")
+    lsc_wer = ColumnContent("LS Clean", "markdown")
+    lso_wer = ColumnContent("LS Other", "markdown")
+    ss_wer = ColumnContent("SPGISpeech", "markdown")
+    tl_wer = ColumnContent("Tedlium", "markdown")
+    # vp_wer = ColumnContent("Voxpopuli", "markdown")
+
+
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+
+def make_best_bold(column, column_name):
+    if column_name == "RTFx ⬆️ ":
+        best_value = column.max()
+    else:
+        best_value = column.min()
+    return column.apply(lambda x: f'#### <span style="color: red;">{float(x):.2f}</span>' if x == best_value else f"{x:.2f}")
+
+