import gradio as gr import pandas as pd import json from constants import BANNER, INTRODUCTION_TEXT, CITATION_TEXT, METRICS_TAB_TEXT, DIR_OUTPUT_REQUESTS from init import is_model_on_hub, upload_file, load_all_info_from_dataset_hub from utils_display import AutoEvalColumn, fields, make_clickable_model, styled_error, styled_message from datetime import datetime, timezone LAST_UPDATED = "Sep 9th 2023" column_names = { "MODEL": "Model", "Avg. WER": "Average WER ⬇️", "RTF": "RTF (1e-3) ⬇️", "Common Voice WER": "Common Voice WER ⬇️", "D_AVG_CV_WER": "Delta AVG-CV WER", } # Skipping testings just uing the numbers computed in the original space for my sanity sake # eval_queue_repo, requested_models, csv_results = load_all_info_from_dataset_hub() # if not csv_results.exists(): # raise Exception(f"CSV file {csv_results} does not exist locally") # # Get csv with data and parse columns # original_df = pd.read_csv(csv_results) data = [ ["nvidia/stt_en_fastconformer_transducer_xlarge", 12.3, 8.06, 7.26], ["nvidia/stt_en_fastconformer_transducer_xxlarge", 14.4, 8.07, 6.07], ["openai/whisper-large-v2", 12.7, 8.16, 10.12], ["nvidia/stt_en_fastconformer_ctc_xxlarge", 5, 8.34, 8.31], ["nvidia/stt_en_conformer_ctc_large", 7.5, 8.39, 9.1], ["openai/whisper-medium.en", 10.7, 8.5, 11.96], ["nvidia/stt_en_fastconformer_ctc_xlarge", 2.9, 8.52, 7.51], ["nvidia/stt_en_fastconformer_ctc_large", 1.8, 8.9, 8.56], ["nvidia/stt_en_fastconformer_transducer_large", 10.4, 8.94, 8.04], ["openai/whisper-large", 12.7, 9.2, 10.92], ["nvidia/stt_en_conformer_transducer_large", 21.8, 9.27, 7.36], ["openai/whisper-small.en", 8.3, 9.34, 15.13], ["nvidia/stt_en_conformer_transducer_small", 17.7, 10.81, 14.35], ["openai/whisper-base.en", 7.2, 11.67, 21.77], ["nvidia/stt_en_conformer_ctc_small", 3.2, 11.77, 16.59], ["patrickvonplaten/wav2vec2-large-960h-lv60-self-4-gram", 20.1, 13.65, 20.05], ["facebook/wav2vec2-large-960h-lv60-self", 2.5, 14.47, 22.15], ["openai/whisper-tiny.en", 9.1, 14.96, 31.09], ["patrickvonplaten/hubert-xlarge-ls960-ft-4-gram", 24.5, 15.11, 19.16], ["speechbrain/asr-wav2vec2-librispeech", 2.6, 15.61, 23.71], ["facebook/hubert-xlarge-ls960-ft", 6.3, 15.81, 22.05], ["facebook/mms-1b-all", 5.9, 15.85, 21.23], ["facebook/hubert-large-ls960-ft", 2.6, 15.93, 23.12], ["facebook/wav2vec2-large-robust-ft-libri-960h", 2.7, 16.07, 22.57], ["facebook/wav2vec2-conformer-rel-pos-large-960h-ft", 5.2, 17, 23.01], ["facebook/wav2vec2-conformer-rope-large-960h-ft", 7.8, 17.06, 23.08], ["facebook/wav2vec2-large-960h", 1.8, 21.76, 34.01], ["facebook/wav2vec2-base-960h", 1.2, 26.41, 41.75] ] columns = [ "Model", "RTF (1e-3) ⬇️", "Average WER ⬇️", "Common Voice WER ⬇️" ] original_df = pd.DataFrame(data, columns=columns) # Formats the columns def formatter(x): x = round(x, 2) return x for col in original_df.columns: if col.lower() == "model": original_df[col] = original_df[col].apply(lambda x: x.replace(x, make_clickable_model(x))) else: original_df[col] = original_df[col].apply(formatter) # For numerical values original_df.rename(columns=column_names, inplace=True) original_df.sort_values(by='Common Voice WER', inplace=True) # Compute delta between average WER and CV WER original_df['Detla Avg. C.V. WER'] = original_df['Average WER ⬇️'] - original_df['Common Voice WER ⬇️'] original_df['Detla Avg. C.V. WER'] = pd.to_numeric(original_df['Detla Avg. C.V. WER'], errors='coerce') # Convert to numerical data type original_df['Detla Avg. C.V. WER'] = original_df[col].apply(lambda x: round(x, 2) if not pd.isna(x) else x) # Round and handle NaN values COLS = [c.name for c in fields(AutoEvalColumn)] TYPES = [c.type for c in fields(AutoEvalColumn)] def request_model(model_text, chbcoco2017): # Determine the selected checkboxes dataset_selection = [] if chbcoco2017: dataset_selection.append("ESB Datasets tests only") if len(dataset_selection) == 0: return styled_error("You need to select at least one dataset") base_model_on_hub, error_msg = is_model_on_hub(model_text) if not base_model_on_hub: return styled_error(f"Base model '{model_text}' {error_msg}") # Construct the output dictionary current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") required_datasets = ', '.join(dataset_selection) eval_entry = { "date": current_time, "model": model_text, "datasets_selected": required_datasets } # Prepare file path DIR_OUTPUT_REQUESTS.mkdir(parents=True, exist_ok=True) fn_datasets = '@ '.join(dataset_selection) filename = model_text.replace("/","@") + "@@" + fn_datasets if filename in requested_models: return styled_error(f"A request for this model '{model_text}' and dataset(s) was already made.") try: filename_ext = filename + ".txt" out_filepath = DIR_OUTPUT_REQUESTS / filename_ext # Write the results to a text file with open(out_filepath, "w") as f: f.write(json.dumps(eval_entry)) upload_file(filename, out_filepath) # Include file in the list of uploaded files requested_models.append(filename) # Remove the local file out_filepath.unlink() return styled_message("🤗 Your request has been submitted and will be evaluated soon!

") except Exception as e: return styled_error(f"Error submitting request!") with gr.Blocks() as demo: gr.HTML(BANNER, elem_id="banner") gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") CUSTOM_MESSAGE = """Legend: This space is a fork of the original [hf-audio/open_asr_leaderboard](https://huggingface.co/spaces/hf-audio/open_asr_leaderboard). It aims to demonstrate how the CommonVoice Test Set provides a relatively accurate approximation of the average WER/CER (Word Error Rate/Character Error Rate) at a significantly lower computational cost. Why is this useful? This space is invaluable because it offers a standardized test set for most languages, enabling us to programmatically select a reasonably effective model for any language supported by CommonVoice. Model, RTF (1e-3) ⬇️, and Average WER ⬇️ were sourced from [hf-audio/open_asr_leaderboard](https://huggingface.co/spaces/hf-audio/open_asr_leaderboard) using the version from September 7, 2023. Results The CommonVoice Test provides a Word Error Rate (WER) within a 20-point margin of the average WER. While not perfect, this indicates that CommonVoice can be a useful tool for quickly identifying a suitable ASR model for a wide range of languages in a programmatic manner. However, it's important to note that it is not sufficient as the sole criterion for choosing the most appropriate architecture. Further considerations may be needed depending on the specific requirements of your ASR application. """ gr.Markdown(CUSTOM_MESSAGE, elem_classes="markdown-text") with gr.Tabs(elem_classes="tab-buttons") as tabs: with gr.TabItem("🏅 Leaderboard", elem_id="od-benchmark-tab-table", id=0): leaderboard_table = gr.components.Dataframe( value=original_df, datatype=TYPES, max_rows=None, elem_id="leaderboard-table", interactive=False, visible=True, ) with gr.TabItem("📈 Metrics", elem_id="od-benchmark-tab-table", id=1): gr.Markdown(METRICS_TAB_TEXT, elem_classes="markdown-text") with gr.TabItem("✉️✨ Request a model here!", elem_id="od-benchmark-tab-table", id=2): with gr.Column(): gr.Markdown("# ✉️✨ Request results for a new model here!", elem_classes="markdown-text") with gr.Column(): gr.Markdown("Select a dataset:", elem_classes="markdown-text") with gr.Column(): model_name_textbox = gr.Textbox(label="Model name (user_name/model_name)") chb_coco2017 = gr.Checkbox(label="COCO validation 2017 dataset", visible=False, value=True, interactive=False) with gr.Column(): mdw_submission_result = gr.Markdown() btn_submitt = gr.Button(value="🚀 Request") btn_submitt.click(request_model, [model_name_textbox, chb_coco2017], mdw_submission_result) gr.Markdown(f"Last updated on **{LAST_UPDATED}**", elem_classes="markdown-text") with gr.Row(): with gr.Accordion("📙 Citation", open=False): gr.Textbox( value=CITATION_TEXT, lines=7, label="Copy the BibTeX snippet to cite this source", elem_id="citation-button", ).style(show_copy_button=True) demo.launch()