Update app.py

app.py

@@ -2,16 +2,17 @@ import gradio as gr
 import pandas as pd
 import json
 from constants import BANNER, INTRODUCTION_TEXT, CITATION_TEXT, METRICS_TAB_TEXT, DIR_OUTPUT_REQUESTS
-from init import is_model_on_hub, upload_file, load_all_info_from_dataset_hub
+from init import is_model_on_hub, load_all_info_from_dataset_hub
 from utils_display import AutoEvalColumn, fields, make_clickable_model, styled_error, styled_message
 from datetime import datetime, timezone
+import torch
 
 LAST_UPDATED = "OCT 2nd 2024"
 
 column_names = {
-    "Model": "Model",
-    "WER": "WER",
-    "CER": "CER",
+    "MODEL": "Model",
+    "WER": "WER ⬇️",
+    "CER": "CER ⬇️",
 }
 
 # Load evaluation results
@@ -31,13 +32,13 @@ def formatter(x):
         return round(x, 2)
 
 for col in original_df.columns:
-    if col == "Model":
+    if col == "model":
         original_df[col] = original_df[col].apply(lambda x: x.replace(x, make_clickable_model(x)))
     else:
         original_df[col] = original_df[col].apply(formatter)
 
 original_df.rename(columns=column_names, inplace=True)
-original_df.sort_values(by='WER', inplace=True)
+original_df.sort_values(by='WER ⬇️', inplace=True)
 
 COLS = [c.name for c in fields(AutoEvalColumn)]
 TYPES = [c.type for c in fields(AutoEvalColumn)]
@@ -49,39 +50,82 @@ def request_model(model_text):
     if not base_model_on_hub:
         return styled_error(f"Base model '{model_text}' {error_msg}")
 
-    # Construct the output dictionary
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-    eval_entry = {
-        "date": current_time,
-        "model": model_text,
-        "dataset": "vargha/common_voice_fa"
-    }
+    # Check if the model has already been evaluated
+    if model_text in original_df['Model'].apply(lambda x: x.split('href="')[1].split('"')[0].replace('https://huggingface.co/', '')).values:
+        return styled_error(f"The model '{model_text}' is already in the leaderboard.")
 
-    # Prepare file path
-    DIR_OUTPUT_REQUESTS.mkdir(parents=True, exist_ok=True)
-
-    filename = model_text.replace("/", "@")
-    if filename in requested_models:
-        return styled_error(f"A request for this model '{model_text}' was already made.")
     try:
-        filename_ext = filename + ".txt"
-        out_filepath = DIR_OUTPUT_REQUESTS / filename_ext
-
-        # Write the results to a text file
-        with open(out_filepath, "w") as f:
-            f.write(json.dumps(eval_entry))
-
-        upload_file(filename, out_filepath)
-
-        # Include file in the list of uploaded files
-        requested_models.append(filename)
-
-        # Remove the local file
-        out_filepath.unlink()
+        # Run the evaluation code
+        from transformers import pipeline
+        from transformers.utils import is_flash_attn_2_available
+        from datasets import load_dataset
+        from tqdm import tqdm
+        from transformers.pipelines.pt_utils import KeyDataset
+        from evaluate import load
+
+        # Load a subset of the Common Voice test dataset for evaluation
+        common_voice_test = load_dataset(
+            "mozilla-foundation/common_voice_17_0", "fa", split="test"
+        ).shuffle(seed=42).select(range(len(load_dataset(
+            "mozilla-foundation/common_voice_17_0", "fa", split="test")) // 15))
+
+        # Initialize the pipeline with the requested model
+        pipe = pipeline(
+            "automatic-speech-recognition",
+            model=model_text,
+            torch_dtype=torch.float32,
+            device=0 if torch.cuda.is_available() else -1,  # Use GPU if available
+        )
+
+        all_predictions = []
+
+        # Run inference
+        for prediction in tqdm(
+            pipe(
+                KeyDataset(common_voice_test, "audio"),
+                max_new_tokens=128,
+                chunk_length_s=30,
+                generate_kwargs={"task": "transcribe"},
+                batch_size=32,
+            ),
+            total=len(common_voice_test),
+        ):
+            all_predictions.append(prediction["text"])
+
+        wer_metric = load("wer")
+        cer_metric = load("cer")
+
+        wer_result = 100 * wer_metric.compute(
+            references=common_voice_test["sentence"], predictions=all_predictions
+        )
+
+        cer_result = 100 * cer_metric.compute(
+            references=common_voice_test["sentence"], predictions=all_predictions
+        )
+
+        # Update the results CSV
+        new_row = {'model': model_text, 'wer': wer_result, 'cer': cer_result}
+        df_results = pd.read_csv(csv_results)
+        df_results = df_results.append(new_row, ignore_index=True)
+        df_results.to_csv(csv_results, index=False)
+
+        # Update the leaderboard DataFrame
+        global original_df
+        original_df = df_results.copy()
+        original_df['Model'] = original_df['model'].apply(lambda x: make_clickable_model(x))
+        original_df['WER ⬇️'] = original_df['wer'].apply(lambda x: round(x, 2))
+        original_df['CER ⬇️'] = original_df['cer'].apply(lambda x: round(x, 2))
+        original_df = original_df[['Model', 'WER ⬇️', 'CER ⬇️']]
+        original_df.sort_values(by='WER ⬇️', inplace=True)
+
+        # Update the leaderboard table in the UI
+        leaderboard_table.update(value=original_df)
+
+        # Return success message
+        return styled_message("🤗 Your model has been evaluated and added to the leaderboard!")
 
-        return styled_message("🤗 Your request has been submitted and will be evaluated soon!</p>")
     except Exception as e:
-        return styled_error(f"Error submitting request: {e}")
+        return styled_error(f"Error during evaluation: {e}")
 
 with gr.Blocks() as demo:
     gr.HTML(BANNER, elem_id="banner")
@@ -89,7 +133,7 @@ with gr.Blocks() as demo:
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("🏅 Leaderboard", elem_id="od-benchmark-tab-table", id=0):
-            leaderboard_table = gr.components.Dataframe(
+            leaderboard_table = gr.Dataframe(
                 value=original_df,
                 datatype=TYPES,
                 elem_id="leaderboard-table",