app.py

import gradio as gr
from huggingface_hub import HfApi, hf_hub_download
from huggingface_hub.repocard import metadata_load
import requests
import re
import pandas as pd
from huggingface_hub import ModelCard


def pass_emoji(passed):
    if passed is True:
        passed = "✅"
    else:
        passed = "❌"
    return passed

api = HfApi()


def get_user_audio_classification_models(hf_username):
    """
    List the user's Audio Classification models
    :param hf_username: User HF username
    """

    models = api.list_models(author=hf_username, filter=["audio-classification"])
    user_model_ids = [x.modelId for x in models]
    models_gtzan = []

    for model in user_model_ids:
        meta = get_metadata(model)
        if meta is None:
            continue
        try:
          if meta["datasets"] == ['marsyas/gtzan']:
              models_gtzan.append(model)
        except: continue
    return models_gtzan


def get_metadata(model_id):
  """
  Get model metadata (contains evaluation data)
  :param model_id
  """
  try:
    readme_path = hf_hub_download(model_id, filename="README.md")
    return metadata_load(readme_path)
  except requests.exceptions.HTTPError:
    # 404 README.md not found
    return None


def extract_accuracy(model_card_content):
    """
    Extract the accuracy value from the models' model card
    :param model_card_content: model card content
    """
    accuracy_pattern = r"Accuracy: (\d+\.\d+)"
    match = re.search(accuracy_pattern, model_card_content)
    if match:
        accuracy = match.group(1)
        return float(accuracy)
    else:
        return None


def parse_metrics_accuracy(model_id):
  """
  Get model card and parse it
  :param model_id: model id
  """
  card = ModelCard.load(model_id)
  return extract_accuracy(card.content)

  
def calculate_best_acc_result(user_model_ids):
  """
  Calculate the best results of a unit
  :param user_model_ids: RL models of a user
  """

  best_result = -100
  best_model = ""

  for model in user_model_ids:
    meta = get_metadata(model)
    if meta is None:
      continue
    accuracy = parse_metrics_accuracy(model)
    if accuracy > best_result:
      best_result = accuracy
      best_model = meta['model-index'][0]["name"]
      
  return best_result, best_model


def certification(hf_username):
  results_certification = [
      {
          "unit": "Unit 4: Audio Classification",
          "task": "audio-classification",
          "baseline_metric": 0.87,
          "best_result": 0,
          "best_model_id": "",
          "passed_": False
      },
  {
          "unit": "Unit 5: TBD",
          "task": "TBD",          
          "baseline_metric": 0.99,
          "best_result": 0,
          "best_model_id": "",
          "passed_": False
  },
  {
          "unit": "Unit 6: TBD",
          "task": "TBD",
          "baseline_metric": 0.99,
          "best_result": 0,
          "best_model_id": "",
          "passed_": False
  },
  {
          "unit": "Unit 7: TBD",
          "task": "TBD",
          "baseline_metric": 0.99,
          "best_result": 0,
          "best_model_id": "",
          "passed_": False
  },
  ] 

  for unit in results_certification:
    unit["passed"] = pass_emoji(unit["passed_"])
    if unit["task"] == "audio-classification":
      user_models = get_user_audio_classification_models(hf_username)
      best_result, best_model_id = calculate_best_acc_result(user_models)
      unit["best_result"] = best_result
      unit["best_model_id"] = best_model_id
      if unit["best_result"] >= unit["baseline_metric"]: 
        unit["passed_"] = True
        unit["passed"] = pass_emoji(unit["passed_"])
    else: 
      # TBD for other units
      continue        
    
  print(results_certification)
 
  df = pd.DataFrame(results_certification)
  df = df[['passed', 'unit', 'task', 'baseline_metric', 'best_result', 'best_model_id']]
  return df


with gr.Blocks() as demo:
    gr.Markdown(f"""
    # 🏆 Check your progress in the Audio Course 🏆
    
    - To get a certificate of completion, you must **pass 3 out of 4 assignments before July 31st 2023**.
    - To get an honors certificate, you must **pass 4 out of 4 assignments before July 31st 2023**.

    To pass an assignment, your model's metric should be equal to or higher than the baseline metric.
    
    Make sure that you have uploaded your model(s) to Hub and type your Hugging Face Username here to check if you pass (in my case MariaK)
    """)
    
    hf_username = gr.Textbox(placeholder="MariaK", label="Your Hugging Face Username")
    check_progress_button = gr.Button(value="Check my progress")
    output = gr.components.Dataframe(value=certification(hf_username))    
    check_progress_button.click(fn=certification, inputs=hf_username, outputs=output)

demo.launch()