|
import gradio as gr |
|
from huggingface_hub import HfApi, hf_hub_download |
|
from huggingface_hub.repocard import metadata_load |
|
import requests |
|
import re |
|
import pandas as pd |
|
from huggingface_hub import ModelCard |
|
|
|
|
|
def pass_emoji(passed): |
|
if passed is True: |
|
passed = "β
" |
|
else: |
|
passed = "β" |
|
return passed |
|
|
|
api = HfApi() |
|
|
|
|
|
def get_user_audio_classification_models(hf_username): |
|
""" |
|
List the user's Audio Classification models |
|
:param hf_username: User HF username |
|
""" |
|
|
|
models = api.list_models(author=hf_username, filter=["audio-classification"]) |
|
user_model_ids = [x.modelId for x in models] |
|
models_gtzan = [] |
|
|
|
for model in user_model_ids: |
|
meta = get_metadata(model) |
|
if meta is None: |
|
continue |
|
try: |
|
if meta["datasets"] == ['marsyas/gtzan']: |
|
models_gtzan.append(model) |
|
except: continue |
|
return models_gtzan |
|
|
|
|
|
def get_metadata(model_id): |
|
""" |
|
Get model metadata (contains evaluation data) |
|
:param model_id |
|
""" |
|
try: |
|
readme_path = hf_hub_download(model_id, filename="README.md") |
|
return metadata_load(readme_path) |
|
except requests.exceptions.HTTPError: |
|
|
|
return None |
|
|
|
|
|
def extract_accuracy(model_card_content): |
|
""" |
|
Extract the accuracy value from the models' model card |
|
:param model_card_content: model card content |
|
""" |
|
accuracy_pattern = r"Accuracy: (\d+\.\d+)" |
|
match = re.search(accuracy_pattern, model_card_content) |
|
if match: |
|
accuracy = match.group(1) |
|
return float(accuracy) |
|
else: |
|
return None |
|
|
|
|
|
def parse_metrics_accuracy(model_id): |
|
""" |
|
Get model card and parse it |
|
:param model_id: model id |
|
""" |
|
card = ModelCard.load(model_id) |
|
return extract_accuracy(card.content) |
|
|
|
|
|
def calculate_best_acc_result(user_model_ids): |
|
""" |
|
Calculate the best results of a unit |
|
:param user_model_ids: RL models of a user |
|
""" |
|
|
|
best_result = -100 |
|
best_model = "" |
|
|
|
for model in user_model_ids: |
|
meta = get_metadata(model) |
|
if meta is None: |
|
continue |
|
accuracy = parse_metrics_accuracy(model) |
|
if accuracy > best_result: |
|
best_result = accuracy |
|
best_model = meta['model-index'][0]["name"] |
|
|
|
return best_result, best_model |
|
|
|
|
|
def certification(hf_username): |
|
results_certification = [ |
|
{ |
|
"unit": "Unit 4: Audio Classification", |
|
"task": "audio-classification", |
|
"baseline_metric": 0.87, |
|
"best_result": 0, |
|
"best_model_id": "", |
|
"passed_": False |
|
}, |
|
{ |
|
"unit": "Unit 5: TBD", |
|
"task": "TBD", |
|
"baseline_metric": 0.99, |
|
"best_result": 0, |
|
"best_model_id": "", |
|
"passed_": False |
|
}, |
|
{ |
|
"unit": "Unit 6: TBD", |
|
"task": "TBD", |
|
"baseline_metric": 0.99, |
|
"best_result": 0, |
|
"best_model_id": "", |
|
"passed_": False |
|
}, |
|
{ |
|
"unit": "Unit 7: TBD", |
|
"task": "TBD", |
|
"baseline_metric": 0.99, |
|
"best_result": 0, |
|
"best_model_id": "", |
|
"passed_": False |
|
}, |
|
] |
|
|
|
for unit in results_certification: |
|
unit["passed"] = pass_emoji(unit["passed_"]) |
|
if unit["task"] == "audio-classification": |
|
user_models = get_user_audio_classification_models(hf_username) |
|
best_result, best_model_id = calculate_best_acc_result(user_models) |
|
unit["best_result"] = best_result |
|
unit["best_model_id"] = best_model_id |
|
if unit["best_result"] >= unit["baseline_metric"]: |
|
unit["passed_"] = True |
|
unit["passed"] = pass_emoji(unit["passed_"]) |
|
else: |
|
|
|
continue |
|
|
|
print(results_certification) |
|
|
|
df = pd.DataFrame(results_certification) |
|
df = df[['passed', 'unit', 'task', 'baseline_metric', 'best_result', 'best_model_id']] |
|
return df |
|
|
|
|
|
with gr.Blocks() as demo: |
|
gr.Markdown(f""" |
|
# π Check your progress in the Audio Course π |
|
|
|
- To get a certificate of completion, you must **pass 3 out of 4 assignments before July 31st 2023**. |
|
- To get an honors certificate, you must **pass 4 out of 4 assignments before July 31st 2023**. |
|
|
|
To pass an assignment, your model's metric should be equal to or higher than the baseline metric. |
|
|
|
Make sure that you have uploaded your model(s) to Hub and type your Hugging Face Username here to check if you pass (in my case MariaK) |
|
""") |
|
|
|
hf_username = gr.Textbox(placeholder="MariaK", label="Your Hugging Face Username") |
|
check_progress_button = gr.Button(value="Check my progress") |
|
output = gr.components.Dataframe(value=certification(hf_username)) |
|
check_progress_button.click(fn=certification, inputs=hf_username, outputs=output) |
|
|
|
demo.launch() |