Spaces:
Running
Running
Tristan Thrush
added metric sort orders, added feature to display all metrics at the same time
c84ed95
| import requests | |
| import pandas as pd | |
| from tqdm.auto import tqdm | |
| import streamlit as st | |
| from huggingface_hub import HfApi, hf_hub_download | |
| from huggingface_hub.repocard import metadata_load | |
| from ascending_metrics import ascending_metrics | |
| import numpy as np | |
| def make_clickable(model_name): | |
| link = "https://huggingface.co/" + model_name | |
| return f'<a target="_blank" href="{link}">{model_name}</a>' | |
| def make_bold(value): | |
| return f'<b>{value}</b>' | |
| def get_model_ids(): | |
| api = HfApi() | |
| models = api.list_models(filter="model-index") | |
| model_ids = [x.modelId for x in models] | |
| return model_ids | |
| def get_metadata(model_id): | |
| try: | |
| readme_path = hf_hub_download(model_id, filename="README.md") | |
| return metadata_load(readme_path) | |
| except Exception: | |
| # 404 README.md not found or problem loading it | |
| return None | |
| def parse_metric_value(value): | |
| if isinstance(value, str): | |
| "".join(value.split("%")) | |
| try: | |
| value = float(value) | |
| except: # noqa: E722 | |
| value = None | |
| elif isinstance(value, list): | |
| if len(value) > 0: | |
| value = value[0] | |
| else: | |
| value = None | |
| value = round(value, 2) if isinstance(value, float) else None | |
| return value | |
| def parse_metrics_rows(meta): | |
| if not isinstance(meta["model-index"], list) or len(meta["model-index"]) == 0 or "results" not in meta["model-index"][0]: | |
| return None | |
| for result in meta["model-index"][0]["results"]: | |
| if "dataset" not in result or "metrics" not in result or "type" not in result["dataset"]: | |
| continue | |
| dataset = result["dataset"]["type"] | |
| if "args" not in result["dataset"]: | |
| continue | |
| row = {"dataset": dataset} | |
| for metric in result["metrics"]: | |
| type = metric["type"].lower().strip() | |
| value = parse_metric_value(metric.get("value", None)) | |
| if value is None: | |
| continue | |
| if type not in row or value < row[type]: | |
| # overwrite the metric if the new value is lower (e.g. with LM) | |
| row[type] = value | |
| yield row | |
| def get_data(): | |
| data = [] | |
| model_ids = get_model_ids() | |
| for model_id in tqdm(model_ids): | |
| meta = get_metadata(model_id) | |
| if meta is None: | |
| continue | |
| for row in parse_metrics_rows(meta): | |
| if row is None: | |
| continue | |
| row["model_id"] = model_id | |
| data.append(row) | |
| return pd.DataFrame.from_records(data) | |
| dataframe = get_data() | |
| selectable_datasets = list(set(dataframe.dataset.tolist())) | |
| st.markdown("# 🤗 Leaderboards") | |
| query_params = st.experimental_get_query_params() | |
| default_dataset = "common_voice" | |
| if "dataset" in query_params: | |
| if len(query_params["dataset"]) > 0 and query_params["dataset"][0] in selectable_datasets: | |
| default_dataset = query_params["dataset"][0] | |
| dataset = st.sidebar.selectbox( | |
| "Dataset", | |
| selectable_datasets, | |
| index=selectable_datasets.index(default_dataset), | |
| ) | |
| dataset_df = dataframe[dataframe.dataset == dataset] | |
| dataset_df = dataset_df.dropna(axis="columns", how="all") | |
| selectable_metrics = list(filter(lambda column: column not in ("model_id", "dataset"), dataset_df.columns)) | |
| metric = st.sidebar.radio( | |
| "Sorting Metric", | |
| selectable_metrics, | |
| ) | |
| dataset_df = dataset_df.filter(["model_id"] + selectable_metrics) | |
| dataset_df = dataset_df.dropna(thresh=2) # Want at least two non-na values (one for model_id and one for a metric). | |
| dataset_df = dataset_df.sort_values(by=metric, ascending=metric in ascending_metrics) | |
| dataset_df = dataset_df.replace(np.nan, '-') | |
| st.markdown( | |
| "Please click on the model's name to be redirected to its model card which includes documentation and examples on how to use it." | |
| ) | |
| # display the model ranks | |
| dataset_df = dataset_df.reset_index(drop=True) | |
| dataset_df.index += 1 | |
| # turn the model ids into clickable links | |
| dataset_df["model_id"] = dataset_df["model_id"].apply(make_clickable) | |
| dataset_df[metric] = dataset_df[metric].apply(make_bold) | |
| # Make the selected metric appear right after model names | |
| cols = dataset_df.columns.tolist() | |
| cols.remove(metric) | |
| cols = cols[:1] + [metric] + cols[1:] | |
| dataset_df = dataset_df[cols] | |
| # Highlight selected metric | |
| def highlight_cols(s): | |
| huggingface_yellow = "#FFD21E" | |
| return "background-color: %s" % huggingface_yellow | |
| dataset_df = dataset_df.style.applymap(highlight_cols, subset=pd.IndexSlice[:, [metric]]) | |
| # Turn table into html | |
| table_html = dataset_df.to_html(escape=False) | |
| table_html = table_html.replace("<th>", '<th align="left">') # left-align the headers | |
| st.write(table_html, unsafe_allow_html=True) | |
| st.markdown( | |
| "Want to beat the Leaderboard? Don't see your model here? Simply add the `hf-leaderboards` tag to your model card alongside your evaluation metrics. See [this commit](https://huggingface.co/facebook/wav2vec2-base-960h/commit/88338305603a4d8db25aca96e669beb5f7dc65cb) as an example." | |
| ) | |