Spaces:
Running
Running
File size: 5,037 Bytes
2d4b120 c84ed95 2d4b120 c84ed95 2d4b120 341b6a4 2d4b120 341b6a4 2d4b120 341b6a4 2d4b120 341b6a4 2d4b120 341b6a4 2d4b120 341b6a4 2d4b120 341b6a4 2d4b120 341b6a4 2d4b120 341b6a4 2d4b120 c84ed95 2d4b120 c84ed95 2d4b120 c84ed95 2d4b120 c84ed95 2d4b120 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 |
import requests
import pandas as pd
from tqdm.auto import tqdm
import streamlit as st
from huggingface_hub import HfApi, hf_hub_download
from huggingface_hub.repocard import metadata_load
from ascending_metrics import ascending_metrics
import numpy as np
def make_clickable(model_name):
link = "https://huggingface.co/" + model_name
return f'<a target="_blank" href="{link}">{model_name}</a>'
def make_bold(value):
return f'<b>{value}</b>'
def get_model_ids():
api = HfApi()
models = api.list_models(filter="model-index")
model_ids = [x.modelId for x in models]
return model_ids
def get_metadata(model_id):
try:
readme_path = hf_hub_download(model_id, filename="README.md")
return metadata_load(readme_path)
except Exception:
# 404 README.md not found or problem loading it
return None
def parse_metric_value(value):
if isinstance(value, str):
"".join(value.split("%"))
try:
value = float(value)
except: # noqa: E722
value = None
elif isinstance(value, list):
if len(value) > 0:
value = value[0]
else:
value = None
value = round(value, 2) if isinstance(value, float) else None
return value
def parse_metrics_rows(meta):
if not isinstance(meta["model-index"], list) or len(meta["model-index"]) == 0 or "results" not in meta["model-index"][0]:
return None
for result in meta["model-index"][0]["results"]:
if "dataset" not in result or "metrics" not in result or "type" not in result["dataset"]:
continue
dataset = result["dataset"]["type"]
if "args" not in result["dataset"]:
continue
row = {"dataset": dataset}
for metric in result["metrics"]:
type = metric["type"].lower().strip()
value = parse_metric_value(metric.get("value", None))
if value is None:
continue
if type not in row or value < row[type]:
# overwrite the metric if the new value is lower (e.g. with LM)
row[type] = value
yield row
@st.cache(ttl=86400)
def get_data():
data = []
model_ids = get_model_ids()
for model_id in tqdm(model_ids):
meta = get_metadata(model_id)
if meta is None:
continue
for row in parse_metrics_rows(meta):
if row is None:
continue
row["model_id"] = model_id
data.append(row)
return pd.DataFrame.from_records(data)
dataframe = get_data()
selectable_datasets = list(set(dataframe.dataset.tolist()))
st.markdown("# 🤗 Leaderboards")
query_params = st.experimental_get_query_params()
default_dataset = "common_voice"
if "dataset" in query_params:
if len(query_params["dataset"]) > 0 and query_params["dataset"][0] in selectable_datasets:
default_dataset = query_params["dataset"][0]
dataset = st.sidebar.selectbox(
"Dataset",
selectable_datasets,
index=selectable_datasets.index(default_dataset),
)
dataset_df = dataframe[dataframe.dataset == dataset]
dataset_df = dataset_df.dropna(axis="columns", how="all")
selectable_metrics = list(filter(lambda column: column not in ("model_id", "dataset"), dataset_df.columns))
metric = st.sidebar.radio(
"Sorting Metric",
selectable_metrics,
)
dataset_df = dataset_df.filter(["model_id"] + selectable_metrics)
dataset_df = dataset_df.dropna(thresh=2) # Want at least two non-na values (one for model_id and one for a metric).
dataset_df = dataset_df.sort_values(by=metric, ascending=metric in ascending_metrics)
dataset_df = dataset_df.replace(np.nan, '-')
st.markdown(
"Please click on the model's name to be redirected to its model card which includes documentation and examples on how to use it."
)
# display the model ranks
dataset_df = dataset_df.reset_index(drop=True)
dataset_df.index += 1
# turn the model ids into clickable links
dataset_df["model_id"] = dataset_df["model_id"].apply(make_clickable)
dataset_df[metric] = dataset_df[metric].apply(make_bold)
# Make the selected metric appear right after model names
cols = dataset_df.columns.tolist()
cols.remove(metric)
cols = cols[:1] + [metric] + cols[1:]
dataset_df = dataset_df[cols]
# Highlight selected metric
def highlight_cols(s):
huggingface_yellow = "#FFD21E"
return "background-color: %s" % huggingface_yellow
dataset_df = dataset_df.style.applymap(highlight_cols, subset=pd.IndexSlice[:, [metric]])
# Turn table into html
table_html = dataset_df.to_html(escape=False)
table_html = table_html.replace("<th>", '<th align="left">') # left-align the headers
st.write(table_html, unsafe_allow_html=True)
st.markdown(
"Want to beat the Leaderboard? Don't see your model here? Simply add the `hf-leaderboards` tag to your model card alongside your evaluation metrics. See [this commit](https://huggingface.co/facebook/wav2vec2-base-960h/commit/88338305603a4d8db25aca96e669beb5f7dc65cb) as an example."
)
|