Params and license metadata from HF API

evals/main.py

@@ -19,6 +19,7 @@ transcription_langs_eval_detailed = languages.iloc[:5]
 
 # ===== run evaluation and aggregate results =====
 
+
 async def evaluate():
     print("running evaluations")
     results = [
@@ -26,7 +27,7 @@ async def evaluate():
         for task in tasks
         for i in range(n_sentences)
         for original_language in langs_eval.itertuples()
-        for model in models
+        for model in models["id"]
         if original_language.in_benchmark
         and (
             model == model_fast
@@ -35,6 +36,7 @@ async def evaluate():
     ]
     return await tqdm_asyncio.gather(*results, miniters=1)
 
+
 def aggregate(results):
     results = pd.DataFrame([r for rs in results for r in rs])
     results = (
@@ -58,32 +60,39 @@ def aggregate(results):
     )
     return results, lang_results, model_results, task_results
 
+
 def mean(lst):
     return sum(lst) / len(lst) if lst else None
 
 
 def fmt_name(s):
-    return " ".join(w.capitalize() for w in s.split("-")).replace("Gpt", "GPT").replace("ai", "AI")
+    return (
+        " ".join(w.capitalize() for w in s.split("-"))
+        .replace("Gpt", "GPT")
+        .replace("ai", "AI")
+    )
+
 
 def serialize(df):
     return df.replace({np.nan: None}).to_dict(orient="records")
 
-def make_model_table(model_results):
-    model_results["task_metric"] = model_results["task"] + "_" + model_results["metric"]
-    model_results = model_results.drop(columns=["task", "metric"])
-    model_table = model_results.pivot(
-        index="model", columns="task_metric", values="score"
-    ).fillna(0)
-    model_table["average"] = model_table.mean(axis=1)
-    model_table = model_table.sort_values(by="average", ascending=False)
-    model_table = model_table.round(2).reset_index()
-    model_table["provider"] = model_table["model"].str.split("/").str[0].apply(fmt_name)
-    model_table["model"] = model_table["model"].str.split("/").str[1].apply(fmt_name)
-    model_table["rank"] = model_table.index + 1
-    model_table = model_table[
-        ["rank", "provider", "model", "average", *model_table.columns[1:-3]]
-    ]
-    return model_table
+
+def make_model_table(df):
+    df["task_metric"] = df["task"] + "_" + df["metric"]
+    df = df.drop(columns=["task", "metric"])
+    task_metrics = df["task_metric"].unique()
+    df = df.pivot(index="model", columns="task_metric", values="score").fillna(0)
+    df["average"] = df[task_metrics].mean(axis=1)
+    df = df.sort_values(by="average", ascending=False).reset_index()
+    for row in [*task_metrics, "average"]:
+        df[row] = df[row].round(2)
+    df = pd.merge(df, models, left_on="model", right_on="id", how="left")
+    df["creation_date"] = df["creation_date"].dt.strftime("%Y-%m-%d")
+    df["provider"] = df["model"].str.split("/").str[0].apply(fmt_name)
+    df["model"] = df["model"].str.split("/").str[1].apply(fmt_name)
+    df["rank"] = df.index + 1
+    df = df[["rank", "provider", "model", "hf_id", "creation_date", "size", "type", "license", "average", *task_metrics]]
+    return df
 
 
 async def main():
@@ -97,7 +106,7 @@ async def main():
     }
     with open("results.json", "w") as f:
         json.dump(all_results, f, indent=2, ensure_ascii=False)
-    
+
     model_table = make_model_table(model_results)
     all_tables = {
         "model_table": serialize(model_table),

evals/models.py

@@ -1,11 +1,13 @@
 from os import getenv
 
+import pandas as pd
 from aiolimiter import AsyncLimiter
 from dotenv import load_dotenv
 from elevenlabs import AsyncElevenLabs
-from huggingface_hub import AsyncInferenceClient
+from huggingface_hub import AsyncInferenceClient, HfApi
 from joblib.memory import Memory
 from openai import AsyncOpenAI
+from requests import HTTPError
 
 # for development purposes, all languages will be evaluated on the fast models
 # and only a sample of languages will be evaluated on all models
@@ -80,3 +82,35 @@ async def transcribe(path, model="elevenlabs/scribe_v1"):
             return await transcribe_huggingface(path, model)
         case _:
             raise ValueError(f"Model {model} not supported")
+
+
+models = pd.DataFrame(models, columns=["id"])
+
+api = HfApi()
+
+def get_metadata(id):
+    try:
+        info = api.model_info(id)
+        license = info.card_data.license.replace("_", " ").replace("mit", "MIT").title()
+        return {
+            "hf_id": info.id,
+            "creation_date": info.created_at,
+            "size": info.safetensors.total,
+            "type": "Open",
+            "license": license,
+        }
+    except HTTPError:
+        return {
+            "hf_id": None,
+            "creation_date": None,
+            "size": None,
+            "type": "Commercial",
+            "license": None,
+        }
+
+models["hf_id"] = models["id"].apply(get_metadata).str["hf_id"]
+models["creation_date"] = models["id"].apply(get_metadata).str["creation_date"]
+models["creation_date"] = pd.to_datetime(models["creation_date"])
+models["size"] = models["id"].apply(get_metadata).str["size"]
+models["type"] = models["id"].apply(get_metadata).str["type"]
+models["license"] = models["id"].apply(get_metadata).str["license"]

frontend/public/results.json

@@ -4,6 +4,11 @@
       "rank": 1,
       "provider": "Google",
       "model": "Gemini 2.0 Flash 001",
+      "hf_id": null,
+      "creation_date": null,
+      "size": null,
+      "type": "Commercial",
+      "license": null,
       "average": 0.72,
       "classification_accuracy": 0.87,
       "language_modeling_chrf": 0.96,
@@ -14,6 +19,11 @@
       "rank": 2,
       "provider": "Google",
       "model": "Gemma 3 27b It",
+      "hf_id": "google/gemma-3-27b-it",
+      "creation_date": "2025-03-01",
+      "size": 27432406640.0,
+      "type": "Open",
+      "license": "Gemma",
       "average": 0.65,
       "classification_accuracy": 0.72,
       "language_modeling_chrf": 0.96,
@@ -24,6 +34,11 @@
       "rank": 3,
       "provider": "OpenAI",
       "model": "GPT 4o Mini",
+      "hf_id": null,
+      "creation_date": null,
+      "size": null,
+      "type": "Commercial",
+      "license": null,
       "average": 0.6,
       "classification_accuracy": 0.52,
       "language_modeling_chrf": 0.95,
@@ -34,6 +49,11 @@
       "rank": 4,
       "provider": "MistralAI",
       "model": "Mistral Small 24b Instruct 2501",
+      "hf_id": "mistralai/Mistral-Small-24B-Instruct-2501",
+      "creation_date": "2025-01-28",
+      "size": 23572403200.0,
+      "type": "Open",
+      "license": "Apache-2.0",
       "average": 0.58,
       "classification_accuracy": 0.55,
       "language_modeling_chrf": 0.86,
@@ -44,6 +64,11 @@
       "rank": 5,
       "provider": "Meta Llama",
       "model": "Llama 3.3 70b Instruct",
+      "hf_id": "meta-llama/Llama-3.3-70B-Instruct",
+      "creation_date": "2024-11-26",
+      "size": 70553706496.0,
+      "type": "Open",
+      "license": "Llama3.3",
       "average": 0.56,
       "classification_accuracy": 0.5,
       "language_modeling_chrf": 0.94,

frontend/src/components/ModelTable.js

@@ -32,15 +32,40 @@ const ModelTable = ({ data }) => {
         );
     };
 
+    const sizeBodyTemplate = (rowData) => {
+        const size = rowData.size;
+        if (size === null) {
+            return <div>N/A</div>;
+        }
+        let sizeStr;
+        if (size < 1000) {
+            sizeStr = size.toFixed(0) + "";
+        } else if (size < 1000 * 1000) {
+            sizeStr = (size / 1000).toFixed(0) + "K";
+        } else if (size < 1000 * 1000 * 1000) {
+            sizeStr = (size / 1000 / 1000).toFixed(0) + "M";
+        } else {
+            sizeStr = (size / 1000 / 1000 / 1000).toFixed(0) + "B";
+        }
+        return <div>{sizeStr}</div>;
+    };
+
+    const modelBodyTemplate = (rowData) => {
+        // bold
+        return <div style={{ fontWeight: 'bold' }}>{rowData.model}</div>;
+    };
+
   return (
-    <DataTable value={table} header={<>AI Models</>} sortField="average" removableSort filters={filters} filterDisplay="menu">
+    <DataTable value={table} header={<>AI Models</>} sortField="average" removableSort filters={filters} filterDisplay="menu" scrollable scrollHeight="500px">
       <Column field="rank" body={rankBodyTemplate} />
-      <Column field="provider" header="Provider" filter filterElement={providerRowFilterTemplate} showFilterMatchModes={false} />
-      <Column field="model" header="Model" filter showFilterMatchModes={false} />
-      <Column field="average" header="Average" sortable />
-      <Column field="translation_chrf" header="Translation" sortable />
-      <Column field="classification_accuracy" header="Classification" sortable />
-      <Column field="language_modeling_chrf" header="Language Modeling" sortable />
+      <Column field="provider" header="Provider" filter filterElement={providerRowFilterTemplate} showFilterMatchModes={false} style={{ minWidth: '5rem' }} />
+      <Column field="model" header="Model" filter showFilterMatchModes={false} style={{ minWidth: '15rem' }} body={modelBodyTemplate} />
+      <Column field="type" header="Type" style={{ minWidth: '10rem' }} />
+      <Column field="size" header="Size" sortable body={sizeBodyTemplate} style={{ minWidth: '5rem' }} />
+      <Column field="average" header="Average" sortable style={{ minWidth: '5rem' }} />
+      <Column field="translation_chrf" header="Translation" sortable style={{ minWidth: '5rem' }} />
+      <Column field="classification_accuracy" header="Classification" sortable style={{ minWidth: '5rem' }} />
+      <Column field="language_modeling_chrf" header="Language Modeling" sortable style={{ minWidth: '5rem' }} />
     </DataTable>
     );
 };