adding merge check - super slow but at least info is displayed

src/display/utils.py

@@ -46,6 +46,7 @@ auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type",
 auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
 auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
 auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+auto_eval_column_dict.append(["merge", ColumnContent, ColumnContent("Merged", "bool", False)])
 auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
 auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
 auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
@@ -72,6 +73,7 @@ baseline_row = {
     AutoEvalColumn.model.name: "<p>Baseline</p>",
     AutoEvalColumn.revision.name: "N/A",
     AutoEvalColumn.precision.name: None,
+    AutoEvalColumn.merge.name: False,
     AutoEvalColumn.average.name: 31.0,
     AutoEvalColumn.arc.name: 25.0,
     AutoEvalColumn.hellaswag.name: 25.0,
@@ -97,6 +99,7 @@ human_baseline_row = {
     AutoEvalColumn.revision.name: "N/A",
     AutoEvalColumn.precision.name: None,
     AutoEvalColumn.average.name: 92.75,
+    AutoEvalColumn.merge.name: False,
     AutoEvalColumn.arc.name: 80.0,
     AutoEvalColumn.hellaswag.name: 95.0,
     AutoEvalColumn.mmlu.name: 89.8,

src/leaderboard/read_evals.py

@@ -5,10 +5,10 @@ import os
 from dataclasses import dataclass
 
 import dateutil
-from datetime import datetime
-from transformers import AutoConfig
 import numpy as np
 
+from huggingface_hub import ModelCard
+
 from src.display.formatting import make_clickable_model
 from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
 from src.submission.check_validity import is_model_on_hub
@@ -32,6 +32,7 @@ class EvalResult:
     num_params: int = 0
     date: str = "" # submission date of request file
     still_on_hub: bool = False
+    merge: bool = False
 
     @classmethod
     def init_from_json_file(self, json_filepath):
@@ -59,6 +60,11 @@ class EvalResult:
             result_key = f"{org}_{model}_{precision.value.name}"
         full_model = "/".join(org_and_model)
 
+        try:
+            merge = any(t in ["merge", "mergedlm"] for t in ModelCard.load(full_model).data.tags)
+        except Exception:
+            merge = False
+
         still_on_hub, error, model_config = is_model_on_hub(
             full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
         )
@@ -105,7 +111,8 @@ class EvalResult:
             precision=precision,  
             revision= config.get("model_sha", ""),
             still_on_hub=still_on_hub,
-            architecture=architecture
+            architecture=architecture,
+            merge=merge
         )
 
     def update_with_request_file(self, requests_path):
@@ -131,6 +138,7 @@ class EvalResult:
             "eval_name": self.eval_name,  # not a column, just a save name,
             AutoEvalColumn.precision.name: self.precision.value.name,
             AutoEvalColumn.model_type.name: self.model_type.value.name,
+            AutoEvalColumn.merge.name: self.merge,
             AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
             AutoEvalColumn.weight_type.name: self.weight_type.value.name,
             AutoEvalColumn.architecture.name: self.architecture,