Change model types

src/display/utils.py

@@ -62,9 +62,9 @@ class ModelDetails:
 
 
 class ModelType(Enum):
-    PT = ModelDetails(name="pretrained", symbol="🟢")
-    FT = ModelDetails(name="fine-tuned", symbol="🔶")
-    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
+    PT = ModelDetails(name="LLM", symbol="🟢")
+    FT = ModelDetails(name="Medical fine-tuned", symbol="🔶")
+    IFT = ModelDetails(name="LLM with Vision", symbol="⭕")
     RL = ModelDetails(name="RL-tuned", symbol="🟦")
     Unknown = ModelDetails(name="", symbol="?")
 
@@ -73,13 +73,13 @@ class ModelType(Enum):
 
     @staticmethod
     def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
+        if "Medical fine-tuned" in type or "🔶" in type:
             return ModelType.FT
-        if "pretrained" in type or "🟢" in type:
+        if "LLM" in type or "🟢" in type:
             return ModelType.PT
         if "RL-tuned" in type or "🟦" in type:
             return ModelType.RL
-        if "instruction-tuned" in type or "⭕" in type:
+        if "LLM with Vision" in type or "⭕" in type:
             return ModelType.IFT
         return ModelType.Unknown
 

src/leaderboard/read_evals.py

@@ -43,6 +43,8 @@ class EvalResult:
         # Precision
         precision = Precision.from_str(config.get("model_dtype"))
 
+        self.model_type = ModelType.from_str(config.get("model_type", "LLM"))
+
         self.num_params = int(config.get("model_params", 0))
 
         # Get model and org
@@ -59,14 +61,14 @@ class EvalResult:
             result_key = f"{org}_{model}_{precision.value.name}"
         full_model = "/".join(org_and_model)
 
-        still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
-        )
-        architecture = "?"
-        if model_config is not None:
-            architectures = getattr(model_config, "architectures", None)
-            if architectures:
-                architecture = ";".join(architectures)
+        # still_on_hub, _, model_config = is_model_on_hub(
+        #     full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
+        # )
+        # architecture = "?"
+        # if model_config is not None:
+        #     architectures = getattr(model_config, "architectures", None)
+        #     if architectures:
+        #         architecture = ";".join(architectures)
 
         # Extract results available in this file (some results are split in several files)
         results = {}
@@ -89,8 +91,8 @@ class EvalResult:
             results=results,
             precision=precision,  
             revision= config.get("model_sha", ""),
-            still_on_hub=still_on_hub,
-            architecture=architecture
+            still_on_hub=self.still_on_hub,
+            architecture=self.architecture
         )
 
     # def update_with_request_file(self, requests_path):