added model architecture filter

app.py

@@ -26,6 +26,7 @@ from src.display.utils import (
     TYPES,
     AutoEvalColumn,
     ModelType,
+    ModelArch,
     Precision,
     WeightType,
     fields,
@@ -74,11 +75,12 @@ def update_table(
     columns: list,
     query: str,
     type_query: list = None,
-    precision_query: str = None,
+    architecture_query: list = None,
     size_query: list = None,
+    precision_query: str = None,
     show_deleted: bool = False,
 ):
-    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
+    filtered_df = filter_models(hidden_df, type_query, architecture_query, size_query, precision_query, show_deleted)
     filtered_df = filter_queries(query, filtered_df)
     df = select_columns(filtered_df, columns, list(hidden_df.columns))
     return df
@@ -122,7 +124,7 @@ def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
 
 
 def filter_models(
-    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
+    df: pd.DataFrame, type_query: list, architecture_query: list, size_query: list, precision_query: list, show_deleted: bool
 ) -> pd.DataFrame:
     # Show all models
     # if show_deleted:
@@ -136,6 +138,11 @@ def filter_models(
         type_emoji = [t[0] for t in type_query]
         filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
 
+    if architecture_query is not None:
+        arch_types = [t for t in architecture_query]
+        filtered_df = filtered_df.loc[df[AutoEvalColumn.architecture.name].isin(arch_types)]
+            # filtered_df = filtered_df.loc[df[AutoEvalColumn.architecture.name].isin(architecture_query + ["None"])]
+    
     if precision_query is not None:
         if AutoEvalColumn.precision.name in df.columns:
             filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
@@ -184,19 +191,19 @@ with demo:
                 with gr.Column(min_width=320):
                     # with gr.Box(elem_id="box-filter"):
                     filter_columns_type = gr.CheckboxGroup(
-                        label="Model types",
+                        label="Model Types",
                         choices=[t.to_str() for t in ModelType],
                         value=[t.to_str() for t in ModelType],
                         interactive=True,
                         elem_id="filter-columns-type",
                     )
-                    # filter_columns_precision = gr.CheckboxGroup(
-                    #     label="Precision",
-                    #     choices=[i.value.name for i in Precision],
-                    #     value=[i.value.name for i in Precision],
-                    #     interactive=True,
-                    #     elem_id="filter-columns-precision",
-                    # )
+                    filter_columns_architecture = gr.CheckboxGroup(
+                        label="Architecture Types",
+                        choices=[i.value.name for i in ModelArch],
+                        value=[i.value.name for i in ModelArch],
+                        interactive=True,
+                        elem_id="filter-columns-architecture",
+                    )
                     # filter_columns_size = gr.CheckboxGroup(
                     #     label="Model sizes (in billions of parameters)",
                     #     choices=list(NUMERIC_INTERVALS.keys()),
@@ -227,13 +234,14 @@ with demo:
                     shown_columns,
                     search_bar,
                     filter_columns_type,
+                    filter_columns_architecture
                 ],
                 leaderboard_table,
             )
             for selector in [
                 shown_columns,
                 filter_columns_type,
-                # filter_columns_precision,
+                filter_columns_architecture,
                 # filter_columns_size,
                 # deleted_models_visibility,
             ]:
@@ -244,6 +252,7 @@ with demo:
                         shown_columns,
                         search_bar,
                         filter_columns_type,
+                        filter_columns_architecture,
                     ],
                     leaderboard_table,
                     queue=True,
@@ -283,6 +292,13 @@ with demo:
                         interactive=True,
                         elem_id="filter-columns-type",
                     )
+                    filter_columns_architecture = gr.CheckboxGroup(
+                        label="Architecture Types",
+                        choices=[i.value.name for i in ModelArch],
+                        value=[i.value.name for i in ModelArch],
+                        interactive=True,
+                        elem_id="filter-columns-architecture",
+                    )
                     # filter_columns_precision = gr.CheckboxGroup(
                     #     label="Precision",
                     #     choices=[i.value.name for i in Precision],
@@ -321,12 +337,14 @@ with demo:
                     shown_columns,
                     search_bar,
                     filter_columns_type,
+                    filter_columns_architecture,
                 ],
                 leaderboard_table,
             )
             for selector in [
                 shown_columns,
                 filter_columns_type,
+                filter_columns_architecture,
                 # filter_columns_precision,
                 # filter_columns_size,
                 # deleted_models_visibility,
@@ -338,6 +356,7 @@ with demo:
                         shown_columns,
                         search_bar,
                         filter_columns_type,
+                        filter_columns_architecture,
                     ],
                     leaderboard_table,
                     queue=True,

src/display/utils.py

@@ -39,6 +39,7 @@ for task in ClinicalTypes:
 # Model information
 auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
 auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
+auto_eval_column_dict.append(["backbone", ColumnContent, ColumnContent("Base Model", "str", False)])
 auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
 auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False, True)])
 auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
@@ -100,6 +101,27 @@ class ModelType(Enum):
         #     return ModelType.IFT
         return ModelType.Unknown
 
+class ModelArch(Enum):
+    Encoder = ModelDetails("Encoder")
+    Decoder = ModelDetails("Decoder")
+    EncoderDecoder = ModelDetails("EncoderDecoder")
+    Unknown = ModelDetails(name="Other", symbol="?")
+
+    def to_str(self, separator=" "):
+        return f"{self.value.name}"
+
+    @staticmethod
+    def from_str(type):
+        if "encoder" in type:
+            return ModelArch.Encoder
+        if "decoder" in type:
+            return ModelArch.Decoder
+        if "encoder-decoder" in type:
+            return ModelArch.EncoderDecoder
+        # if "unknown" in type:
+        #     return ModelArch.Unknown
+        return ModelArch.Unknown
+
 
 class WeightType(Enum):
     Adapter = ModelDetails("Adapter")

src/leaderboard/read_evals.py

@@ -8,7 +8,7 @@ import dateutil
 import numpy as np
 
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Precision, Tasks, WeightType, ClinicalTypes
+from src.display.utils import AutoEvalColumn, ModelType, ModelArch, Precision, Tasks, WeightType, ClinicalTypes
 from src.submission.check_validity import is_model_on_hub
 
 
@@ -27,6 +27,7 @@ class EvalResult:
     model_type: ModelType = ModelType.Unknown  # Pretrained, fine tuned, ...
     weight_type: WeightType = WeightType.Original  # Original or Adapter
     architecture: str = "Unknown"
+    backbone:str = "Unknown"
     license: str = "?"
     likes: int = 0
     num_params: int = 0
@@ -43,7 +44,9 @@ class EvalResult:
 
         # Precision
         precision = Precision.from_str(config.get("model_dtype"))
+        model_architecture = ModelArch.from_str(config.get("architecture"))
         model_type = ModelType.from_str(config.get("model_type", ""))
+        # print(model_architecture, model_type)
         license = config.get("license", "?")
         num_params = config.get("num_params", "?")
 
@@ -64,11 +67,11 @@ class EvalResult:
         still_on_hub, _, model_config = is_model_on_hub(
             full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
         )
-        architecture = "?"
+        backbone = "?"
         if model_config is not None:
-            architectures = getattr(model_config, "architectures", None)
-            if architectures:
-                architecture = ";".join(architectures)
+            backbones = getattr(model_config, "architectures", None)
+            if backbones:
+                backbone = ";".join(backbones)
 
         # Extract results available in this file (some results are split in several files)
         dataset_results = {}
@@ -105,7 +108,8 @@ class EvalResult:
             precision=precision,
             revision=config.get("model_sha", ""),
             still_on_hub=still_on_hub,
-            architecture=architecture,
+            architecture=model_architecture,
+            backbone=backbone,
             model_type=model_type,
             num_params=num_params,
             license=license
@@ -141,7 +145,8 @@ class EvalResult:
                 AutoEvalColumn.model_type.name: self.model_type.value.name,
                 AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
                 AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-                AutoEvalColumn.architecture.name: self.architecture,
+                AutoEvalColumn.architecture.name: self.architecture.name,
+                AutoEvalColumn.backbone.name: self.backbone,
                 AutoEvalColumn.model.name: make_clickable_model(self.full_model),
                 AutoEvalColumn.revision.name: self.revision,
                 AutoEvalColumn.average.name: average,
@@ -164,7 +169,8 @@ class EvalResult:
                 AutoEvalColumn.model_type.name: self.model_type.value.name,
                 AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
                 AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-                AutoEvalColumn.architecture.name: self.architecture,
+                AutoEvalColumn.architecture.name: self.architecture.name,
+                AutoEvalColumn.backbone.name: self.backbone,
                 AutoEvalColumn.model.name: make_clickable_model(self.full_model),
                 AutoEvalColumn.revision.name: self.revision,
                 AutoEvalColumn.average.name: average,