sync from github

app.py

@@ -11,6 +11,7 @@ import time
 from apscheduler.schedulers.background import BackgroundScheduler
 
 from huggingface_hub import snapshot_download
+from pytz import utc
 
 from src.display.about import (
     CITATION_BUTTON_LABEL,
@@ -159,6 +160,7 @@ def filter_models(df: pd.DataFrame, type_query: list, size_query: list, precisio
     type_emoji = [t[0] for t in type_query]
     filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
     filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
+    filtered_df = filtered_df.loc[df[AutoEvalColumn.inference_framework.name].isin(size_query)]
 
     # numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
     # params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
@@ -257,7 +259,7 @@ with demo:
                                 for c in fields(AutoEvalColumn)
                                 if c.displayed_by_default and not c.hidden and not c.never_hidden
                             ],
-                            label="Select columns to show",
+                            label="Tasks",
                             elem_id="column-select",
                             interactive=True,
                         )
@@ -477,7 +479,7 @@ with demo:
                 show_copy_button=True,
             )
 
-scheduler = BackgroundScheduler()
+scheduler = BackgroundScheduler(timezone=utc)
 
 scheduler.add_job(restart_space, "interval", hours=6)
 

backend-cli.py

@@ -458,6 +458,7 @@ def get_args():
     parser.add_argument("--gpu-type", type=str, default="NVIDIA-A100-PCIe-80GB", 
                         help="GPU type. NVIDIA-A100-PCIe-80GB; NVIDIA-RTX-A5000-24GB; NVIDIA-H100-PCIe-80GB")
     parser.add_argument("--debug_repo", action="store_true", help="Use debug repo")
+    parser.add_argument("--model_type", type=str, default="chat", help="Model type")
     return parser.parse_args()
 
 
@@ -488,7 +489,8 @@ if __name__ == "__main__":
                         json_filepath="",
                         precision=precision,  # Use precision from arguments
                         inference_framework=args.inference_framework,  # Use inference framework from arguments
-                        gpu_type=args.gpu_type
+                        gpu_type=args.gpu_type,
+                        model_type=args.model_type,
                     )
                     curr_gpu_type = get_gpu_details()
                     if eval_request.gpu_type != curr_gpu_type:

src/display/about.py

@@ -19,8 +19,8 @@ Columns and Metrics:
 - E2E(s): Average End to End generation time in seconds.
 - PRE(s): Prefilling Time of input prompt in seconds.
 - T/s: Tokens throughout per second.
-- MBU(%): Model Bandwidth Utilization.
-- MFU(%): Model FLOPs Utilization.
+- S-MBU(%): Sparse Model Bandwidth Utilization.
+- S-MFU(%): Sparse Model FLOPs Utilization.
 - Precision: The precison of used model.
 
 """

src/display/utils.py

@@ -18,8 +18,8 @@ GPU_Power = 'Power(W)'
 GPU_Mem = 'Mem(G)'
 GPU_Name = "GPU"
 GPU_Util = 'Util(%)'
-MFU = 'MFU(%)'
-MBU = 'MBU(%)'
+MFU = 'S-MFU(%)'
+MBU = 'S-MBU(%)'
 BATCH_SIZE = 'bs'
 PRECISION = "Precision"
 system_metrics_to_name_map = {
@@ -106,7 +106,7 @@ auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "ma
 # # auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Avg", "number", True)])
 
 # Inference framework
-auto_eval_column_dict.append(["inference_framework", ColumnContent, ColumnContent(f"{InFrame}", "str", True)])
+auto_eval_column_dict.append(["inference_framework", ColumnContent, ColumnContent(f"{InFrame}", "str", True, dummy=True)])
 
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
@@ -126,15 +126,15 @@ for task in Tasks:
 
 
 # Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", True)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False, dummy=True)])
+# auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
+# auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", True, dummy=True)])
+# auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
+# auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+# auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
+# auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+# auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
 # Dummy column for the search bar (hidden by the custom CSS)
 auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
 
@@ -160,10 +160,10 @@ class ModelDetails:
 
 
 class ModelType(Enum):
-    PT = ModelDetails(name="pretrained", symbol="🟢")
-    FT = ModelDetails(name="fine-tuned on domain-specific datasets", symbol="🔶")
+    # PT = ModelDetails(name="pretrained", symbol="🟢")
+    # FT = ModelDetails(name="fine-tuned on domain-specific datasets", symbol="🔶")
     chat = ModelDetails(name="chat models (RLHF, DPO, IFT, ...)", symbol="💬")
-    merges = ModelDetails(name="base merges and moerges", symbol="🤝")
+    # merges = ModelDetails(name="base merges and moerges", symbol="🤝")
     Unknown = ModelDetails(name="", symbol="?")
 
     def to_str(self, separator=" "):
@@ -171,22 +171,23 @@ class ModelType(Enum):
 
     @staticmethod
     def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
-            return ModelType.FT
-        if "pretrained" in type or "🟢" in type:
-            return ModelType.PT
+        # if "fine-tuned" in type or "🔶" in type:
+        #     return ModelType.FT
+        # if "pretrained" in type or "🟢" in type:
+        #     return ModelType.PT
         if any([k in type for k in ["instruction-tuned", "RL-tuned", "chat", "🟦", "⭕", "💬"]]):
             return ModelType.chat
-        if "merge" in type or "🤝" in type:
-            return ModelType.merges
+        # if "merge" in type or "🤝" in type:
+        #     return ModelType.merges
         return ModelType.Unknown
 
 
 class InferenceFramework(Enum):
     # "moe-infinity", hf-chat
-    MoE_Infinity = ModelDetails("moe-infinity")
+    # MoE_Infinity = ModelDetails("moe-infinity")
     HF_Chat = ModelDetails("hf-chat")
     VLLM = ModelDetails("vllm_moe")
+    TRTLLM = ModelDetails("tensorrt_llm")
     Unknown = ModelDetails("?")
 
     def to_str(self):
@@ -194,8 +195,10 @@ class InferenceFramework(Enum):
 
     @staticmethod
     def from_str(inference_framework: str):
-        if inference_framework in ["moe-infinity"]:
-            return InferenceFramework.MoE_Infinity
+        # if inference_framework in ["moe-infinity"]:
+        #     return InferenceFramework.MoE_Infinity
+        if inference_framework in ["tensorrt_llm"]:
+            return InferenceFramework.TRTLLM
         if inference_framework in ["hf-chat"]:
             return InferenceFramework.HF_Chat
         if inference_framework in ["vllm_moe"]:
@@ -225,28 +228,28 @@ class WeightType(Enum):
 
 
 class Precision(Enum):
-    float32 = ModelDetails("float32")
-    float16 = ModelDetails("float16")
+    # float32 = ModelDetails("float32")
+    # float16 = ModelDetails("float16")
     bfloat16 = ModelDetails("bfloat16")
     qt_8bit = ModelDetails("8bit")
     qt_4bit = ModelDetails("4bit")
-    qt_GPTQ = ModelDetails("GPTQ")
+    # qt_GPTQ = ModelDetails("GPTQ")
     Unknown = ModelDetails("?")
 
     @staticmethod
     def from_str(precision: str):
-        if precision in ["torch.float32", "float32"]:
-            return Precision.float32
-        if precision in ["torch.float16", "float16"]:
-            return Precision.float16
+        # if precision in ["torch.float32", "float32"]:
+        #     return Precision.float32
+        # if precision in ["torch.float16", "float16"]:
+        #     return Precision.float16
         if precision in ["torch.bfloat16", "bfloat16"]:
             return Precision.bfloat16
         if precision in ["8bit"]:
             return Precision.qt_8bit
         if precision in ["4bit"]:
             return Precision.qt_4bit
-        if precision in ["GPTQ", "None"]:
-            return Precision.qt_GPTQ
+        # if precision in ["GPTQ", "None"]:
+        #     return Precision.qt_GPTQ
         return Precision.Unknown
 
 

src/leaderboard/read_evals.py

@@ -140,6 +140,7 @@ class EvalResult:
             revision=config.get("model_sha", ""),
             still_on_hub=still_on_hub,
             architecture=architecture,
+            model_type=ModelType.from_str(config.get("model_type", "")),
             inference_framework=inference_framework,
         )
 
@@ -174,22 +175,22 @@ class EvalResult:
 
         # breakpoint()
         # average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
-
+        
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
             AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.model_type.name: self.model_type.value.name,
+            # AutoEvalColumn.model_type.name: self.model_type.value.name,
             AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            AutoEvalColumn.architecture.name: self.architecture,
+            # AutoEvalColumn.weight_type.name: self.weight_type.value.name,
+            # AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
             AutoEvalColumn.dummy.name: self.full_model,
-            AutoEvalColumn.revision.name: self.revision,
-            # AutoEvalColumn.average.name: average,
-            AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
-            AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+            # AutoEvalColumn.revision.name: self.revision,
+            # # AutoEvalColumn.average.name: average,
+            # AutoEvalColumn.license.name: self.license,
+            # AutoEvalColumn.likes.name: self.likes,
+            # AutoEvalColumn.params.name: self.num_params,
+            # AutoEvalColumn.still_on_hub.name: self.still_on_hub,
             AutoEvalColumn.inference_framework.name: self.inference_framework,
         }