update

app.py

@@ -21,33 +21,27 @@ HF_TOKEN = os.environ.get("HF_TOKEN", None)
 LLM_PERF_DATASET_REPO = "optimum/llm-perf-dataset"
 MACHINE_TO_HARDWARE = {"hf-dgx-01": "A100-80GB 🖥️"}
 ALL_COLUMNS_MAPPING = {
-    # model
     "Model": "Model 🤗",
     "Arch": "Arch 🏛️",
-    "Size": "Size 📏",
+    "Size": "Params (B) 📏",
     # deployment settings
     "backend.name": "Backend 🏭",
     "backend.torch_dtype": "Dtype 📥",
-    "optimizations": "Optimizations 🛠️",
+    "optimization": "Optimization 🛠️",
     "quantization": "Quantization 🗜️",
-    # quality measurements
-    "Score": "Avg Score (%) ⬆️",
-    # throughput measurements
+    # measurements
+    "Score": "Open LLM Score (%) ⬆️",
     "decode.throughput(tokens/s)": "Decode Throughput (tokens/s) ⬆️",
     "generate.throughput(tokens/s)": "E2E Throughput (tokens/s) ⬆️",
-    # latency measurements
     "forward.latency(s)": "Prefill Latency (s) ⬇️",
     "generate.latency(s)": "E2E Latency (s) ⬇️",
-    # memory measurements
     "generate.max_memory_allocated(MB)": "Allocated Memory (MB) ⬇️",
     "generate.max_memory_reserved(MB)": "Reserved Memory (MB) ⬇️",
     "generate.max_memory_used(MB)": "Used Memory (MB) ⬇️",
-    # energy measurements
     "generate.energy_consumption(tokens/kWh)": "Energy (tokens/kWh) ⬇️",
 }
 SORTING_COLUMN = ["Score", "generate.throughput(tokens/s)"]
 SORTING_ASCENDING = [False, True]
-
 ALL_COLUMNS_DATATYPES = [
     # open llm
     "markdown",
@@ -70,17 +64,18 @@ ALL_COLUMNS_DATATYPES = [
     "number",
     "number",
 ]
+# download data
+hf_hub_download(
+    repo_id="optimum/llm-perf-dataset",
+    filename="open-llm.csv",
+    local_dir="dataset",
+    repo_type="dataset",
+    token=HF_TOKEN,
+)
+OPEN_LLM = pd.read_csv("dataset/open-llm.csv")
 
-
-def get_benchmark_df(machine="hf-dgx-01"):
-    # download data
-    hf_hub_download(
-        repo_id="optimum/llm-perf-dataset",
-        filename="open-llm.csv",
-        local_dir="dataset",
-        repo_type="dataset",
-        token=HF_TOKEN,
-    )
+MACHINE_TO_DATAFRAME = {}
+for machine in MACHINE_TO_HARDWARE:
     hf_hub_download(
         repo_id="optimum/llm-perf-dataset",
         filename=f"{machine}/full-report.csv",
@@ -88,11 +83,13 @@ def get_benchmark_df(machine="hf-dgx-01"):
         repo_type="dataset",
         token=HF_TOKEN,
     )
-    open_llm = pd.read_csv("dataset/open-llm.csv")
-    full_report = pd.read_csv(f"dataset/{machine}/full-report.csv")
+    MACHINE_TO_DATAFRAME[machine] = pd.read_csv(f"dataset/{machine}/full-report.csv")
+
 
+def get_benchmark_df(machine="hf-dgx-01"):
     # merge on model
-    merged_df = open_llm.merge(full_report, left_on="Model", right_on="model")
+    llm_perf = MACHINE_TO_DATAFRAME[machine].copy()
+    merged_df = OPEN_LLM.merge(llm_perf, left_on="Model", right_on="model")
     # transpose energy consumption
     merged_df["generate.energy_consumption(tokens/kWh)"] = (
         1 / merged_df["generate.energy_consumption(kWh/token)"].fillna(1)
@@ -102,8 +99,8 @@ def get_benchmark_df(machine="hf-dgx-01"):
         merged_df["generate.energy_consumption(tokens/kWh)"] == 1,
         "generate.energy_consumption(tokens/kWh)",
     ] = pd.NA
-    # add optimizations column
-    merged_df["optimizations"] = merged_df[
+    # add optimization column
+    merged_df["optimization"] = merged_df[
         ["backend.to_bettertransformer", "backend.use_flash_attention_2"]
     ].apply(
         lambda x: "BetterTransformer"
@@ -135,10 +132,10 @@ def get_benchmark_table(bench_df):
     copy_df["Model 🤗"] = copy_df["Model 🤗"].apply(process_model_name)
     copy_df["Arch 🏛️"] = copy_df["Arch 🏛️"].apply(process_model_arch)
     # process quantization
-    copy_df["Avg Score (%) ⬆️"] = copy_df.apply(
-        lambda x: f"{x['Avg Score (%) ⬆️']}**"
+    copy_df["Open LLM Score (%) ⬆️"] = copy_df.apply(
+        lambda x: f"{x['Open LLM Score (%) ⬆️']}**"
         if x["Quantization 🗜️"] in ["BnB.4bit", "GPTQ.4bit"]
-        else x["Avg Score (%) ⬆️"],
+        else x["Open LLM Score (%) ⬆️"],
         axis=1,
     )
     return copy_df
@@ -151,7 +148,7 @@ def get_benchmark_chart(bench_df):
     # plot
     fig = px.scatter(
         copy_df,
-        y="Avg Score (%) ⬆️",
+        y="Open LLM Score (%) ⬆️",
         x="E2E Latency (s) ⬇️",
         size="Allocated Memory (MB) ⬇️",
         color="Arch 🏛️",
@@ -167,7 +164,7 @@ def get_benchmark_chart(bench_df):
             "yanchor": "top",
         },
         xaxis_title="Per 1000 Tokens Latency (s)",
-        yaxis_title="Avg Open LLM Score (%)",
+        yaxis_title="Open LLM Score (%)",
         legend_title="LLM Architecture",
         width=1200,
         height=600,
@@ -188,7 +185,7 @@ def filter_query(
     backends,
     datatypes,
     optimizations,
-    quantization_scheme,
+    quantizations,
     score,
     memory,
     machine,
@@ -198,29 +195,9 @@ def filter_query(
         raw_df["Model 🤗"].str.contains(text, case=False)
         & raw_df["Backend ����"].isin(backends)
         & raw_df["Dtype 📥"].isin(datatypes)
-        & (
-            pd.concat(
-                [
-                    raw_df["Optimizations 🛠️"].str.contains(optimization, case=False)
-                    for optimization in optimizations
-                ],
-                axis=1,
-            ).any(axis="columns")
-            if len(optimizations) > 0
-            else True
-        )
-        & (
-            pd.concat(
-                [
-                    raw_df["Quantization 🗜️"].str.contains(quantization, case=False)
-                    for quantization in quantization_scheme
-                ],
-                axis=1,
-            ).any(axis="columns")
-            if len(quantization_scheme) > 0
-            else True
-        )
-        & (raw_df["Avg Score (%) ⬆️"] >= score)
+        & raw_df["Optimization 🛠️"].isin(optimizations)
+        & raw_df["Quantization 🗜️"].isin(quantizations)
+        & (raw_df["Open LLM Score (%) ⬆️"] >= score)
         & (raw_df["Allocated Memory (MB) ⬇️"] <= memory)
     ]
     filtered_table = get_benchmark_table(filtered_df)
@@ -289,7 +266,7 @@ with demo:
             with gr.Row():
                 with gr.Column(scale=1):
                     score_slider = gr.Slider(
-                        label="Open LLM Score 📈",
+                        label="Open LLM Score (%) 📈",
                         info="🎚️ Slide to minimum Open LLM score",
                         value=0,
                         elem_id="threshold-slider",
@@ -321,12 +298,12 @@ with demo:
                         elem_id="dtype-checkboxes",
                     )
                 with gr.Column(scale=1):
-                    optimizations_checkboxes = gr.CheckboxGroup(
+                    optimization_checkboxes = gr.CheckboxGroup(
                         label="Optimizations 🛠️",
-                        choices=["None", "BetterTransformer"],
-                        value=["None", "BetterTransformer"],
-                        info="☑️ Select the optimizations",
-                        elem_id="optimizations-checkboxes",
+                        choices=["None", "BetterTransformer", "FlashAttentionV2"],
+                        value=["None", "BetterTransformer", "FlashAttentionV2"],
+                        info="☑️ Select the optimization",
+                        elem_id="optimization-checkboxes",
                     )
                 with gr.Column(scale=1):
                     quantization_checkboxes = gr.CheckboxGroup(
@@ -348,7 +325,7 @@ with demo:
                         search_bar,
                         backend_checkboxes,
                         datatype_checkboxes,
-                        optimizations_checkboxes,
+                        optimization_checkboxes,
                         quantization_checkboxes,
                         score_slider,
                         memory_slider,

script.py

@@ -0,0 +1,14 @@
+from huggingface_hub import hf_hub_download
+import pandas as pd
+
+
+hf_hub_download(
+    repo_id="optimum/llm-perf-dataset",
+    filename="open-llm.csv",
+    local_dir="dataset",
+    repo_type="dataset",
+)
+
+open_llm = pd.read_csv("dataset/open-llm.csv")
+print(open_llm["Arch"].unique())
+print(open_llm[open_llm["Arch"] == "rwkv"]["Model"].unique())

src/utils.py

@@ -18,26 +18,31 @@ def change_tab(query_param):
 
 
 LLM_MODEL_ARCHS = {
-    # branded ?
-    "gpt_bigcode": "GPT-BigCode 🌸",
+    "mixformer-sequential": "Phi φ",
     "RefinedWebModel": "Falcon 🦅",
+    "gpt_bigcode": "StarCoder ⭐",
     "RefinedWeb": "Falcon 🦅",
     "baichuan": "Baichuan 🌊",
+    "mistral": "Mistral Ⓜ️",
+    "codegen": "CodeGen ♾️",
+    "falcon": "Falcon 🦅",
     "bloom": "Bloom 🌸",
     "llama": "LLaMA 🦙",
-    # unbranded ? suggest something
+    "mpt": "MPT 🧱",
+    "Yi": "Yi 人",
+    # suggest something
+    "stablelm_epoch": "StableLM-Epoch",
     "stablelm_alpha": "StableLM-Alpha",
     "gpt_neox": "GPT-NeoX",
     "gpt_neo": "GPT-Neo",
-    "codegen": "CodeGen",
     "chatglm": "ChatGLM",
+    "internlm": "InternLM",
     "gpt2": "GPT-2",
     "gptj": "GPT-J",
     "xglm": "XGLM",
     "rwkv": "RWKV",
     "bart": "BART",
     "opt": "OPT",
-    "mpt": "MPT",
 }