added peak memory and made scores clickable

app.py

@@ -4,9 +4,9 @@ import gradio as gr
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
 
-from src.assets.text_content import TITLE, INTRODUCTION_TEXT, CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT
-from src.utils import restart_space, load_dataset_repo, make_clickable_model
-from src.assets.css_html_js import custom_css, get_window_url_params
+from src.assets.text_content import TITLE, INTRODUCTION_TEXT, SINGLE_A100_TEXT, CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT
+from src.utils import restart_space, load_dataset_repo, make_clickable_model, make_clickable_score, extract_score_from_clickable
+from src.assets.css_html_js import custom_css
 
 
 LLM_PERF_LEADERBOARD_REPO = "optimum/llm-perf-leaderboard"
@@ -18,9 +18,10 @@ COLUMNS_MAPPING = {
     "backend.name": "Backend 🏭",
     "backend.torch_dtype": "Datatype 📥",
     "average": "Average H4 Score ⬆️",
+    "forward.peak_memory(MB)": "Peak Memory (MB) ⬇️",
     "generate.throughput(tokens/s)": "Throughput (tokens/s) ⬆️",
 }
-COLUMNS_DATATYPES = ["markdown", "str", "str", "number", "number", "number"]
+COLUMNS_DATATYPES = ["markdown", "str", "str", "markdown", "number", "number"]
 SORTING_COLUMN = ["Throughput (tokens/s) ⬆️"]
 
 
@@ -34,17 +35,15 @@ def get_benchmark_df(benchmark):
     # load
     bench_df = pd.read_csv(
         f"./llm-perf-dataset/reports/{benchmark}/inference_report.csv")
+
     scores_df = pd.read_csv(
         f"./llm-perf-dataset/reports/average_scores.csv")
-    # merge on model
-    bench_df = bench_df.merge(
-        scores_df, how="left", left_on="model", right_on="model")
+    bench_df = bench_df.merge(scores_df, on="model", how="left")
+    bench_df["average"] = bench_df["average"].apply(
+        make_clickable_score)
 
     # preprocess
     bench_df["model"] = bench_df["model"].apply(make_clickable_model)
-    # set none datatype to float32
-    bench_df["backend.torch_dtype"] = bench_df["backend.torch_dtype"].fillna(
-        "float32")
     # filter
     bench_df = bench_df[list(COLUMNS_MAPPING.keys())]
     # rename
@@ -55,37 +54,38 @@ def get_benchmark_df(benchmark):
     return bench_df
 
 
-def change_tab(query_param):
-    query_param = query_param.replace("'", '"')
-    query_param = json.loads(query_param)
-
-    if (
-        isinstance(query_param, dict)
-        and "tab" in query_param
-        and query_param["tab"] == "evaluation"
-    ):
-        return gr.Tabs.update(selected=1)
-    else:
-        return gr.Tabs.update(selected=0)
+# def change_tab(query_param):
+#     query_param = query_param.replace("'", '"')
+#     query_param = json.loads(query_param)
 
+#     if (
+#         isinstance(query_param, dict)
+#         and "tab" in query_param
+#         and query_param["tab"] == "evaluation"
+#     ):
+#         return gr.Tabs.update(selected=1)
+#     else:
+#         return gr.Tabs.update(selected=0)
 
-def submit_query(single_df, multi_df, text, backends, datatypes, threshold):
 
-    filtered_single = single_df[
-        single_df["Model 🤗"].str.contains(text) &
-        single_df["Backend 🏭"].isin(backends) &
-        single_df["Datatype 📥"].isin(datatypes) &
-        (single_df["Average H4 Score ⬆️"] >= threshold)
-    ]
+def submit_query(text, backends, datatypes, threshold, raw_dfs):
+    filtered_dfs = []
+    for raw_df in raw_dfs:
+        # extract the average score (float) from the clickable score (clickable markdown)
+        raw_df["Average H4 Score ⬆️"] = raw_df["Average H4 Score ⬆️"].apply(
+            extract_score_from_clickable)
+        filtered_df = raw_df[
+            raw_df["Model 🤗"].str.contains(text) &
+            raw_df["Backend 🏭"].isin(backends) &
+            raw_df["Datatype 📥"].isin(datatypes) &
+            (raw_df["Average H4 Score ⬆️"] >= threshold)
+        ]
+        filtered_df["Average H4 Score ⬆️"] = filtered_df["Average H4 Score ⬆️"].apply(
+            make_clickable_score)
 
-    filtered_multi = multi_df[
-        multi_df["Model 🤗"].str.contains(text) &
-        multi_df["Backend 🏭"].isin(backends) &
-        multi_df["Datatype 📥"].isin(datatypes) &
-        (multi_df["Average H4 Score ⬆️"] >= threshold)
-    ]
+        filtered_dfs.append(filtered_df)
 
-    return filtered_single, filtered_multi
+    return filtered_dfs
 
 
 # Define demo interface
@@ -96,29 +96,29 @@ with demo:
 
     with gr.Row():
         search_bar = gr.Textbox(
-            label="Search 🔎",
-            info="Search for a model and press Submit 🚀",
+            label="Model 🤗",
+            info="Search for a model name",
             elem_id="search-bar",
         )
         backend_checkboxes = gr.CheckboxGroup(
+            label="Backends 🏭",
             choices=["pytorch", "onnxruntime"],
             value=["pytorch", "onnxruntime"],
-            label="Backends 🏭",
             info="Select the backends",
             elem_id="backend-checkboxes",
         )
         datatype_checkboxes = gr.CheckboxGroup(
+            label="Datatypes 📥",
             choices=["float32", "float16"],
             value=["float32", "float16"],
-            label="Datatypes 📥",
             info="Select the load datatypes",
             elem_id="datatype-checkboxes",
         )
 
     with gr.Row():
         threshold_slider = gr.Slider(
-            label="H4 Threshold 📈",
-            info="Filter by average H4 score",
+            label="Average H4 Score 📈",
+            info="Filter by minimum average H4 score",
             value=0.0,
             elem_id="threshold-slider",
         )
@@ -132,13 +132,6 @@ with demo:
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("🖥️ A100-80GB Benchmark 🏋️", elem_id="A100-benchmark", id=0):
-
-            SINGLE_A100_TEXT = """<h3>Single-GPU (1xA100):</h3>
-            <ul>
-                <li>Singleton Batch (1)</li>
-                <li>Thousand Tokens (1000)</li>
-            </ul>
-            """
             gr.HTML(SINGLE_A100_TEXT)
 
             single_A100_df = get_benchmark_df(benchmark="1xA100-80GB")
@@ -158,35 +151,15 @@ with demo:
                 visible=False,
             )
 
-        with gr.TabItem("🖥️ 4xA100-80GB Benchmark 🏋️", elem_id="4xA100-benchmark", id=1):
-            MULTI_A100_TEXT = """<h3>Multi-GPU (4xA100):</h3>
-            <ul>
-                <li>Singleton Batch (1)</li>
-                <li>Thousand Tokens (1000)</li>
-                <li>Using <a href="https://huggingface.co/docs/accelerate" target="_blank">Accelerate</a>'s Auto Device Map</li>
-            </ul>"""
-            gr.HTML(MULTI_A100_TEXT)
-            multi_A100_df = get_benchmark_df(benchmark="4xA100-80GB")
-            multi_A100_leaderboard = gr.components.Dataframe(
-                value=multi_A100_df,
-                datatype=COLUMNS_DATATYPES,
-                headers=list(COLUMNS_MAPPING.values()),
-                elem_id="4xA100-table",
-            )
-            # Dummy Leaderboard table for handling the case when the user uses backspace key
-            multi_A100_for_search = gr.components.Dataframe(
-                value=multi_A100_df,
-                datatype=COLUMNS_DATATYPES,
-                headers=list(COLUMNS_MAPPING.values()),
-                max_rows=None,
-                visible=False,
-            )
-
         # Callbacks
-        submit_button.click(submit_query,
-                            [single_A100_for_search, multi_A100_for_search, search_bar,
-                             backend_checkboxes, datatype_checkboxes, threshold_slider],
-                            [single_A100_leaderboard, multi_A100_leaderboard])
+        submit_button.click(
+            submit_query,
+            [
+                search_bar, backend_checkboxes, datatype_checkboxes, threshold_slider,
+                single_A100_for_search
+            ],
+            [single_A100_leaderboard]
+        )
 
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):
@@ -196,13 +169,13 @@ with demo:
                 elem_id="citation-button",
             ).style(show_copy_button=True)
 
-    dummy = gr.Textbox(visible=False)
-    demo.load(
-        change_tab,
-        dummy,
-        tabs,
-        _js=get_window_url_params,
-    )
+    # dummy = gr.Textbox(visible=False)
+    # demo.load(
+    #     change_tab,
+    #     dummy,
+    #     tabs,
+    #     _js=get_window_url_params,
+    # )
 
 # Restart space every hour
 scheduler = BackgroundScheduler()

src/assets/text_content.py

@@ -9,6 +9,14 @@ Anyone from the community can request a model or a hardware+backend configuratio
 [Config files](https://github.com/huggingface/optimum-benchmark/blob/main/examples/bert.yaml) (which can be used with Optimum-Benchmark) will be available soon for reproduction, questioning and correction of our results.
 """
 
+SINGLE_A100_TEXT = """<h3>Single-GPU (1xA100):</h3>
+<ul>
+    <li>Singleton Batch (1)</li>
+    <li>Thousand Tokens (1000)</li>
+</ul>
+"""
+
+
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results."
 CITATION_BUTTON_TEXT = r"""@misc{open-llm-perf-leaderboard,
   author = {Ilyas Moutawwakil},

src/utils.py

@@ -60,3 +60,12 @@ def make_clickable_model(model_name):
         link = OASST_LINK
 
     return model_hyperlink(link, model_name)
+
+
+def make_clickable_score(score):
+    link = f"https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard"
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{score}</a>'
+
+
+def extract_score_from_clickable(clickable_score) -> float:
+    return float(clickable_score.split(">")[1].split("<")[0])