filtering plot

app.py

@@ -5,7 +5,7 @@ import plotly.express as px
 from apscheduler.schedulers.background import BackgroundScheduler
 
 from src.assets.text_content import TITLE, INTRODUCTION_TEXT, SINGLE_A100_TEXT, CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT
-from src.utils import restart_space, load_dataset_repo, make_clickable_model, make_clickable_score, submit_query
+from src.utils import restart_space, load_dataset_repo, make_clickable_model, make_clickable_score
 from src.assets.css_html_js import custom_css
 
 
@@ -16,10 +16,11 @@ OPTIMUM_TOKEN = os.environ.get("OPTIMUM_TOKEN", None)
 COLUMNS_MAPPING = {
     "model": "Model 🤗",
     "backend.name": "Backend 🏭",
-    "backend.torch_dtype": "Datatype 📥",
+    "backend.torch_dtype": "Load Dtype 📥",
     "forward.peak_memory(MB)": "Peak Memory (MB) ⬇️",
     "generate.throughput(tokens/s)": "Throughput (tokens/s) ⬆️",
-    "h4_score": "Average H4 Score ⬆️",
+    "h4_score": "Average Open LLM Score ⬆️",
+
 }
 COLUMNS_DATATYPES = ["markdown", "str", "str", "number", "number", "markdown"]
 SORTING_COLUMN = ["Throughput (tokens/s) ⬆️"]
@@ -28,7 +29,7 @@ SORTING_COLUMN = ["Throughput (tokens/s) ⬆️"]
 llm_perf_dataset_repo = load_dataset_repo(LLM_PERF_DATASET_REPO, OPTIMUM_TOKEN)
 
 
-def get_benchmark_df(benchmark):
+def get_benchmark_df(benchmark="1xA100-80GB"):
     if llm_perf_dataset_repo:
         llm_perf_dataset_repo.git_pull()
 
@@ -39,41 +40,38 @@ def get_benchmark_df(benchmark):
         f"./llm-perf-dataset/reports/additional_data.csv")
     bench_df = bench_df.merge(scores_df, on="model", how="left")
 
-    # preprocess
-    bench_df["model"] = bench_df["model"].apply(make_clickable_model)
-    bench_df["h4_score"] = bench_df["h4_score"].apply(make_clickable_score)
+    return bench_df
+
+
+def get_benchmark_table(bench_df):
+
     # filter
     bench_df = bench_df[list(COLUMNS_MAPPING.keys())]
     # rename
     bench_df.rename(columns=COLUMNS_MAPPING, inplace=True)
     # sort
     bench_df.sort_values(by=SORTING_COLUMN, ascending=False, inplace=True)
+    # transform
+    bench_df["Model 🤗"] = bench_df["Model 🤗"].apply(make_clickable_model)
+    bench_df["Average Open LLM Score ⬆️"] = bench_df["Average Open LLM Score ⬆️"].apply(
+        make_clickable_score)
 
     return bench_df
 
 
-# Dataframes
-single_A100_df = get_benchmark_df(benchmark="1xA100-80GB")
-
-
-def get_benchmark_plot(benchmark):
-    if llm_perf_dataset_repo:
-        llm_perf_dataset_repo.git_pull()
-
-    # load
-    bench_df = pd.read_csv(
-        f"./llm-perf-dataset/reports/{benchmark}.csv")
-    scores_df = pd.read_csv(
-        f"./llm-perf-dataset/reports/additional_data.csv")
-    bench_df = bench_df.merge(scores_df, on="model", how="left")
+def get_benchmark_plot(bench_df):
 
+    # untill falcon gets fixed / natively supported
     bench_df = bench_df[bench_df["generate.latency(s)"] < 100]
 
     fig = px.scatter(
-        bench_df, x="h4_score", y="generate.latency(s)",
+        bench_df, x="generate.latency(s)", y="h4_score",
         color='model_type', symbol='backend.name', size='forward.peak_memory(MB)',
         custom_data=['model', 'backend.name', 'backend.torch_dtype',
                      'forward.peak_memory(MB)', 'generate.throughput(tokens/s)'],
+        symbol_sequence=['triangle-up', 'circle'],
+        # as many distinct colors as there are model_type,backend.name couples
+        color_discrete_sequence=px.colors.qualitative.Light24,
     )
 
     fig.update_layout(
@@ -83,11 +81,18 @@ def get_benchmark_plot(benchmark):
             'xanchor': 'center',
             'yanchor': 'top'
         },
-        xaxis_title="Average H4 Score",
-        yaxis_title="Latency per 1000 Tokens (s)",
-        legend_title="Model Type, Backend",
-        width=1200,
+        xaxis_title="Per 1000 Tokens Latency (s)",
+        yaxis_title="Average Open LLM Score",
+        legend_title="Model Type and Backend",
+        width=1000,
         height=600,
+        legend=dict(
+            orientation="h",
+            yanchor="bottom",
+            y=-0.35,
+            xanchor="center",
+            x=0.5
+        )
     )
 
     fig.update_traces(
@@ -97,16 +102,35 @@ def get_benchmark_plot(benchmark):
             "Datatype: %{customdata[2]}",
             "Peak Memory (MB): %{customdata[3]}",
             "Throughput (tokens/s): %{customdata[4]}",
-            "Latency per 1000 Tokens (s): %{y}",
-            "Average H4 Score: %{x}"
+            "Per 1000 Tokens Latency (s): %{y}",
+            "Average Open LLM Score: %{x}",
         ])
     )
 
     return fig
 
 
-# Plots
-single_A100_plot = get_benchmark_plot(benchmark="1xA100-80GB")
+def filter_query(text, backends, datatypes, threshold, benchmark="1xA100-80GB"):
+
+    raw_df = get_benchmark_df(benchmark=benchmark)
+
+    filtered_df = raw_df[
+        raw_df["model"].str.lower().str.contains(text.lower()) &
+        raw_df["backend.name"].isin(backends) &
+        raw_df["Dbackend.torch_dtype"].isin(datatypes) &
+        (raw_df["h4_score"] >= threshold)
+    ]
+
+    filtered_table = get_benchmark_table(filtered_df)
+    filtered_plot = get_benchmark_plot(filtered_df)
+
+    return filtered_table, filtered_plot
+
+
+# Dataframes
+single_A100_df = get_benchmark_df(benchmark="1xA100-80GB")
+single_A100_table = get_benchmark_table(single_A100_df)
+single_A100_plot = get_benchmark_plot(single_A100_df)
 
 # Demo interface
 demo = gr.Blocks(css=custom_css)
@@ -142,7 +166,7 @@ with demo:
             elem_id="datatype-checkboxes",
         )
         threshold_slider = gr.Slider(
-            label="Average H4 Score 📈",
+            label="Average Open LLM Score 📈",
             info="lter by minimum average H4 score",
             value=0.0,
             elem_id="threshold-slider",
@@ -161,28 +185,11 @@ with demo:
 
             # Original leaderboard table
             single_A100_leaderboard = gr.components.Dataframe(
-                value=single_A100_df,
+                value=single_A100_table,
                 datatype=COLUMNS_DATATYPES,
                 headers=list(COLUMNS_MAPPING.values()),
                 elem_id="1xA100-table",
             )
-            # Dummy dataframe for search
-            single_A100_for_search = gr.components.Dataframe(
-                value=single_A100_df,
-                datatype=COLUMNS_DATATYPES,
-                headers=list(COLUMNS_MAPPING.values()),
-                max_rows=None,
-                visible=False,
-            )
-
-        submit_button.click(
-            submit_query,
-            [
-                search_bar, backend_checkboxes, datatype_checkboxes, threshold_slider,
-                single_A100_for_search
-            ],
-            [single_A100_leaderboard]
-        )
 
         with gr.TabItem("🖥️ A100-80GB Plot 📊", id=1):
             # Original leaderboard plot
@@ -195,6 +202,12 @@ with demo:
                 show_label=False,
             )
 
+        submit_button.click(
+            filter_query,
+            [search_bar, backend_checkboxes, datatype_checkboxes, threshold_slider],
+            [single_A100_leaderboard]
+        )
+
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):
             citation_button = gr.Textbox(

src/utils.py

@@ -66,23 +66,3 @@ def make_clickable_model(model_name):
 def make_clickable_score(score):
     link = f"https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard"
     return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{score}</a>'
-
-
-def extract_score_from_clickable(clickable_score) -> float:
-    return float(re.findall(r"\d+\.\d+", clickable_score)[-1])
-
-
-def submit_query(text, backends, datatypes, threshold, raw_df):
-    raw_df["Average H4 Score ⬆️"] = raw_df["Average H4 Score ⬆️"].apply(
-        extract_score_from_clickable)
-
-    filtered_df = raw_df[
-        raw_df["Model 🤗"].str.lower().str.contains(text.lower()) &
-        raw_df["Backend 🏭"].isin(backends) &
-        raw_df["Datatype 📥"].isin(datatypes) &
-        (raw_df["Average H4 Score ⬆️"] >= threshold)
-    ]
-
-    filtered_df["Average H4 Score ⬆️"] = filtered_df["Average H4 Score ⬆️"].apply(
-        make_clickable_score)
-    return filtered_df