update table and plots

src/components/filters.py

@@ -12,11 +12,24 @@ def render_grouping_options() -> List[str]:
         "flash_attn",
         "cache_type_k",
         "cache_type_v",
-        "PP Value",
-        "TG Value",
+        "PP Config",
+        "TG Config",
+        "n_context",
+        "n_batch",
+        "n_ubatch",
     ]
 
-    default_groups = ["Model ID", "Device", "Platform"]
+    default_groups = [
+        "Model ID",
+        "Device",
+        "Platform",
+        "n_threads",
+        "flash_attn",
+        "cache_type_k",
+        "cache_type_v",
+        "PP Config",
+        "TG Config",
+    ]
 
     selected_groups = st.multiselect(
         "Group Results By",
@@ -39,10 +52,12 @@ def render_column_visibility() -> Set[str]:
             "Memory Usage (%)",
         ],
         "Benchmark Info": [
-            "PP Value",
-            "TG Value",
-            "Prompt Processing",
-            "Token Generation",
+            "PP Config",
+            "TG Config",
+            "Prompt Processing (mean)",
+            "Prompt Processing (std)",
+            "Token Generation (mean)",
+            "Token Generation (std)",
         ],
         "Model Info": [
             "Model",
@@ -54,6 +69,9 @@ def render_column_visibility() -> Set[str]:
             "flash_attn",
             "cache_type_k",
             "cache_type_v",
+            "n_context",
+            "n_batch",
+            "n_ubatch",
         ],
     }
 
@@ -63,8 +81,16 @@ def render_column_visibility() -> Set[str]:
         "Platform",
         "Model",
         "Model Size",
-        "Prompt Processing",
-        "Token Generation",
+        "Prompt Processing (mean)",
+        "Prompt Processing (std)",
+        "Token Generation (mean)",
+        "Token Generation (std)",
+        "n_threads",
+        "flash_attn",
+        "cache_type_k",
+        "cache_type_v",
+        "PP Config",
+        "TG Config",
     }
 
     with st.expander("Column Visibility", expanded=False):

src/components/visualizations.py

@@ -4,11 +4,16 @@ import pandas as pd
 from typing import Optional, Dict, List, Set
 
 
-def create_performance_plot(df: pd.DataFrame, metric: str, title: str):
+def create_performance_plot(
+    df: pd.DataFrame, metric: str, title: str, hover_data: List[str] = None
+):
     """Create a performance comparison plot"""
     if df.empty:
         return None
 
+    if hover_data is None:
+        hover_data = ["CPU Cores", "Memory Usage (GB)"]
+
     fig = px.bar(
         df,
         x="Device",
@@ -17,11 +22,11 @@ def create_performance_plot(df: pd.DataFrame, metric: str, title: str):
         title=title,
         template="plotly_white",
         barmode="group",
-        hover_data=["CPU Cores", "Memory Usage (GB)", "Model Size"],
+        hover_data=hover_data,
     )
     fig.update_layout(
         xaxis_title="Device",
-        yaxis_title="Time (ms)",
+        yaxis_title="Token/sec",
         legend_title="Platform",
         plot_bgcolor="white",
         height=400,
@@ -34,14 +39,16 @@ def filter_dataframe(df: pd.DataFrame, filters: Dict) -> pd.DataFrame:
     if df.empty:
         return df
 
+    filtered_df = df.copy()
+
     # Basic filters
     basic_filters = filters["basic_filters"]
     if basic_filters["model"] != "All":
-        df = df[df["Model ID"] == basic_filters["model"]]
+        filtered_df = filtered_df[filtered_df["Model ID"] == basic_filters["model"]]
     if basic_filters["platform"] != "All":
-        df = df[df["Platform"] == basic_filters["platform"]]
+        filtered_df = filtered_df[filtered_df["Platform"] == basic_filters["platform"]]
     if basic_filters["device"] != "All":
-        df = df[df["Device"] == basic_filters["device"]]
+        filtered_df = filtered_df[filtered_df["Device"] == basic_filters["device"]]
 
     # Benchmark configuration filters
     benchmark_config = filters["benchmark_config"]
@@ -49,45 +56,44 @@ def filter_dataframe(df: pd.DataFrame, filters: Dict) -> pd.DataFrame:
         pp_min, pp_max = benchmark_config["pp_range"]
         tg_min, tg_max = benchmark_config["tg_range"]
 
-        # Extract PP/TG values if not already present
-        if "PP Value" not in df.columns:
-            df["PP Value"] = df["Benchmark"].apply(
-                lambda x: int(x.split("pp: ")[1].split(",")[0])
-            )
-        if "TG Value" not in df.columns:
-            df["TG Value"] = df["Benchmark"].apply(
-                lambda x: int(x.split("tg: ")[1].split(")")[0])
-            )
+        pp_values = filtered_df["PP Config"]
+        tg_values = filtered_df["TG Config"]
 
-        df = df[
-            (df["PP Value"] >= pp_min)
-            & (df["PP Value"] <= pp_max)
-            & (df["TG Value"] >= tg_min)
-            & (df["TG Value"] <= tg_max)
+        filtered_df = filtered_df[
+            (pp_values >= pp_min)
+            & (pp_values <= pp_max)
+            & (tg_values >= tg_min)
+            & (tg_values <= tg_max)
         ]
 
     # Advanced settings filters
     advanced = filters["advanced_settings"]
     if advanced["n_threads"]:
-        df["n_threads"] = df["initSettings"].apply(lambda x: x.get("n_threads"))
-        df = df[df["n_threads"].isin(advanced["n_threads"])]
+        n_threads = filtered_df["initSettings"].apply(lambda x: x.get("n_threads"))
+        filtered_df = filtered_df[n_threads.isin(advanced["n_threads"])]
 
     if advanced["flash_attn"]:
-        df["flash_attn"] = df["initSettings"].apply(lambda x: x.get("flash_attn"))
-        df = df[df["flash_attn"].isin(advanced["flash_attn"])]
+        flash_attn = filtered_df["initSettings"].apply(lambda x: x.get("flash_attn"))
+        filtered_df = filtered_df[flash_attn.isin(advanced["flash_attn"])]
 
     if advanced["cache_type"]:
-        df["cache_type_k"] = df["initSettings"].apply(lambda x: x.get("cache_type_k"))
-        df["cache_type_v"] = df["initSettings"].apply(lambda x: x.get("cache_type_v"))
-        df = df[
-            (df["cache_type_k"].isin(advanced["cache_type"]))
-            & (df["cache_type_v"].isin(advanced["cache_type"]))
+        cache_type_k = filtered_df["initSettings"].apply(
+            lambda x: x.get("cache_type_k")
+        )
+        cache_type_v = filtered_df["initSettings"].apply(
+            lambda x: x.get("cache_type_v")
+        )
+        filtered_df = filtered_df[
+            (cache_type_k.isin(advanced["cache_type"]))
+            & (cache_type_v.isin(advanced["cache_type"]))
         ]
 
     if advanced["max_memory_usage"] < 100:
-        df = df[df["Memory Usage (%)"] <= advanced["max_memory_usage"]]
+        filtered_df = filtered_df[
+            filtered_df["Memory Usage (%)"] <= advanced["max_memory_usage"]
+        ]
 
-    return df
+    return filtered_df
 
 
 def render_performance_plots(df: pd.DataFrame, filters: Dict):
@@ -102,15 +108,15 @@ def render_performance_plots(df: pd.DataFrame, filters: Dict):
         st.warning("No data matches the selected filters for plotting.")
         return
 
-    # Extract PP/TG values if not already present
-    if "PP Value" not in filtered_df.columns:
-        filtered_df["PP Value"] = filtered_df["Benchmark"].apply(
-            lambda x: int(x.split("pp: ")[1].split(",")[0])
-        )
-    if "TG Value" not in filtered_df.columns:
-        filtered_df["TG Value"] = filtered_df["Benchmark"].apply(
-            lambda x: int(x.split("tg: ")[1].split(")")[0])
-        )
+    ## # Extract PP/TG values if not already present
+    ## if "PP Value" not in filtered_df.columns:
+    ##     filtered_df["PP Value"] = filtered_df["Benchmark"].apply(
+    ##         lambda x: int(x.split("pp: ")[1].split(",")[0])
+    ##     )
+    ## if "TG Value" not in filtered_df.columns:
+    ##     filtered_df["TG Value"] = filtered_df["Benchmark"].apply(
+    ##         lambda x: int(x.split("tg: ")[1].split(")")[0])
+    ##     )
 
     # Extract initSettings if not already present
     if "n_threads" not in filtered_df.columns:
@@ -127,30 +133,83 @@ def render_performance_plots(df: pd.DataFrame, filters: Dict):
             lambda x: x.get("cache_type_v")
         )
 
-    # Group by device and platform for plotting
-    plot_group = (
-        filtered_df.groupby(["Device", "Platform"])
-        .agg(
-            {
-                "Prompt Processing": "mean",
-                "Token Generation": "mean",
-                "Memory Usage (%)": "mean",
-                "Memory Usage (GB)": "mean",
-                "CPU Cores": "first",
-                "Model Size": "first",
-                "PP Value": "first",
-                "TG Value": "first",
-            }
-        )
-        .reset_index()
+    # Build aggregation dictionary based on available columns
+    agg_dict = {}
+
+    # Always include performance metrics
+    agg_dict.update(
+        {
+            "Prompt Processing": "mean",
+            "Token Generation": "mean",
+        }
+    )
+
+    # Include memory metrics if available
+    if "Memory Usage (%)" in filtered_df.columns:
+        agg_dict["Memory Usage (%)"] = "mean"
+    if "Memory Usage (GB)" in filtered_df.columns:
+        agg_dict["Memory Usage (GB)"] = "mean"
+
+    # Include device info if available
+    if "CPU Cores" in filtered_df.columns:
+        agg_dict["CPU Cores"] = "first"
+
+    # Include config values
+    agg_dict.update(
+        {
+            "PP Config": "first",
+            "TG Config": "first",
+        }
     )
 
+    # Group by device and platform for plotting
+    plot_group = filtered_df.groupby(["Device", "Platform"]).agg(agg_dict).reset_index()
+
+    # Flatten column names and rename them
+    # plot_group.columns = [
+    #     col[0] if col[1] == "" else f"{col[0]} ({col[1]})" for col in plot_group.columns
+    # ]
+    # print("plot_group2:", plot_group)
+
+    # Rename columns for display
+    column_mapping = {
+        "Prompt Processing": "PP Avg (t/s)",
+        #"Prompt Processing (std)": "PP Std (t/s)",
+        "Prompt Processing (count)": "Runs",
+        "Token Generation": "TG Avg (t/s)",
+        #"Token Generation (std)": "TG Std (t/s)",
+        "Memory Usage (%) (mean)": "Memory Usage (%)",
+        "Memory Usage (GB) (mean)": "Memory Usage (GB)",
+        "PP Config (first)": "PP Config",
+        "TG Config (first)": "TG Config",
+        "Model Size (first)": "Model Size",
+        "CPU Cores (first)": "CPU Cores",
+        "Total Memory (GB) (first)": "Total Memory (GB)",
+        "n_threads (first)": "n_threads",
+        "flash_attn (first)": "flash_attn",
+        "cache_type_k (first)": "cache_type_k",
+        "cache_type_v (first)": "cache_type_v",
+        "n_context (first)": "n_context",
+        "n_batch (first)": "n_batch",
+        "n_ubatch (first)": "n_ubatch",
+    }
+    plot_group = plot_group.rename(columns=column_mapping)
+
+    # Define hover data based on available columns
+    hover_data = []
+    if "CPU Cores" in plot_group.columns:
+        hover_data.append("CPU Cores")
+    if "Memory Usage (GB)" in plot_group.columns:
+        hover_data.append("Memory Usage (GB)")
+
+    # Create plots
     col1, col2 = st.columns(2)
     with col1:
         fig1 = create_performance_plot(
             plot_group,
-            "Prompt Processing",
-            f"Prompt Processing Time (PP: {plot_group['PP Value'].iloc[0]})",
+            "PP Avg (t/s)",
+            f"Prompt Processing (PP: {plot_group['PP Config'].iloc[0]})",
+            hover_data=hover_data,
         )
         if fig1:
             st.plotly_chart(fig1, use_container_width=True)
@@ -158,8 +217,9 @@ def render_performance_plots(df: pd.DataFrame, filters: Dict):
     with col2:
         fig2 = create_performance_plot(
             plot_group,
-            "Token Generation",
-            f"Token Generation Time (TG: {plot_group['TG Value'].iloc[0]})",
+            "TG Avg (t/s)",
+            f"Token Generation (TG: {plot_group['TG Config'].iloc[0]})",
+            hover_data=hover_data,
         )
         if fig2:
             st.plotly_chart(fig2, use_container_width=True)
@@ -177,49 +237,46 @@ def render_leaderboard_table(df: pd.DataFrame, filters: Dict):
         st.warning("No data matches the selected filters.")
         return
 
-    # Extract settings from benchmark results
-    filtered_df["PP Value"] = filtered_df["Benchmark"].apply(
-        lambda x: int(x.split("pp: ")[1].split(",")[0])
-    )
-    filtered_df["TG Value"] = filtered_df["Benchmark"].apply(
-        lambda x: int(x.split("tg: ")[1].split(")")[0])
-    )
-
-    # Extract initSettings
-    filtered_df["n_threads"] = filtered_df["initSettings"].apply(
-        lambda x: x.get("n_threads")
-    )
-    filtered_df["flash_attn"] = filtered_df["initSettings"].apply(
-        lambda x: x.get("flash_attn")
-    )
-    filtered_df["cache_type_k"] = filtered_df["initSettings"].apply(
-        lambda x: x.get("cache_type_k")
-    )
-    filtered_df["cache_type_v"] = filtered_df["initSettings"].apply(
-        lambda x: x.get("cache_type_v")
-    )
-
     # Group by selected columns
     grouping_cols = filters["grouping"]
     if not grouping_cols:
         grouping_cols = ["Model ID", "Device", "Platform"]  # Default grouping
 
+    # Define aggregations (excluding grouping columns)
     agg_dict = {
-        "Prompt Processing": ["mean", "count", "std"],
-        "Token Generation": ["mean", "std"],
-        "Memory Usage (%)": "mean",
-        "Memory Usage (GB)": "mean",
-        "Total Memory (GB)": "first",
-        "CPU Cores": "first",
-        "Model Size": "first",
-        "PP Value": "first",
-        "TG Value": "first",
-        "n_threads": "first",
-        "flash_attn": "first",
-        "cache_type_k": "first",
-        "cache_type_v": "first",
+        col: agg
+        for col, agg in {
+            "Prompt Processing": ["mean", "std"],
+            "Token Generation": ["mean", "std"],
+            #"Memory Usage (%)": "mean",
+            "Memory Usage (GB)": "mean", # For a given model, device, platform, mem should be the same.
+            "Total Memory (GB)": "first", # For a given model, device, platform, mem should be the same.
+            "CPU Cores": "first", # For a given model, device, platform, cpu cores should be the same.
+            "Model Size": "first", # model size should be the same for all.
+        }.items()
+        if col not in grouping_cols
     }
 
+    # Extract initSettings if needed
+    init_settings_cols = {
+        "n_threads": "n_threads",
+        "flash_attn": "flash_attn",
+        "cache_type_k": "cache_type_k",
+        "cache_type_v": "cache_type_v",
+        "n_context": "n_context",
+        "n_batch": "n_batch",
+        "n_ubatch": "n_ubatch",
+    }
+
+    for col, setting in init_settings_cols.items():
+        if col not in filtered_df.columns:
+            filtered_df[col] = filtered_df["initSettings"].apply(
+                lambda x: x.get(setting)
+            )
+            if col not in grouping_cols:
+                agg_dict[col] = "first"
+
+    # Group and aggregate
     grouped_df = filtered_df.groupby(grouping_cols).agg(agg_dict).reset_index()
 
     # Flatten column names
@@ -227,12 +284,6 @@ def render_leaderboard_table(df: pd.DataFrame, filters: Dict):
         col[0] if col[1] == "" else f"{col[0]} ({col[1]})" for col in grouped_df.columns
     ]
 
-    # Sort by Model Size, PP Value, and TG time
-    grouped_df = grouped_df.sort_values(
-        by=["Model Size (first)", "PP Value (first)", "Token Generation (mean)"],
-        ascending=[False, True, True],
-    )
-
     # Round numeric columns
     numeric_cols = [
         col
@@ -241,17 +292,41 @@ def render_leaderboard_table(df: pd.DataFrame, filters: Dict):
     ]
     grouped_df[numeric_cols] = grouped_df[numeric_cols].round(2)
 
+    # Sort using the actual column names we have
+    sort_cols = []
+    if "Model Size (first)" in grouped_df.columns:
+        sort_cols.append("Model Size (first)")
+    if "PP Config (first)" in grouped_df.columns:
+        sort_cols.append("PP Config (first)")
+    if "Token Generation (mean)" in grouped_df.columns:
+        sort_cols.append("Token Generation (mean)")
+
+    if sort_cols:  # Only sort if we have columns to sort by
+        grouped_df = grouped_df.sort_values(
+            by=sort_cols, ascending=[False] + [True] * (len(sort_cols) - 1)
+        )
+
     # Rename columns for display
     column_mapping = {
-        "Prompt Processing (mean)": "PP Avg (ms)",
-        "Prompt Processing (std)": "PP Std",
+        "Prompt Processing (mean)": "PP Avg (t/s)",
+        "Prompt Processing (std)": "PP Std (t/s)",
         "Prompt Processing (count)": "Runs",
-        "Token Generation (mean)": "TG Avg (ms)",
-        "Token Generation (std)": "TG Std",
+        "Token Generation (mean)": "TG Avg (t/s)",
+        "Token Generation (std)": "TG Std (t/s)",
         "Memory Usage (%) (mean)": "Memory Usage (%)",
         "Memory Usage (GB) (mean)": "Memory Usage (GB)",
-        "PP Value (first)": "PP Value",
-        "TG Value (first)": "TG Value",
+        "PP Config (first)": "PP Config",
+        "TG Config (first)": "TG Config",
+        "Model Size (first)": "Model Size",
+        "CPU Cores (first)": "CPU Cores",
+        "Total Memory (GB) (first)": "Total Memory (GB)",
+        "n_threads (first)": "n_threads",
+        "flash_attn (first)": "flash_attn",
+        "cache_type_k (first)": "cache_type_k",
+        "cache_type_v (first)": "cache_type_v",
+        "n_context (first)": "n_context",
+        "n_batch (first)": "n_batch",
+        "n_ubatch (first)": "n_ubatch",
     }
     grouped_df = grouped_df.rename(columns=column_mapping)
 
@@ -262,38 +337,63 @@ def render_leaderboard_table(df: pd.DataFrame, filters: Dict):
         column_name_mapping = {
             "Device": "Device",
             "Platform": "Platform",
-            "CPU Cores": "CPU Cores (first)",
-            "Total Memory (GB)": "Total Memory (GB) (first)",
+            "CPU Cores": "CPU Cores",
+            "Total Memory (GB)": "Total Memory (GB)",
             "Memory Usage (%)": "Memory Usage (%)",
-            "PP Value": "PP Value",
-            "TG Value": "TG Value",
-            "Prompt Processing": "PP Avg (ms)",
-            "Token Generation": "TG Avg (ms)",
+            "PP Config": "PP Config",
+            "TG Config": "TG Config",
+            "Prompt Processing (mean)": "PP Avg (t/s)",
+            "Token Generation (mean)": "TG Avg (t/s)",
+            "Prompt Processing (std)": "PP Std (t/s)",
+            "Token Generation (std)": "TG Std (t/s)",
             "Model": "Model ID",
-            "Model Size": "Model Size (first)",
+            "Model Size": "Model Size",
             "Model ID": "Model ID",
-            "n_threads": "n_threads (first)",
-            "flash_attn": "flash_attn (first)",
-            "cache_type_k": "cache_type_k (first)",
-            "cache_type_v": "cache_type_v (first)",
+            "n_threads": "n_threads",
+            "flash_attn": "flash_attn",
+            "cache_type_k": "cache_type_k",
+            "cache_type_v": "cache_type_v",
+            "n_context": "n_context",
+            "n_batch": "n_batch",
+            "n_ubatch": "n_ubatch",
         }
-        display_cols = [
-            column_name_mapping[col]
-            for col in visible_cols
-            if col in column_name_mapping
-        ]
+
+        # Convert visible columns and grouping columns to their mapped names
+        mapped_visible = {column_name_mapping.get(col, col) for col in visible_cols}
+        mapped_grouping = {
+            column_name_mapping.get(col, col) for col in filters["grouping"]
+        }
+
+        # Combine both sets to get unique columns
+        all_cols = mapped_visible | mapped_grouping
+
+        # Create final display columns list while preserving grouping columns order
+        display_cols = []
+
+        # First add grouping columns in their original order
+        for col in filters["grouping"]:
+            mapped_col = column_name_mapping.get(col, col)
+            if mapped_col in all_cols:
+                display_cols.append(mapped_col)
+                all_cols.remove(mapped_col)
+
+        # Then add remaining columns
+        display_cols.extend(sorted(all_cols))
     else:
         # Default columns if none selected
         display_cols = [
             "Device",
             "Platform",
             "Model ID",
-            "Model Size (first)",
+            "Model Size",
             "PP Avg (ms)",
             "TG Avg (ms)",
             "Memory Usage (%)",
         ]
 
+    # Ensure all display columns exist in the DataFrame
+    display_cols = [col for col in display_cols if col in grouped_df.columns]
+
     # Display the filtered and grouped table
     st.dataframe(
         grouped_df[display_cols],

src/services/firebase.py

@@ -68,6 +68,8 @@ def format_leaderboard_data(submissions: List[dict]) -> pd.DataFrame:
                     "Device": device_info.get("model", "Unknown"),
                     "Platform": device_info.get("systemName", "Unknown"),
                     "Benchmark": f"{benchmark_result.get('config', {}).get('label', 'Unknown')} (pp: {benchmark_result.get('config', {}).get('pp', 'N/A')}, tg: {benchmark_result.get('config', {}).get('tg', 'N/A')})",
+                    "PP Config": benchmark_result.get("config", {}).get("pp", "N/A"),
+                    "TG Config": benchmark_result.get("config", {}).get("tg", "N/A"),
                     "Model": benchmark_result.get("modelName", "Unknown"),
                     "Model Size": format_params_in_b(
                         benchmark_result.get("modelNParams", 0)
@@ -97,6 +99,7 @@ def format_leaderboard_data(submissions: List[dict]) -> pd.DataFrame:
                     "Model ID": benchmark_result.get("modelId", "Unknown"),
                     "OID": benchmark_result.get("oid"),
                     "initSettings": benchmark_result.get("initSettings"),
+                    "Version": device_info.get("version", "Unknown"),
                 }
             )
         except Exception as e: