initial commit

README.md

@@ -6,7 +6,58 @@ colorTo: blue
 sdk: gradio
 sdk_version: 5.8.0
 app_file: app.py
-pinned: false
+pinned: true
+license: apache-2.0
+tags: [edge llm leaderboard, llm edge leaderboard, llm, edge, leaderboard]
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# LLM-perf leaderboard
+
+## 📝 About
+The Edge-LLM Leaderboard is a leaderboard to gauge practical performance and quality of edge LLMs.
+Its aim is to benchmark the performance (throughput and memory) 
+of Large Language Models (LLMs) on Edge hardware - starting with a Raspberry Pi 5 (8GB) based on the ARM Cortex A76 CPU.
+
+Anyone from the community can request a new base model or edge hardware/backend/optimization
+configuration for automated benchmarking:
+
+- Model evaluation requests will be made live soon, in the meantime feel free to email to - arnav[dot]chavan[@]nyunai[dot]com
+
+## ✍️ Details
+
+- To avoid multi-thread discrepencies, all 4 threads are used on the Pi 5.
+- LLMs are running on a singleton batch with a prompt size of 512 and generating 128 tokens.
+
+All of our throughput benchmarks are ran by this single tool
+[llama-bench](https://github.com/ggerganov/llama.cpp/tree/master/examples/llama-bench)
+using the power of [llama.cpp](https://github.com/ggerganov/llama.cpp) to guarantee reproducibility and consistency.
+
+## 🏃 How to run locally 
+
+To run the Edge-LLM Leaderboard locally on your machine, follow these steps:
+
+### 1. Clone the Repository
+
+First, clone the repository to your local machine:
+
+```bash
+git clone https://huggingface.co/spaces/nyunai/edge-llm-leaderboard
+cd edge-llm-leaderboard
+```
+
+### 2. Install the Required Dependencies
+
+Install the necessary Python packages listed in the requirements.txt file:
+`pip install -r requirements.txt`
+
+###  3. Run the Application
+
+You can run the Gradio application in one of the following ways:
+- Option 1: Using Python
+`python app.py`
+- Option 2: Using Gradio CLI (include hot-reload)
+`gradio app.py`
+
+### 4. Access the Application
+
+Once the application is running, you can access it locally in your web browser at http://127.0.0.1:7860/

app.py

@@ -0,0 +1,118 @@
+import gradio as gr
+
+import src.dependency  # noqa
+from src.assets import custom_css
+
+# from src.attention import create_attn_plots
+from src.content import ABOUT, CITATION_BUTTON, CITATION_BUTTON_LABEL, LOGO, TITLE
+from src.hardware import load_hardware_configs
+from src.leaderboard import create_leaderboard_table
+from src.llm_perf import get_llm_perf_df
+from src.panel import (
+    create_control_callback,
+    create_control_panel,
+    create_select_callback,
+)
+
+configs = load_hardware_configs("hardware.yaml")
+
+
+demo = gr.Blocks(
+    css=custom_css,
+    theme=gr.themes.Default(primary_hue="indigo", secondary_hue="indigo"),
+)
+with demo:
+    gr.HTML(LOGO, elem_classes="logo")
+    gr.HTML(TITLE, elem_classes="title")
+    ####################### HARDWARE TABS #######################
+    with gr.Tabs(elem_classes="tabs"):
+        for id, config in enumerate(configs):
+            with gr.TabItem(config.description, id=id):
+                ####################### HARDWARE DETAILS #######################
+                if config.detail:
+                    gr.Markdown(config.detail, elem_classes="descriptive-text")
+
+                ######################## CONTROL PANEL #######################
+                (
+                    filter_button,
+                    machine_value,
+                    backends_value,
+                    hardware_type_value,
+                    memory_slider,
+                    quantization_checkboxes,
+                ) = create_control_panel(
+                    machine=config.machine,
+                    backends=config.backends,
+                    hardware_provider=config.hardware_provider,
+                    hardware_type=config.hardware_type,
+                )
+                ####################### HARDWARE SUBTABS #######################
+                with gr.Tabs(elem_classes="subtabs"):
+                    open_llm_perf_df = get_llm_perf_df(
+                        machine=config.machine,
+                        backends=config.backends,
+                        hardware_type=config.hardware_type,
+                    )
+                    ####################### LEADERBOARD TAB #######################
+                    with gr.TabItem("Leaderboard 🏅", id=0):
+                        search_bar, columns_checkboxes, leaderboard_table = (
+                            create_leaderboard_table(open_llm_perf_df)
+                        )
+                    ###################### ATTENTIONS SPEEDUP TAB #######################
+                    # with gr.TabItem("Attention 📈", id=2):
+                    #     attn_prefill_plot, attn_decode_plot = create_attn_plots(
+                    #         open_llm_perf_df
+                    #     )
+                    # ####################### KERNELS SPEEDUP TAB #######################
+                    # with gr.TabItem("Kernels 📈", id=4):
+                    #     quant_krnl_prefill_plot, quant_krnl_decode_plot = (
+                    #         create_quant_krnl_plots(llm_perf_df)
+                    #     )
+                ####################### CONTROL CALLBACK #######################
+                create_control_callback(
+                    filter_button,
+                    # inputs
+                    machine_value,
+                    backends_value,
+                    hardware_type_value,
+                    memory_slider,
+                    quantization_checkboxes,
+                    # interactive
+                    columns_checkboxes,
+                    search_bar,
+                    # outputs
+                    leaderboard_table,
+                    # attn_prefill_plot,
+                    # attn_decode_plot,
+                    # quant_krnl_prefill_plot,
+                    # quant_krnl_decode_plot,
+                )
+
+                create_select_callback(
+                    # inputs
+                    machine_value,
+                    backends_value,
+                    hardware_type_value,
+                    # interactive
+                    columns_checkboxes,
+                    search_bar,
+                    # outputs
+                    leaderboard_table,
+                )
+
+        ####################### ABOUT TAB #######################
+        with gr.TabItem("About 📖", id=len(configs)):
+            gr.Markdown(ABOUT, elem_classes="descriptive-text")
+    ####################### CITATION
+    with gr.Row():
+        with gr.Accordion("📙 Citation", open=False):
+            citation_button = gr.Textbox(
+                value=CITATION_BUTTON,
+                label=CITATION_BUTTON_LABEL,
+                elem_id="citation-button",
+                show_copy_button=True,
+            )
+
+if __name__ == "__main__":
+    # Launch demo
+    demo.queue().launch()

dataset/llm-perf-leaderboard-Raspberry Pi 5(8GB).csv

@@ -0,0 +1,126 @@
+Model,Quantization,Params (B),Model Size (GB),Prefill (tokens/s),Decode (tokens/s),Backend
+gemma-2-9b,Q8_0,10.159,10.796,2.169,0.012,llama_cpp
+DeepSeek-V2-Lite,Q4_K_M,15.706,10.36,4.304,1.764,llama_cpp
+aya-expanse-8b,Q8_0,9.077,9.644,3.1,0.027,llama_cpp
+Yi-1.5-9B,Q8_0,8.829,9.382,2.585,0.019,llama_cpp
+Qwen2.5-14B,Q4_K_M,14.77,8.982,1.916,0.018,llama_cpp
+DeepSeek-V2-Lite,Q4_0_4_4,15.706,8.901,7.788,3.867,llama_cpp
+Phi-3-medium-128k-instruct,Q4_K_M,13.96,8.566,1.819,0.02,llama_cpp
+Hermes-3-Llama-3.1-8B,Q8_0,8.03,8.533,3.286,0.922,llama_cpp
+Qwen2.5-14B,Q4_0_4_4,14.77,8.512,4.698,0.028,llama_cpp
+internlm2_5-7b-chat,Q8_0,7.738,8.222,3.258,1.238,llama_cpp
+dolphin-2.9.2-qwen2-7b,Q8_0,7.616,8.093,4.241,1.301,llama_cpp
+Qwen2.5-7B,Q8_0,7.616,8.093,4.253,1.302,llama_cpp
+Phi-3-medium-128k-instruct,Q4_0_4_4,13.96,7.896,4.715,0.038,llama_cpp
+NexusRaven-V2-13B,Q4_K_M,13.016,7.865,2.066,0.035,llama_cpp
+Mistral-7B-Instruct-v0.3,Q8_0,7.248,7.702,4.104,1.29,llama_cpp
+dolphin-2.9.3-mistral-7B-32k,Q8_0,7.248,7.702,4.135,1.294,llama_cpp
+Yarn-Mistral-7b-128k,Q8_0,7.242,7.695,4.082,1.292,llama_cpp
+Starling-LM-7B-beta,Q8_0,7.242,7.695,4.132,1.296,llama_cpp
+Mistral-Nemo-Base-2407,Q4_K_M,12.248,7.469,2.453,1.358,llama_cpp
+NexusRaven-V2-13B,Q4_0_4_4,13.016,7.365,4.979,1.348,llama_cpp
+OLMoE-1B-7B-0924,Q8_0,6.919,7.358,26.942,7.489,llama_cpp
+OLMo-7B-0724-hf,Q8_0,6.888,7.319,4.515,1.371,llama_cpp
+mpt-7b-instruct,Q8_0,6.856,7.285,4.287,1.367,llama_cpp
+Amber,Q8_0,6.738,7.16,4.442,1.373,llama_cpp
+Mistral-Nemo-Base-2407,Q4_0_4_4,12.248,7.064,9.103,1.48,llama_cpp
+gemma-2-9b,Q4_K_M,10.159,6.508,3.531,1.629,llama_cpp
+Yarn-Solar-10b-64k,Q4_K_M,10.732,6.461,2.905,1.503,llama_cpp
+SOLAR-10.7B-v1.0,Q4_K_M,10.732,6.461,2.925,1.505,llama_cpp
+SOLAR-10.7B-Instruct-v1.0,Q4_K_M,10.732,6.461,2.916,1.506,llama_cpp
+Yi-1.5-6B,Q8_0,6.061,6.441,5.269,1.584,llama_cpp
+gemma-2-9b,Q4_0_4_4,10.159,6.19,10.553,1.757,llama_cpp
+SOLAR-10.7B-v1.0,Q4_0_4_4,10.732,6.072,9.315,1.635,llama_cpp
+SOLAR-10.7B-Instruct-v1.0,Q4_0_4_4,10.732,6.072,9.332,1.635,llama_cpp
+Yarn-Solar-10b-64k,Q4_0_4_4,10.732,6.072,9.352,1.638,llama_cpp
+aya-expanse-8b,Q4_K_M,9.077,5.906,4.406,1.911,llama_cpp
+aya-23-8B,Q4_K_M,9.077,5.906,4.428,1.914,llama_cpp
+aya-expanse-8b,Q4_0_4_4,9.077,5.647,14.074,2.05,llama_cpp
+aya-23-8B,Q4_0_4_4,9.077,5.647,14.113,2.051,llama_cpp
+Yi-1.5-9B,Q4_K_M,8.829,5.327,3.681,1.85,llama_cpp
+Yi-1.5-9B,Q4_0_4_4,8.829,5.035,11.33,2.0,llama_cpp
+Hermes-3-Llama-3.1-8B,Q4_K_M,8.03,4.913,4.375,2.078,llama_cpp
+Llama-3.1-8B,Q4_K_M,8.03,4.913,4.403,2.086,llama_cpp
+internlm2_5-7b-chat,Q4_K_M,7.738,4.711,4.4,2.133,llama_cpp
+Qwen2.5-7B,Q4_K_M,7.616,4.677,4.769,2.201,llama_cpp
+dolphin-2.9.2-qwen2-7b,Q4_K_M,7.616,4.677,4.759,2.204,llama_cpp
+Llama-3.1-8B,Q4_0_4_4,8.03,4.653,13.99,2.245,llama_cpp
+Hermes-3-Llama-3.1-8B,Q4_0_4_4,8.03,4.653,14.006,2.245,llama_cpp
+internlm2_5-7b-chat,Q4_0_4_4,7.738,4.451,14.036,2.31,llama_cpp
+mpt-7b-instruct,Q4_K_M,6.856,4.442,4.162,2.213,llama_cpp
+Qwen2.5-7B,Q4_0_4_4,7.616,4.425,15.563,2.386,llama_cpp
+dolphin-2.9.2-qwen2-7b,Q4_0_4_4,7.616,4.425,15.58,2.387,llama_cpp
+dolphin-2.9.3-mistral-7B-32k,Q4_K_M,7.248,4.372,4.387,2.227,llama_cpp
+Mistral-7B-Instruct-v0.3,Q4_K_M,7.248,4.372,4.462,2.241,llama_cpp
+Starling-LM-7B-beta,Q4_K_M,7.242,4.368,4.406,2.234,llama_cpp
+Yarn-Mistral-7b-128k,Q4_K_M,7.242,4.368,4.434,2.245,llama_cpp
+OLMoE-1B-7B-0924,Q4_K_M,6.919,4.212,26.902,12.119,llama_cpp
+OLMo-7B-0724-hf,Q4_K_M,6.888,4.183,4.706,2.339,llama_cpp
+dolphin-2.9.3-mistral-7B-32k,Q4_0_4_4,7.248,4.113,14.053,2.427,llama_cpp
+Mistral-7B-Instruct-v0.3,Q4_0_4_4,7.248,4.113,14.177,2.43,llama_cpp
+Starling-LM-7B-beta,Q4_0_4_4,7.242,4.108,14.068,2.427,llama_cpp
+Yarn-Mistral-7b-128k,Q4_0_4_4,7.242,4.108,14.139,2.436,llama_cpp
+Amber,Q4_K_M,6.738,4.08,4.594,2.351,llama_cpp
+Phi-3.5-mini-instruct,Q8_0,3.821,4.06,7.951,2.423,llama_cpp
+Phi-3-mini-128k-instruct,Q8_0,3.821,4.06,7.947,2.426,llama_cpp
+mpt-7b-instruct,Q4_0_4_4,6.856,3.964,14.569,2.533,llama_cpp
+OLMoE-1B-7B-0924,Q4_0_4_4,6.919,3.926,50.413,12.989,llama_cpp
+Amber,Q4_0_4_4,6.738,3.825,14.442,2.57,llama_cpp
+Yi-1.5-6B,Q4_K_M,6.061,3.672,5.58,2.72,llama_cpp
+Qwen2.5-3B,Q8_0,3.397,3.61,10.473,2.939,llama_cpp
+Yi-1.5-6B,Q4_0_4_4,6.061,3.478,17.017,2.945,llama_cpp
+dolphin-2.9.4-gemma2-2b,Q8_0,3.204,3.405,13.966,3.381,llama_cpp
+gemma-2-2b,Q8_0,3.204,3.405,13.996,3.385,llama_cpp
+stable-code-instruct-3b,Q8_0,2.795,2.971,10.668,3.316,llama_cpp
+Phi-3.5-mini-instruct,Q4_K_M,3.821,2.393,7.502,3.936,llama_cpp
+Phi-3-mini-128k-instruct,Q4_K_M,3.821,2.393,7.519,3.938,llama_cpp
+Llama-3.2-3B,Q4_K_M,3.607,2.335,10.691,4.674,llama_cpp
+Llama-3.2-3B,Q4_0_4_4,3.607,2.233,31.72,5.025,llama_cpp
+gemma-2-2b,Q4_K_M,3.204,2.186,14.202,5.253,llama_cpp
+dolphin-2.9.4-gemma2-2b,Q4_K_M,3.204,2.186,14.218,5.253,llama_cpp
+Qwen2.5-3B,Q4_K_M,3.397,2.179,10.638,4.808,llama_cpp
+Phi-3.5-mini-instruct,Q4_0_4_4,3.821,2.175,23.369,4.428,llama_cpp
+Phi-3-mini-128k-instruct,Q4_0_4_4,3.821,2.175,23.461,4.436,llama_cpp
+gemma-2-2b,Q4_0_4_4,3.204,2.107,40.616,5.552,llama_cpp
+dolphin-2.9.4-gemma2-2b,Q4_0_4_4,3.204,2.107,40.977,5.58,llama_cpp
+Qwen2.5-3B,Q4_0_4_4,3.397,2.072,32.434,5.239,llama_cpp
+internlm2_5-1_8b-chat,Q8_0,1.889,2.007,19.329,5.279,llama_cpp
+SmolLM2-1.7B-Instruct,Q8_0,1.812,1.926,17.524,5.177,llama_cpp
+Qwen2.5-1.5B,Q8_0,1.777,1.889,21.927,5.793,llama_cpp
+stable-code-instruct-3b,Q4_K_M,2.795,1.707,10.803,5.564,llama_cpp
+stable-code-instruct-3b,Q4_0_4_4,2.795,1.607,28.926,5.957,llama_cpp
+Yi-Coder-1.5B,Q8_0,1.476,1.569,23.894,6.596,llama_cpp
+OLMo-1B-0724-hf,Q8_0,1.28,1.36,27.787,7.591,llama_cpp
+Qwen2.5-1.5B,Q4_K_M,1.777,1.172,22.326,9.56,llama_cpp
+internlm2_5-1_8b-chat,Q4_K_M,1.889,1.17,19.453,8.56,llama_cpp
+TinyLlama-1.1B-Chat-v1.0,Q8_0,1.1,1.169,28.472,8.637,llama_cpp
+TinyLlama_v1.1,Q8_0,1.1,1.169,28.538,8.652,llama_cpp
+SmolLM2-1.7B-Instruct,Q4_K_M,1.812,1.136,17.72,8.497,llama_cpp
+Qwen2.5-1.5B,Q4_0_4_4,1.777,1.12,65.915,10.128,llama_cpp
+internlm2_5-1_8b-chat,Q4_0_4_4,1.889,1.112,57.736,9.243,llama_cpp
+SmolLM2-1.7B-Instruct,Q4_0_4_4,1.812,1.072,50.27,9.239,llama_cpp
+Llama-3.2-1B,Q4_K_M,1.498,1.015,30.451,11.51,llama_cpp
+Llama-3.2-1B,Q4_0_4_4,1.498,0.979,86.772,12.364,llama_cpp
+Yi-Coder-1.5B,Q4_K_M,1.476,0.962,23.267,10.03,llama_cpp
+Yi-Coder-1.5B,Q4_0_4_4,1.476,0.865,67.713,11.422,llama_cpp
+OLMo-1B-0724-hf,Q4_K_M,1.28,0.79,28.276,12.321,llama_cpp
+OLMo-1B-0724-hf,Q4_0_4_4,1.28,0.746,84.882,13.339,llama_cpp
+Qwen2.5-0.5B,Q8_0,0.63,0.67,75.456,18.06,llama_cpp
+TinyLlama-1.1B-Chat-v1.0,Q4_K_M,1.1,0.667,29.44,14.305,llama_cpp
+TinyLlama_v1.1,Q4_K_M,1.1,0.667,29.397,14.346,llama_cpp
+TinyLlama-1.1B-Chat-v1.0,Q4_0_4_4,1.1,0.636,77.823,15.509,llama_cpp
+TinyLlama_v1.1,Q4_0_4_4,1.1,0.636,77.943,15.543,llama_cpp
+Qwen2.5-0.5B,Q4_K_M,0.63,0.537,52.916,22.324,llama_cpp
+Qwen2.5-0.5B,Q4_0_4_4,0.63,0.491,189.874,26.738,llama_cpp
+gpt2-medium,Q8_0,0.406,0.436,83.423,23.016,llama_cpp
+SmolLM2-360M-Instruct,Q8_0,0.409,0.435,79.518,22.857,llama_cpp
+SmolLM2-360M-Instruct,Q4_K_M,0.409,0.319,55.774,30.718,llama_cpp
+SmolLM2-360M-Instruct,Q4_0_4_4,0.409,0.277,173.275,37.176,llama_cpp
+gpt2-medium,Q4_K_M,0.406,0.269,73.615,33.913,llama_cpp
+gpt2-medium,Q4_0_4_4,0.406,0.247,178.73,37.89,llama_cpp
+gpt2,Q8_0,0.163,0.176,302.932,68.191,llama_cpp
+SmolLM2-135M-Instruct,Q8_0,0.163,0.173,212.146,57.992,llama_cpp
+SmolLM2-135M-Instruct,Q4_K_M,0.163,0.134,153.439,73.272,llama_cpp
+SmolLM2-135M-Instruct,Q4_0_4_4,0.163,0.12,381.667,86.735,llama_cpp
+gpt2,Q4_K_M,0.163,0.111,269.906,92.707,llama_cpp
+gpt2,Q4_0_4_4,0.163,0.105,582.32,101.509,llama_cpp

hardware.yaml

@@ -0,0 +1,10 @@
+- machine: Raspberry Pi 5(8GB)
+  description: Cortex A76
+  hardware_provider: ARM
+  hardware_type: arm cortex a76
+  subsets:
+    - Q8_0
+    - Q4_K_M
+    - Q4_0_4_4
+  backends:
+    - llama_cpp

requirements.txt

@@ -0,0 +1,3 @@
+huggingface_hub
+gradio>=5.0.0
+pandas

src/assets.py

@@ -0,0 +1,53 @@
+custom_css = """
+.logo {
+    width: 300px;
+    height: auto;
+    margin: 0 auto;
+    max-width: 100%
+    object-fit: contain;
+    overflow: visible !important;
+}
+.text {
+    font-size: 16px !important;
+}
+
+.tabs button {
+    font-size: 20px;
+}
+.subtabs button {
+    font-size: 20px;
+}
+
+.descriptive-text span {
+    font-size: 16px !important;
+}
+
+#control-panel span {
+    font-size: 20px !important;
+}
+#search-bar span {
+    font-size: 16px !important;
+}
+#threshold-slider span {
+    font-size: 16px !important;
+}
+#memory-slider span {
+    font-size: 16px !important;
+}
+#columns-checkboxes span {
+    font-size: 16px !important;
+}
+#backend-checkboxes span {
+    font-size: 16px !important;
+}
+#quantization-checkboxes span {
+    font-size: 16px !important;
+}
+
+#leaderboard-table td:first-child,
+#leaderboard-table th:first-child {
+    max-width: 300px;
+    overflow: auto;
+    white-space: nowrap;
+}
+"""

src/content.py

@@ -0,0 +1,35 @@
+LOGO = '<img src="https://nyunai.com/assets/images/logo.png">'
+
+TITLE = """<h1 align="center" id="space-title"> Edge-LLM Leaderboard </h1>"""
+
+ABOUT = """
+## 📝 About
+The Edge-LLM Leaderboard is a leaderboard to gauge practical performance and quality of edge LLMs.
+Its aim is to benchmark the performance (throughput and memory) 
+of Large Language Models (LLMs) on Edge hardware - starting with a Raspberry Pi 5 (8GB) based on the ARM Cortex A76 CPU.
+
+Anyone from the community can request a new base model or edge hardware/backend/optimization
+configuration for automated benchmarking:
+
+- Model evaluation requests will be made live soon, in the meantime feel free to email to - arnav[dot]chavan[@]nyunai[dot]com
+
+## ✍️ Details
+
+- To avoid multi-thread discrepencies, all 4 threads are used on the Pi 5.
+- LLMs are running on a singleton batch with a prompt size of 512 and generating 128 tokens.
+
+All of our throughput benchmarks are ran by this single tool
+[llama-bench](https://github.com/ggerganov/llama.cpp/tree/master/examples/llama-bench)
+using the power of [llama.cpp](https://github.com/ggerganov/llama.cpp) to guarantee reproducibility and consistency.
+"""
+
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results."
+CITATION_BUTTON = r"""@misc{edge-llm-leaderboard,
+  author = {},
+  title = {Edge-LLM Leaderboard},
+  year = {2024},
+  publisher = {},
+  howpublished = "\url{https://huggingface.co/spaces/nyunai/edge-llm-leaderboard}",
+}
+"""

src/dependency.py

@@ -0,0 +1,3 @@
+import os
+
+os.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "1"

src/hardware.py

@@ -0,0 +1,26 @@
+from typing import Any, Dict, List, Optional
+
+import yaml
+
+
+class HardwareConfig:
+    def __init__(self, data: Dict[str, Any]):
+        self.machine: str = data["machine"]
+        self.description: str = data["description"]
+        self.hardware_provider: str = data["hardware_provider"]
+        self.hardware_type: str = data["hardware_type"]
+        self.subsets: List[str] = data["subsets"]
+        self.backends: List[str] = data["backends"]
+        self.detail: Optional[str] = data.get("detail", None)
+
+    def __repr__(self) -> str:
+        return (
+            f"HardwareConfig(machine='{self.machine}', description='{self.description}', "
+            f"hardware_provider={self.hardware_provider}, hardware_type={self.hardware_type}, subsets={self.subsets}, backends={self.backends})"
+        )
+
+
+def load_hardware_configs(file_path: str) -> List[HardwareConfig]:
+    with open(file_path, "r") as file:
+        data = yaml.safe_load(file)
+    return [HardwareConfig(config) for config in data]

src/leaderboard.py

@@ -0,0 +1,70 @@
+import gradio as gr
+
+from src.utils import model_hyperlink
+
+LEADERBOARD_COLUMN_TO_DATATYPE = {
+    # open llm
+    "Model": "markdown",
+    "Quantization": "str",
+    # primary measurements
+    "Prefill (tokens/s)": "number",
+    "Decode (tokens/s)": "number",
+    "Model Size (GB)": "number",
+    # deployment settings
+    "Backend": "str",
+    # additional measurements
+    # "Reserved Memory (MB)": "number",
+    # "Used Memory (MB)": "number",
+    "Params (B)": "number",
+}
+
+PRIMARY_COLUMNS = [
+    "Model",
+    "Quantization",
+    "Prefill (tokens/s)",
+    "Decode (tokens/s)",
+    "Model Size (GB)",
+]
+
+
+def process_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+    return model_hyperlink(link, model_name)
+
+
+def get_leaderboard_df(llm_perf_df):
+    df = llm_perf_df.copy()
+    # transform for leaderboard
+    df["Model"] = df["Model"].apply(process_model)
+    return df
+
+
+def create_leaderboard_table(llm_perf_df):
+    # get dataframe
+    leaderboard_df = get_leaderboard_df(llm_perf_df)
+
+    # create search bar
+    with gr.Row():
+        search_bar = gr.Textbox(
+            label="Model",
+            info="🔍 Search for a model name",
+            elem_id="search-bar",
+        )
+    # create checkboxes
+    with gr.Row():
+        columns_checkboxes = gr.CheckboxGroup(
+            label="Columns 📊",
+            value=PRIMARY_COLUMNS,
+            choices=list(LEADERBOARD_COLUMN_TO_DATATYPE.keys()),
+            info="☑️ Select the columns to display",
+            elem_id="columns-checkboxes",
+        )
+    # create table
+    leaderboard_table = gr.components.Dataframe(
+        value=leaderboard_df[PRIMARY_COLUMNS],
+        datatype=list(LEADERBOARD_COLUMN_TO_DATATYPE.values()),
+        headers=list(LEADERBOARD_COLUMN_TO_DATATYPE.keys()),
+        elem_id="leaderboard-table",
+    )
+
+    return search_bar, columns_checkboxes, leaderboard_table

src/llm_perf.py

@@ -0,0 +1,101 @@
+import os
+from typing import List
+
+import pandas as pd
+
+DATASET_DIRECTORY = "dataset"
+
+COLUMNS_MAPPING = {
+    "config.name": "Quantization",
+    "config.backend.model": "Model",
+    # primary measurements
+    "report.prefill.throughput.value": "Prefill (tokens/s)",
+    "report.decode.throughput.value": "Decode (tokens/s)",
+    "report.memory": "Model Size (GB)",
+    # deployment settings
+    "config.backend.name": "Backend",
+    "quantization": "Quantization",
+    # additional information
+    "#Params (B)": "Params (B)",
+}
+SORTING_COLUMNS = ["Model Size (GB)", "Decode (tokens/s)", "Prefill (tokens/s)"]
+SORTING_ASCENDING = [False, True, True]
+
+
+def get_raw_llm_perf_df(
+    machine: str, backends: List[str], hardware_type: str
+):
+    dfs = []
+    try:
+        dfs.append(
+            pd.read_csv("/Users/arnavchavan/leaderboard/benchmark_results.csv")
+            # pd.read_csv(
+            #     f"hf://datasets/nyunai/edge-llm-leaderboard/perf-df-{hardware_type}-{machine}-{backends}.csv"
+            # )
+        )
+    except Exception:
+        print("Dataset not found for:")
+        print(f"  • Machine: {machine}")
+        print(f"  • Hardware Type: {hardware_type}")
+        url = f"https://huggingface.co/datasets/nyunai/edge-llm-leaderboard/blob/main/perf-df-{hardware_type}-{machine}-{backends}.csv"
+        print(f"  • URL: {url}")
+
+    if len(dfs) == 0:
+        raise ValueError(
+            f"No datasets found for machine {machine}, check your hardware.yml config file or your datatset on huggingface"
+        )
+
+    perf_df = pd.concat(dfs)
+    # llm_df = pd.read_csv(
+    #     "hf://datasets/optimum-benchmark/llm-perf-leaderboard/llm-df.csv"
+    # )
+
+    # llm_perf_df = pd.merge(
+    #     llm_df, perf_df, left_on="Model", right_on="config.backend.model"
+    # )
+
+    return perf_df
+
+
+def processed_llm_perf_df(llm_perf_df):
+    # llm_perf_df["architecture"] = llm_perf_df["config.backend.model"].apply(
+    #     process_architectures
+    # )
+    # round numerical columns
+    llm_perf_df = llm_perf_df.round(
+        {
+            "Prefill (tokens/s)": 3,
+            "Decode (tokens/s)": 3,
+            "Model Size (GB)": 3,
+            "#Params (B)": 3,
+        }
+    )
+    # sort by metric
+    llm_perf_df.sort_values(
+        by=SORTING_COLUMNS,
+        ascending=SORTING_ASCENDING,
+        inplace=True,
+    )
+
+    return llm_perf_df
+
+
+def get_llm_perf_df(
+    machine: str, backends: List[str], hardware_type: str
+):
+    if not os.path.exists(DATASET_DIRECTORY):
+        os.makedirs(DATASET_DIRECTORY)
+
+    if os.path.exists(f"{DATASET_DIRECTORY}/llm-perf-leaderboard-{machine}.csv"):
+        llm_perf_df = pd.read_csv(
+            f"{DATASET_DIRECTORY}/llm-perf-leaderboard-{machine}.csv"
+        )
+    else:
+        print(f"Dataset machine {machine} not found, downloading...")
+        llm_perf_df = get_raw_llm_perf_df(machine, backends, hardware_type)
+        llm_perf_df = processed_llm_perf_df(llm_perf_df)
+        llm_perf_df.to_csv(
+            f"{DATASET_DIRECTORY}/llm-perf-leaderboard-{machine}.csv", index=False
+        )
+
+    return llm_perf_df

src/panel.py

@@ -0,0 +1,206 @@
+from typing import List
+
+import gradio as gr
+
+from src.leaderboard import get_leaderboard_df
+from src.llm_perf import get_llm_perf_df
+
+# from attention_implementations import get_attn_decode_fig, get_attn_prefill_fig
+# from custom_kernels import get_kernel_decode_fig, get_kernel_prefill_fig
+
+
+def create_control_panel(
+    machine: str,
+    backends: List[str],
+    hardware_provider: str,
+    hardware_type: str,
+):
+    # controls
+    machine_value = gr.State(value=machine)
+    backends_value = gr.State(value=backends)
+    hardware_type_value = gr.State(value=hardware_type)
+
+    if hardware_provider == "ARM":
+        backends = ["llama.cpp"]
+        quantizations = ["Q8_0", "Q4_K_M", "Q4_0_4_4"]
+    else:
+        raise ValueError(f"Unknown hardware provider: {hardware_provider}")
+
+    with gr.Accordion("Control Panel", open=False, elem_id="control-panel"):
+        with gr.Row():
+            with gr.Column(scale=2, variant="panel"):
+                memory_slider = gr.Slider(
+                    label="Model Size (GB) 📈",
+                    info="🎚️ Slide to maximum Model Size",
+                    minimum=0,
+                    maximum=16,
+                    value=16,
+                    elem_id="memory-slider",
+                )
+            with gr.Column(scale=1, variant="panel"):
+                quantization_checkboxes = gr.CheckboxGroup(
+                    label="Quantizations",
+                    choices=quantizations,
+                    value=quantizations,
+                    info="☑️ Select the quantization schemes",
+                    elem_id="quantization-checkboxes",
+                    elem_classes="boxed-option",
+                )
+        with gr.Row():
+            filter_button = gr.Button(
+                value="Filter 🚀",
+                elem_id="filter-button",
+                elem_classes="boxed-option",
+            )
+
+    return (
+        filter_button,
+        machine_value,
+        backends_value,
+        hardware_type_value,
+        memory_slider,
+        quantization_checkboxes,
+    )
+
+def filter_rows_fn(
+    machine,
+    backends,
+    hardware_type,
+    # inputs
+    memory,
+    quantizations,
+    # interactive
+    columns,
+    search,
+):
+    llm_perf_df = get_llm_perf_df(
+        machine=machine, backends=backends, hardware_type=hardware_type
+    )
+    # print(attentions)
+    # print(llm_perf_df["Attention 👁️"].unique())
+    filtered_llm_perf_df = llm_perf_df[
+        llm_perf_df["Model"].str.contains(search, case=False)
+        & llm_perf_df["Quantization"].isin(quantizations)
+        & (llm_perf_df["Model Size (GB)"] <= memory)
+    ]
+    selected_filtered_llm_perf_df = select_columns_fn(
+        machine, backends, hardware_type, columns, search, filtered_llm_perf_df
+    )
+    # filtered_bt_prefill_fig = get_bt_prefill_fig(filtered_df)
+    # filtered_bt_decode_fig = get_bt_decode_fig(filtered_df)
+    # filtered_fa2_prefill_fig = get_fa2_prefill_fig(filtered_df)
+    # filtered_fa2_decode_fig = get_fa2_decode_fig(filtered_df)
+    # filtered_quant_prefill_fig = get_quant_prefill_fig(filtered_df)
+    # filtered_quant_decode_fig = get_quant_decode_fig(filtered_df)
+
+    return [
+        selected_filtered_llm_perf_df,
+        # filtered_bt_prefill_fig,
+        # filtered_bt_decode_fig,
+        # filtered_fa2_prefill_fig,
+        # filtered_fa2_decode_fig,
+        # filtered_quant_prefill_fig,
+        # filtered_quant_decode_fig,
+    ]
+
+
+def create_control_callback(
+    # button
+    filter_button,
+    # fixed
+    machine_value,
+    backends_value,
+    hardware_type_value,
+    # inputs
+    memory_slider,
+    quantization_checkboxes,
+    # interactive
+    columns_checkboxes,
+    search_bar,
+    # outputs
+    leaderboard_table,
+    # attn_prefill_plot,
+    # attn_decode_plot,
+    # fa2_prefill_plot,
+    # fa2_decode_plot,
+    # quant_prefill_plot,
+    # quant_decode_plot,
+):
+    filter_button.click(
+        fn=filter_rows_fn,
+        inputs=[
+            # fixed
+            machine_value,
+            backends_value,
+            hardware_type_value,
+            # inputs
+            memory_slider,
+            quantization_checkboxes,
+            # interactive
+            columns_checkboxes,
+            search_bar,
+        ],
+        outputs=[
+            leaderboard_table,
+            # attn_prefill_plot,
+            # attn_decode_plot,
+            # fa2_prefill_plot,
+            # fa2_decode_plot,
+            # quant_prefill_plot,
+            # quant_decode_plot,
+        ],
+    )
+
+
+def select_columns_fn(
+    machine, backends, hardware_type, columns, search, llm_perf_df=None
+):
+    if llm_perf_df is None:
+        llm_perf_df = get_llm_perf_df(
+            machine=machine,
+            backends=backends,
+            hardware_type=hardware_type,
+        )
+
+    selected_leaderboard_df = get_leaderboard_df(llm_perf_df)
+    selected_leaderboard_df = selected_leaderboard_df[
+        selected_leaderboard_df["Model"].str.contains(search, case=False)
+    ]
+    selected_leaderboard_df = selected_leaderboard_df[columns]
+
+    return selected_leaderboard_df
+
+
+def create_select_callback(
+    # fixed
+    machine_value,
+    backends_value,
+    hardware_type_value,
+    # interactive
+    columns_checkboxes,
+    search_bar,
+    # outputs
+    leaderboard_table,
+):
+    columns_checkboxes.change(
+        fn=select_columns_fn,
+        inputs=[
+            machine_value,
+            backends_value,
+            hardware_type_value,
+            columns_checkboxes,
+            search_bar,
+        ],
+        outputs=[leaderboard_table],
+    )
+    search_bar.change(
+        fn=select_columns_fn,
+        inputs=[
+            machine_value,
+            backends_value,
+            hardware_type_value,
+            columns_checkboxes,
+            search_bar,
+        ],
+        outputs=[leaderboard_table],
+    )

src/utils.py

@@ -0,0 +1,4 @@
+
+
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'