File size: 2,441 Bytes
ab5f5f1
 
 
 
 
 
 
a8a6326
 
ab5f5f1
0232cf1
 
 
ab5f5f1
a8a6326
 
 
 
 
ab5f5f1
0232cf1
a8a6326
 
0232cf1
 
ab5f5f1
 
 
 
0232cf1
 
 
 
 
 
 
 
 
 
ab5f5f1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
0232cf1
 
 
 
 
 
 
 
a8a6326
0232cf1
 
 
 
 
 
 
 
ab5f5f1
 
0232cf1
ab5f5f1
 
a8a6326
ab5f5f1
 
0232cf1
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
import gradio as gr

from src.utils import model_hyperlink, process_score


LEADERBOARD_COLUMN_TO_DATATYPE = {
    # open llm
    "Model πŸ€—": "markdown",
    "Experiment πŸ§ͺ": "str",
    # primary measurements
    "Prefill (s)": "number",
    "Decode (tokens/s)": "number",
    "Memory (MB)": "number",
    "Energy (tokens/kWh)": "number",
    # deployment settings
    "DType πŸ“₯": "str",
    "Backend 🏭": "str",
    "Optimization πŸ› οΈ": "str",
    "Quantization πŸ—œοΈ": "str",
    # additional measurements
    "Architecture πŸ›οΈ": "markdown",
    "Params (B)": "number",
    "Open LLM Score (%)": "number",
    "End-to-End (s)": "number",
    "End-to-End (tokens/s)": "number",
    "Reserved Memory (MB)": "number",
    "Used Memory (MB)": "number",
}

PRIMARY_COLUMNS = [
    "Model πŸ€—",
    "Experiment πŸ§ͺ",
    "Prefill (s)",
    "Decode (tokens/s)",
    "Memory (MB)",
    "Energy (tokens/kWh)",
    "Open LLM Score (%)",
]


def process_model(model_name):
    link = f"https://huggingface.co/{model_name}"
    return model_hyperlink(link, model_name)


def get_leaderboard_df(llm_perf_df):
    df = llm_perf_df.copy()
    # transform for leaderboard
    df["Model πŸ€—"] = df["Model πŸ€—"].apply(process_model)
    # process quantization for leaderboard
    df["Open LLM Score (%)"] = df.apply(
        lambda x: process_score(x["Open LLM Score (%)"], x["Quantization πŸ—œοΈ"]),
        axis=1,
    )
    return df


def create_leaderboard_table(llm_perf_df):
    # get dataframe
    leaderboard_df = get_leaderboard_df(llm_perf_df)

    # create search bar
    with gr.Row():
        search_bar = gr.Textbox(
            label="Model πŸ€—",
            info="πŸ” Search for a model name",
            elem_id="search-bar",
        )
    # create checkboxes
    with gr.Row():
        columns_checkboxes = gr.CheckboxGroup(
            label="Columns πŸ“Š",
            value=PRIMARY_COLUMNS,
            choices=list(LEADERBOARD_COLUMN_TO_DATATYPE.keys()),
            info="β˜‘οΈ Select the columns to display",
            elem_id="columns-checkboxes",
        )
    # create table
    leaderboard_table = gr.components.Dataframe(
        value=leaderboard_df[PRIMARY_COLUMNS],
        datatype=list(LEADERBOARD_COLUMN_TO_DATATYPE.values()),
        headers=list(LEADERBOARD_COLUMN_TO_DATATYPE.keys()),
        elem_id="leaderboard-table",
    )

    return search_bar, columns_checkboxes, leaderboard_table