import gradio as gr from src.utils import model_hyperlink, process_score LEADERBOARD_COLUMN_TO_DATATYPE = { # open llm "Model ๐Ÿค—": "markdown", "Experiment ๐Ÿงช": "str", # primary measurements "Prefill (s)": "number", "Decode (tokens/s)": "number", "Memory (MB)": "number", "Energy (tokens/kWh)": "number", # deployment settings "Backend ๐Ÿญ": "str", "Precision ๐Ÿ“ฅ": "str", "Quantization ๐Ÿ—œ๏ธ": "str", "Attention ๐Ÿ‘๏ธ": "str", "Kernel โš›๏ธ": "str", # additional measurements # "Reserved Memory (MB)": "number", # "Used Memory (MB)": "number", "Open LLM Score (%)": "number", "End-to-End (s)": "number", "Architecture ๐Ÿ›๏ธ": "str", "Params (B)": "number", } PRIMARY_COLUMNS = [ "Model ๐Ÿค—", "Experiment ๐Ÿงช", "Prefill (s)", "Decode (tokens/s)", "Memory (MB)", "Energy (tokens/kWh)", "Open LLM Score (%)", ] def process_model(model_name): link = f"https://huggingface.co/{model_name}" return model_hyperlink(link, model_name) def get_leaderboard_df(llm_perf_df): df = llm_perf_df.copy() # transform for leaderboard df["Model ๐Ÿค—"] = df["Model ๐Ÿค—"].apply(process_model) # process quantization for leaderboard df["Open LLM Score (%)"] = df.apply( lambda x: process_score(x["Open LLM Score (%)"], x["Quantization ๐Ÿ—œ๏ธ"]), axis=1 ) return df def create_leaderboard_table(llm_perf_df): # get dataframe leaderboard_df = get_leaderboard_df(llm_perf_df) # create search bar with gr.Row(): search_bar = gr.Textbox( label="Model ๐Ÿค—", info="๐Ÿ” Search for a model name", elem_id="search-bar", ) # create checkboxes with gr.Row(): columns_checkboxes = gr.CheckboxGroup( label="Columns ๐Ÿ“Š", value=PRIMARY_COLUMNS, choices=list(LEADERBOARD_COLUMN_TO_DATATYPE.keys()), info="โ˜‘๏ธ Select the columns to display", elem_id="columns-checkboxes", ) # create table leaderboard_table = gr.components.Dataframe( value=leaderboard_df[PRIMARY_COLUMNS], datatype=list(LEADERBOARD_COLUMN_TO_DATATYPE.values()), headers=list(LEADERBOARD_COLUMN_TO_DATATYPE.keys()), elem_id="leaderboard-table", ) return search_bar, columns_checkboxes, leaderboard_table