|
import io |
|
import json |
|
|
|
import gradio as gr |
|
import pandas as pd |
|
from huggingface_hub import HfFileSystem |
|
|
|
|
|
RESULTS_DATASET_ID = "datasets/open-llm-leaderboard/results" |
|
EXCLUDED_KEYS = { |
|
"pretty_env_info", |
|
"chat_template", |
|
"group_subtasks", |
|
} |
|
EXCLUDED_RESULTS_KEYS = { |
|
"leaderboard", |
|
} |
|
EXCLUDED_RESULTS_LEADERBOARDS_KEYS = { |
|
"alias", |
|
} |
|
|
|
DEFAULT_HTML_TABLE = """ |
|
<table> |
|
<thead> |
|
<tr> |
|
<th>Parameters</th> |
|
<th>Model-1</th> |
|
<th>Model-2</th> |
|
</tr> |
|
</thead> |
|
<tbody> |
|
</tbody> |
|
</table> |
|
""" |
|
|
|
TASKS = { |
|
"leaderboard_arc_challenge": ("ARC", "leaderboard_arc_challenge"), |
|
"leaderboard_bbh": ("BBH", "leaderboard_bbh"), |
|
"leaderboard_gpqa": ("GPQA", "leaderboard_gpqa"), |
|
"leaderboard_ifeval": ("IFEval", "leaderboard_ifeval"), |
|
"leaderboard_math_hard": ("MATH", "leaderboard_math"), |
|
"leaderboard_mmlu": ("MMLU", "leaderboard_mmlu"), |
|
"leaderboard_mmlu_pro": ("MMLU-Pro", "leaderboard_mmlu_pro"), |
|
"leaderboard_musr": ("MuSR", "leaderboard_musr"), |
|
} |
|
|
|
fs = HfFileSystem() |
|
|
|
|
|
def fetch_result_paths(): |
|
paths = fs.glob(f"{RESULTS_DATASET_ID}/**/**/*.json") |
|
return paths |
|
|
|
|
|
def filter_latest_result_path_per_model(paths): |
|
from collections import defaultdict |
|
|
|
d = defaultdict(list) |
|
for path in paths: |
|
model_id, _ = path[len(RESULTS_DATASET_ID) +1:].rsplit("/", 1) |
|
d[model_id].append(path) |
|
return {model_id: max(paths) for model_id, paths in d.items()} |
|
|
|
|
|
def get_result_path_from_model(model_id, result_path_per_model): |
|
return result_path_per_model[model_id] |
|
|
|
|
|
def load_data(result_path) -> pd.DataFrame: |
|
with fs.open(result_path, "r") as f: |
|
data = json.load(f) |
|
return data |
|
|
|
|
|
def load_result(model_id): |
|
result_path = get_result_path_from_model(model_id, latest_result_path_per_model) |
|
data = load_data(result_path) |
|
df = to_dataframe(data) |
|
result = [ |
|
|
|
to_vertical(filter_results(df)), |
|
to_vertical(filter_configs(df)), |
|
] |
|
return result |
|
|
|
|
|
def to_vertical(df): |
|
df = df.T.rename_axis("Parameters") |
|
df.index = df.index.str.join(".") |
|
return df |
|
|
|
|
|
def to_dataframe(data): |
|
df = pd.json_normalize([{key: value for key, value in data.items() if key not in EXCLUDED_KEYS}]) |
|
|
|
df.columns = list(map(lambda x: tuple(x.split(".")), df.columns)) |
|
df.index = [data.get("model_name", "Model")] |
|
return df |
|
|
|
|
|
def filter_results(df): |
|
df = df.loc[:, df.columns.str[0] == "results"] |
|
df = df.loc[:, ~df.columns.str[1].isin(EXCLUDED_RESULTS_KEYS)] |
|
|
|
df = df.loc[:, ~df.columns.str[2].isin(EXCLUDED_RESULTS_LEADERBOARDS_KEYS)] |
|
df.columns = df.columns.str[1:] |
|
df.columns = map(lambda x: (x[0].removeprefix("leaderboard_"), *x[1:]), df.columns) |
|
return df |
|
|
|
|
|
def filter_configs(df): |
|
df = df.loc[:, df.columns.str[0] == "configs"] |
|
|
|
|
|
df.columns = df.columns.str[1:] |
|
df.columns = map(lambda x: (x[0].removeprefix("leaderboard_"), *x[1:]), df.columns) |
|
return df |
|
|
|
|
|
def concat_result_1(result_1, results): |
|
results = pd.read_html(io.StringIO(results))[0] |
|
df = ( |
|
pd.concat([result_1, results.iloc[:, [0, 2]].set_index("Parameters")], axis=1) |
|
.reset_index() |
|
) |
|
return df |
|
|
|
|
|
def display_dataframe(df): |
|
|
|
return ( |
|
df.style |
|
.format(na_rep="") |
|
.hide(axis="index") |
|
.to_html() |
|
) |
|
|
|
|
|
def concat_result_2(result_2, results): |
|
results = pd.read_html(io.StringIO(results))[0] |
|
df = ( |
|
pd.concat([results.iloc[:, [0, 1]].set_index("Parameters"), result_2], axis=1) |
|
.reset_index() |
|
) |
|
return df |
|
|
|
|
|
def render_result_1(model_id, task, *results): |
|
result = load_result(model_id) |
|
concat_results = [concat_result_1(*result_args) for result_args in zip(result, results)] |
|
if task: |
|
concat_results = [df[df["Parameters"].str.startswith(task[len("leaderboard_"):])] for df in concat_results] |
|
return [display_dataframe(df) for df in concat_results] |
|
|
|
|
|
def render_result_2(model_id, task, *results): |
|
result = load_result(model_id) |
|
concat_results = [concat_result_2(*result_args) for result_args in zip(result, results)] |
|
if task: |
|
concat_results = [df[df["Parameters"].str.startswith(task[len("leaderboard_"):])] for df in concat_results] |
|
return [display_dataframe(df) for df in concat_results] |
|
|
|
|
|
def render_results(model_id_1, model_id_2, task, *results): |
|
results = render_result_1(model_id_1, task, *results) |
|
return render_result_2(model_id_2, task, *results) |
|
|
|
|
|
|
|
latest_result_path_per_model = filter_latest_result_path_per_model(fetch_result_paths()) |
|
|
|
with gr.Blocks(fill_height=True) as demo: |
|
gr.HTML("<h1 style='text-align: center;'>Compare Results of the 🤗 Open LLM Leaderboard</h1>") |
|
gr.HTML("<h3 style='text-align: center;'>Select 2 results to load and compare</h3>") |
|
|
|
with gr.Row(): |
|
with gr.Column(): |
|
model_id_1 = gr.Dropdown(choices=list(latest_result_path_per_model.keys()), label="Results") |
|
load_btn_1 = gr.Button("Load") |
|
with gr.Column(): |
|
model_id_2 = gr.Dropdown(choices=list(latest_result_path_per_model.keys()), label="Results") |
|
load_btn_2 = gr.Button("Load") |
|
with gr.Row(): |
|
task = gr.Radio( |
|
["All"] + list(TASKS.values()), |
|
label="Tasks", |
|
info="Evaluation tasks to be displayed", |
|
value="All", |
|
) |
|
|
|
results = [] |
|
with gr.Row(): |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
with gr.Tab("Results"): |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
results.append(gr.HTML(value=DEFAULT_HTML_TABLE)) |
|
with gr.Tab("Configs"): |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
results.append(gr.HTML(value=DEFAULT_HTML_TABLE)) |
|
|
|
load_btn_1.click( |
|
fn=render_result_1, |
|
inputs=[model_id_1, task, *results], |
|
outputs=[*results], |
|
) |
|
load_btn_2.click( |
|
fn=render_result_2, |
|
inputs=[model_id_2, task, *results], |
|
outputs=[*results], |
|
) |
|
task.change( |
|
fn=render_results, |
|
inputs=[model_id_1, model_id_2, task, *results], |
|
outputs=[*results], |
|
) |
|
|
|
demo.launch() |
|
|
