app.py

import html
import json
from pathlib import Path

import gradio as gr
import matplotlib.pyplot as plt
import pandas as pd
from apscheduler.schedulers.background import BackgroundScheduler
from gradio_leaderboard import ColumnFilter, Leaderboard, SelectColumns
from huggingface_hub import snapshot_download

from src.about import (
    INTRODUCTION_TEXT,
    LLM_BENCHMARKS_TEXT,
    TITLE,
)
from src.display.css_html_js import custom_css, custom_js
from src.display.utils import (
    BENCHMARK_COLS,
    COLS,
    EVAL_COLS,
    AutoEvalColumn,
    fields,
)
from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
from src.populate import get_evaluation_queue_df, get_leaderboard_df

plt.switch_backend("Agg")


def restart_space():
    API.restart_space(repo_id=REPO_ID)


ENABLE_REMOTE_DATA_SYNC = False

if ENABLE_REMOTE_DATA_SYNC:
    try:
        print(EVAL_REQUESTS_PATH)
        snapshot_download(
            repo_id=QUEUE_REPO,
            local_dir=EVAL_REQUESTS_PATH,
            repo_type="dataset",
            tqdm_class=None,
            etag_timeout=30,
            token=TOKEN,
        )
    except Exception:
        restart_space()
    try:
        print(EVAL_RESULTS_PATH)
        snapshot_download(
            repo_id=RESULTS_REPO,
            local_dir=EVAL_RESULTS_PATH,
            repo_type="dataset",
            tqdm_class=None,
            etag_timeout=30,
            token=TOKEN,
        )
    except Exception:
        restart_space()

    LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)

    (
        finished_eval_queue_df,
        running_eval_queue_df,
        pending_eval_queue_df,
    ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)


def init_leaderboard(dataframe):
    if dataframe is None or dataframe.empty:
        raise ValueError("Leaderboard DataFrame is empty or None.")
    return Leaderboard(
        value=dataframe,
        datatype=[c.type for c in fields(AutoEvalColumn)],
        select_columns=SelectColumns(
            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
            label="Select Columns to Display:",
        ),
        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
        filter_columns=[
            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
            ColumnFilter(
                AutoEvalColumn.params.name,
                type="slider",
                min=0.01,
                max=150,
                label="Select the number of parameters (B)",
            ),
            ColumnFilter(AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True),
        ],
        bool_checkboxgroup_label="Hide models",
        interactive=False,
    )
def create_score_df(json_path: Path, decimals: int):
    data = json.loads(json_path.read_text(encoding="utf-8"))

    if not data:
        return pd.DataFrame()

    records = []
    for item in data:
        flat_record = {"ID": item.get("ID"), "Model": item.get("Model")}
        for category, sub_items in item.items():
            if category in ("ID", "Model"):
                continue
            if isinstance(sub_items, dict):
                cleaned_category = str(category).replace("\n", " ")
                for sub_category, value in sub_items.items():
                    header = f"{cleaned_category}\n{sub_category}"
                    flat_record[header] = value
        records.append(flat_record)

    df = pd.DataFrame(records)
    score_cols = [c for c in df.columns if c not in ("ID", "Model")]
    for col in score_cols:
        df[col] = df[col].apply(
            lambda v: ("" if pd.isna(v) else (f"{float(v):.{decimals}f}" if isinstance(v, (int, float)) else v))
        )
    return df


def dataframe_height(df: pd.DataFrame):
    rows = 0 if df is None else int(getattr(df, "shape", (0, 0))[0])
    row_px = 40
    header_px = 44
    padding_px = 96
    height = header_px + (rows * row_px) + padding_px
    return max(320, min(1200, height))


def create_grouped_score_table_html(df: pd.DataFrame):
    if df is None or df.empty:
        return '<div class="score-table-scroll"><table class="score-table"></table></div>'

    fixed_cols = [c for c in ("ID", "Model") if c in df.columns]

    group_to_cols = {}
    group_order = []
    data_cols_in_order = []

    for col in df.columns:
        if col in fixed_cols:
            continue
        if isinstance(col, str) and "\n" in col:
            group, metric = col.split("\n", 1)
        else:
            group, metric = str(col), str(col)

        if group not in group_to_cols:
            group_to_cols[group] = []
            group_order.append(group)

        group_to_cols[group].append((metric, col))
        data_cols_in_order.append(col)

    parts = ['<div class="score-table-scroll"><table class="score-table">']

    parts.append("<colgroup>")
    if len(fixed_cols) >= 1:
        parts.append('<col style="width:40px">')
    if len(fixed_cols) >= 2:
        parts.append('<col style="width:220px">')
    for _ in range(len(data_cols_in_order)):
        parts.append('<col style="width:180px">')
    parts.append("</colgroup>")

    parts.append("<thead>")

    parts.append("<tr>")
    for col in fixed_cols:
        parts.append(f'<th rowspan="2"><div class="score-cell">{html.escape(str(col))}</div></th>')
    merged_groups = set()
    for group in group_order:
        metrics = group_to_cols[group]
        if len(metrics) == 1 and str(metrics[0][0]) == str(group):
            merged_groups.add(group)
            parts.append(f'<th rowspan="2"><div class="score-cell">{html.escape(str(group))}</div></th>')
        else:
            parts.append(
                f'<th colspan="{len(metrics)}"><div class="score-cell">{html.escape(str(group))}</div></th>'
            )
    parts.append("</tr>")

    parts.append("<tr>")
    for group in group_order:
        if group in merged_groups:
            continue
        for metric, _ in group_to_cols[group]:
            parts.append(f"<th><div class=\"score-cell\">{html.escape(str(metric))}</div></th>")
    parts.append("</tr>")

    parts.append("</thead>")
    parts.append("<tbody>")

    col_indices = [df.columns.get_loc(c) for c in (fixed_cols + data_cols_in_order)]
    for row in df.itertuples(index=False, name=None):
        parts.append("<tr>")
        for idx in col_indices:
            value = row[idx]
            text = "" if pd.isna(value) else str(value)
            parts.append(f"<td><div class=\"score-cell\">{html.escape(text)}</div></td>")
        parts.append("</tr>")

    parts.append("</tbody></table></div>")
    return "".join(parts)


def create_raw_score_df():
    raw_path = Path(__file__).resolve().parent / "src" / "raw_score.json"
    return create_score_df(raw_path, decimals=2)


def create_unweighted_z_score_df():
    z_path = Path(__file__).resolve().parent / "src" / "unweighted_z_score.json"
    return create_score_df(z_path, decimals=4)


def create_weighted_z_score_df():
    z_path = Path(__file__).resolve().parent / "src" / "weighted_z_score.json"
    data = json.loads(z_path.read_text(encoding="utf-8"))

    if not data:
        return pd.DataFrame()

    records = []
    for item in data:
        flat_record = {"ID": item.get("ID"), "Model": item.get("Model")}
        for category, sub_items in item.items():
            if category in ("ID", "Model"):
                continue
            if not isinstance(sub_items, dict):
                continue

            cleaned_category = str(category).replace("\n", " ")
            for sub_category, value in sub_items.items():
                if cleaned_category == "Overall Score":
                    header = "Overall Score"
                else:
                    header = f"{cleaned_category}\n{sub_category}"
                flat_record[header] = value
        records.append(flat_record)

    df = pd.DataFrame(records)

    overall_col = "Overall Score" if "Overall Score" in df.columns else None
    if overall_col is None:
        for c in df.columns:
            if isinstance(c, str) and c.endswith("\nOverall Score"):
                overall_col = c
                break

    if overall_col is not None:
        df[overall_col] = pd.to_numeric(df[overall_col], errors="coerce")
        df = df.sort_values(by=overall_col, ascending=False, kind="mergesort")

        cols = list(df.columns)
        fixed = [c for c in ("ID", "Model") if c in cols]
        rest = [c for c in cols if c not in set(fixed + [overall_col])]
        df = df[fixed + [overall_col] + rest]

    score_cols = [c for c in df.columns if c not in ("ID", "Model")]
    for col in score_cols:
        df[col] = pd.to_numeric(df[col], errors="coerce").apply(
            lambda v: "" if pd.isna(v) else f"{float(v) * 100:.2f}%"
        )

    if "ID" in df.columns:
        df["ID"] = pd.to_numeric(df["ID"], errors="coerce").astype("Int64")

    return df


def create_weights_table_html():
    weights_path = Path(__file__).resolve().parent / "src" / "weights.json"
    payload = json.loads(weights_path.read_text(encoding="utf-8"))
    bounds = payload["bounds"]
    min_row = bounds["min_row"]
    min_col = bounds["min_col"]
    rows = payload["rows"]

    covered = set()
    spans = {}
    for m in payload["merges"]:
        r1, c1, r2, c2 = m["r1"], m["c1"], m["r2"], m["c2"]
        spans[(r1, c1)] = {"rowspan": r2 - r1 + 1, "colspan": c2 - c1 + 1}
        for r in range(r1, r2 + 1):
            for c in range(c1, c2 + 1):
                if (r, c) != (r1, c1):
                    covered.add((r, c))

    parts = ['<div class="weights-scroll"><table class="weights-table">']
    for r_index, row in enumerate(rows, start=min_row):
        parts.append("<tr>")
        for c_index, value in enumerate(row, start=min_col):
            if (r_index, c_index) in covered:
                continue
            span = spans.get((r_index, c_index))
            attrs = ""
            if span is not None:
                attrs = f" rowspan=\"{span['rowspan']}\" colspan=\"{span['colspan']}\""
            tag = "th" if r_index == min_row else "td"
            if r_index != min_row:
                if isinstance(value, (int, float)):
                    value = f"{float(value) * 100:.2f}%"
                elif isinstance(value, str):
                    stripped = value.strip()
                    try:
                        value = f"{float(stripped) * 100:.2f}%"
                    except Exception:
                        pass
            text = "" if value is None else html.escape(str(value))
            parts.append(f"<{tag}{attrs}>{text}</{tag}>")
        parts.append("</tr>")
    parts.append("</table></div>")
    return "".join(parts)


def parse_number(value):
    if value is None or (isinstance(value, float) and pd.isna(value)):
        return None
    if isinstance(value, (int, float)):
        return float(value)
    text = str(value).strip()
    if text == "":
        return None
    if text.endswith("%"):
        try:
            return float(text[:-1])
        except Exception:
            return None
    try:
        return float(text)
    except Exception:
        return None


def shorten_label(text: str, max_len: int = 18):
    text = "" if text is None else str(text)
    if len(text) <= max_len:
        return text
    return text[: max_len - 1] + "…"


def build_metric_choices(raw_df: pd.DataFrame):
    choices = [("Overall Score", "weighted::Overall Score")]
    for col in raw_df.columns:
        if col in ("ID", "Model"):
            continue
        if isinstance(col, str) and "\n" in col:
            group, metric = col.split("\n", 1)
            label = f"{group} / {metric}"
        else:
            label = str(col)
        choices.append((label, f"raw::{col}"))
    return choices


def build_rank_bar_plot(metric_key: str):
    if not isinstance(metric_key, str) or "::" not in metric_key:
        metric_key = "weighted::Overall Score"
    source, col = metric_key.split("::", 1)

    if source == "weighted":
        df = WEIGHTED_Z_DF
    else:
        df = RAW_SCORE_DF

    if df is None or df.empty or col not in df.columns:
        fig, ax = plt.subplots(figsize=(10, 4))
        ax.set_axis_off()
        return fig

    series = []
    for _, row in df.iterrows():
        model = row.get("Model", "")
        value = parse_number(row.get(col))
        if value is None:
            continue
        series.append((str(model), float(value)))

    series.sort(key=lambda x: x[1], reverse=True)

    labels = [shorten_label(m) for m, _ in series]
    values = [v for _, v in series]
    n = len(values)
    width = min(22, max(10, 0.55 * max(1, n)))
    fig, ax = plt.subplots(figsize=(width, 5))
    ax.bar(range(n), values)
    ax.set_xticks(range(n))
    ax.set_xticklabels(labels, rotation=35, ha="right")

    title = "Overall Score" if (source == "weighted" and col == "Overall Score") else str(col)
    ax.set_title(title)
    ax.margins(x=0.01)
    fig.tight_layout()
    return fig


RAW_SCORE_DF = create_raw_score_df()
UNWEIGHTED_Z_DF = create_unweighted_z_score_df()
WEIGHTED_Z_DF = create_weighted_z_score_df()

METRIC_CHOICES = build_metric_choices(RAW_SCORE_DF)
DEFAULT_METRIC = "weighted::Overall Score"

SCORE_TABLE_HEIGHT_CSS = f"""
#raw-score-table .score-table-scroll {{
    max-height: {dataframe_height(RAW_SCORE_DF)}px;
}}

#unweighted-z-table .score-table-scroll {{
    max-height: {dataframe_height(UNWEIGHTED_Z_DF)}px;
}}

#weighted-z-table .score-table-scroll {{
    max-height: {dataframe_height(WEIGHTED_Z_DF)}px;
}}
"""

demo = gr.Blocks(css=custom_css + SCORE_TABLE_HEIGHT_CSS)


with demo:
    gr.HTML(custom_js)
    gr.HTML(TITLE)
    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")

    with gr.Tabs(elem_classes="tab-buttons") as tabs:
        with gr.TabItem("Result", elem_id="result-tab", id=0):
            with gr.Tabs(elem_classes="tab-buttons") as nested_tabs:
                with gr.TabItem("Chart"):
                    metric = gr.Dropdown(choices=METRIC_CHOICES, value=DEFAULT_METRIC, label="Metric")
                    chart = gr.Plot(value=build_rank_bar_plot(DEFAULT_METRIC))
                    metric.change(build_rank_bar_plot, inputs=metric, outputs=chart)
                with gr.TabItem("Raw Score"):
                    gr.HTML(create_grouped_score_table_html(RAW_SCORE_DF), elem_id="raw-score-table")
                with gr.TabItem("Weights"):
                    gr.HTML(create_weights_table_html(), elem_id="weights-table")
                with gr.TabItem("Unweighted Z-score"):
                    gr.HTML(create_grouped_score_table_html(UNWEIGHTED_Z_DF), elem_id="unweighted-z-table")
                with gr.TabItem("Weighted Z-score"):
                    gr.HTML(create_grouped_score_table_html(WEIGHTED_Z_DF), elem_id="weighted-z-table")

        with gr.TabItem("About", elem_id="llm-benchmark-tab-table", id=2):
            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")


scheduler = BackgroundScheduler()
scheduler.add_job(restart_space, "interval", seconds=1800)
scheduler.start()
demo.queue(default_concurrency_limit=40).launch()