app.py

"""
gradio app that reads results.jsonl and display it in a table, title is "AgentRewardBench Leaderboard"
"""

import json
import os
from pathlib import Path

import gradio as gr
import pandas as pd


RESULTS_PATH = Path(__file__).with_name("results.jsonl")


def load_data():
    # read the jsonl file
    data = []
    with RESULTS_PATH.open("r", encoding="utf-8") as f:
        for line in f:
            if line.strip():
                data.append(json.loads(line))

    df = pd.DataFrame(data)
    # remove Recall and F1 columns
    df = df.drop(columns=["Recall", "F1"], errors="ignore")

    # if a field called "Overall" exists, sort the dataframe by "Overall"
    if "Overall" in df.columns:
        df = df.sort_values(by="Overall", ascending=False)

    # if a field called "Project URL" exists, add a link to the project url
    if "Project URL" in df.columns:
        df["Author"] = "[" + df["Author"] + "](" + df["Project URL"] + ")"
        # remove the Project URL column
        df = df.drop(columns=["Project URL"])

    # if a field called "Logs URL" exists, add a link to the logs url
    if "Logs URL" in df.columns:
        df["Logs"] = df["Logs URL"].apply(lambda x: f"[🔗]({x})" if pd.notna(x) and x else "✖️")
        df = df.drop(columns=["Logs URL"])

    # return the dataframe
    return df


with gr.Blocks() as demo:
    gr.Markdown(
        """
        # AgentRewardBench Leaderboard

        
        | [**💾Code**](https://github.com/McGill-NLP/agent-reward-bench) |[**📄Paper**](https://arxiv.org/abs/2504.08942) | [**🌐Website**](https://agent-reward-bench.github.io) | 
        | :--: | :--: | :--: |
        | [**🤗Dataset**](https://huggingface.co/datasets/McGill-NLP/agent-reward-bench) | [**💻Demo**](https://huggingface.co/spaces/McGill-NLP/agent-reward-bench-demo) |  [**🏆Leaderboard**](https://huggingface.co/spaces/McGill-NLP/agent-reward-bench-leaderboard) | 

        
        This is the leaderboard for the AgentRewardBench. The scores are based on the results of the agents on the benchmark. We report the *precision* score.
        [Open an issue to submit your results to the leadeboard](https://github.com/McGill-NLP/agent-reward-bench/issues/new?template=add-results-to-leaderboard.yml). We will review your results and add them to the leaderboard.
        """
    )
    df = load_data()
    gr.DataFrame(df, show_label=False, datatype="markdown")


if __name__ == "__main__":
    demo.queue(default_concurrency_limit=40).launch(
        server_name="0.0.0.0",
        server_port=int(os.getenv("PORT", "7860")),
    )