app.py

import json
import gradio as gr
from huggingface_hub import CommitScheduler
from pathlib import Path
import requests
from huggingface_hub import HfApi, HfFolder
import os

# Use the token from an environment variable
token = os.getenv('stb_leaderboard_json')
HfFolder.save_token(token)

def load_data():
    url = "https://huggingface.co/datasets/stabletoolbench/StableToolBench_data/resolve/main/leaderboard_data.json"
    response = requests.get(url)
    data = response.json()
    return data

# load data from the dataset
existing_data = load_data()

# Create local folders for temporary storage of modified data files
dataset_dir = Path("my_dataset")
dataset_dir.mkdir(parents=True, exist_ok=True)

# initialize CommitScheduler
scheduler = CommitScheduler(
    repo_id="stabletoolbench/StableToolBench_data",  # 替换为你的用户名和数据集仓库名
    repo_type="dataset",
    folder_path=dataset_dir,
    path_in_repo="",
)

# Generate table data based on Average Score ranking by default
def generate_table(data):
    if not data:
        return [], []  # No data, return empty headers and rows

    # Make sure all entries have a 'Scores' dictionary
    valid_data = [entry for entry in data if 'Scores' in entry and isinstance(entry['Scores'], dict)]

    # Sort the data based on the 'Average' score in descending order
    sorted_data = sorted(valid_data, key=lambda x: x['Scores'].get('Average', 0), reverse=True)

    # Now, build the table rows with the sorted data
    headers = ["Method"] + list(sorted_data[0]['Scores'].keys()) if sorted_data else ["Method"]

    rows = []
    for entry in sorted_data:
        row = [entry['Method']] + [entry['Scores'].get(key, "N/A") for key in headers[1:]]
        rows.append(row)

    return headers, rows

# Raw data for the paper
protected_methods = ["GPT-4-Turbo-Preview (DFS)", "GPT-3.5-Turbo-1106 (DFS)", "GPT-4-0613 (DFS)", "GPT-3.5-Turbo-0613 (DFS)", "GPT-4-Turbo-Preview (CoT)", "ToolLLaMA v2 (DFS)", "GPT-4-0613 (CoT)", "GPT-3.5-Turbo-1106 (CoT)", "GPT-3.5-Turbo-0613 (CoT)", "ToolLLaMA v2 (CoT)"]

# Merge uploaded data into existing data
def merge_data(uploaded_data_json):
    # No need to call json.loads here because uploaded_data is already a Python dict
    new_data = uploaded_data_json

    with scheduler.lock:

        # Define a helper function to merge scores for an entry
        def merge_scores(existing_scores, new_scores):
            for key, value in new_scores.items():
                existing_scores[key] = value

        # Merge 'SolvablePassRateScores'
        for new_entry in new_data["SolvablePassRateScores"]:
            if new_entry["Method"] not in protected_methods:
                existing_entry = next(
                    (item for item in existing_data["SolvablePassRateScores"] if item["Method"] == new_entry["Method"]),
                    None)
                if existing_entry:
                    # It's a non-protected method, update it
                    merge_scores(existing_entry["Scores"], new_entry["Scores"])
                else:
                    # It's a new method to add to the list
                    existing_data["SolvablePassRateScores"].append(new_entry)

        # Merge 'SolvableWinRateScores'
        for new_entry in new_data["SolvableWinRateScores"]:
            if new_entry["Method"] not in protected_methods:
                existing_entry = next(
                    (item for item in existing_data["SolvableWinRateScores"] if item["Method"] == new_entry["Method"]),
                    None)
                if existing_entry:
                    merge_scores(existing_entry["Scores"], new_entry["Scores"])
                else:
                    existing_data["SolvableWinRateScores"].append(new_entry)

        data_file_path = dataset_dir / "leaderboard_data.json"
        with open(data_file_path, 'w') as file:
            json.dump(existing_data, file, indent=4)

    return existing_data

def process_file(file_info):
    if file_info is not None:
        with open(file_info, "r") as uploaded_file:
            data_content = uploaded_file.read()
            uploaded_data_json = json.loads(data_content)
        # Merge the uploaded data
        merge_data(uploaded_data_json)

    pass_rate_table, win_rate_table = refresh_table_data()
    return pass_rate_table, win_rate_table

def refresh_table_data():
    # 重新加载数据
    new_data = load_data()
    # 重新生成表格数据
    new_pass_rate_data = generate_table(new_data["SolvablePassRateScores"])[1]
    new_win_rate_data = generate_table(new_data["SolvableWinRateScores"])[1]
    # 返回新的表格数据
    return new_pass_rate_data, new_win_rate_data

with gr.Blocks() as app:

    # The large title
    gr.Markdown("# StableToolBench Leaderboard")

    # The introductory content
    gr.Markdown("""
            **Large Language Models (LLMs)** have witnessed remarkable advancements in recent years, prompting the exploration of tool learning, which integrates LLMs with external tools to address diverse real-world challenges. Assessing the capability of LLMs to utilise tools necessitates large-scale and stable benchmarks. However, previous works relied on either hand-crafted online tools with limited scale, or large-scale real online APIs suffering from instability of API status. To address this problem, we introduce StableToolBench, a benchmark evolving from ToolBench, proposing a virtual API server and stable evaluation system. The virtual API server contains a caching system and API simulators which are complementary to alleviate the change in API status. Meanwhile, the stable evaluation system designs solvable pass and win rates using GPT-4 as the automatic evaluator to eliminate the randomness during evaluation. Experimental results demonstrate the stability of StableToolBench, and further discuss the effectiveness of API simulators, the caching system, and the evaluation system.
            """)
    gr.Markdown(""" ### For further information, please refer to: """)
    buttons_html = """
        <style>
            .custom-link-button {
                font-size: 18px !important; /* Adjust the font size as needed */
                padding: 10px 15px !important; /* Add some padding */
                margin: 5px !important;
                color: white !important; /* Text color */
                background-color: #106BA3 !important; /* Background color */
                text-decoration: none !important; /* Remove underline from links */
                display: inline-block !important;
                border-radius: 5px !important; /* Rounded corners */
                border: none !important; /* Remove borders */
                cursor: pointer !important; /* Mouse pointer on hover */
                text-align: center !important;
            }
            .custom-link-button:hover {
                background-color: #0D5B8F !important;
            }
        </style>
        <a href="https://arxiv.org/pdf/2403.07714.pdf" target="_blank" class="custom-link-button">Paper</a>
        <a href="https://arxiv.org/abs/2403.07714" target="_blank" class="custom-link-button">arXiv</a>
        <a href="https://github.com/zhichengg/StableToolBench" target="_blank" class="custom-link-button">Code</a>
        <a href="https://drive.google.com/file/d/1XUiCMA5NV359UGR-eknF0TcXORuR7RXj/view?pli=1" target="_blank" class="custom-link-button">Cache Data</a>
        """

    gr.HTML(buttons_html)

    gr.Markdown("## Solvable Pass Rate Scores")
    headers1, rows1 = generate_table(existing_data["SolvablePassRateScores"])
    table1 = gr.Dataframe(headers=headers1, value=rows1, interactive=False)

    gr.Markdown("## Solvable Win Rate Scores")
    headers2, rows2 = generate_table(existing_data["SolvableWinRateScores"])
    table2 = gr.Dataframe(headers=headers2, value=rows2, interactive=False)

    refresh_button = gr.Button("Refresh Leaderboards")
    refresh_button.click(
        fn=refresh_table_data,
        outputs=[table1, table2]
    )

    gr.Markdown("## Upload Your Own Results")
    gr.Markdown("""
            If you would like to contribute to the leaderboard, please follow the JSON structure below for your method's scores.

            **Solvable Pass Rate Scores Template:**
            ```json
            {
                "SolvablePassRateScores": [
                    {
                        "Method": "Your Method Name",
                        "Scores": {
                        "I1 Instruction": 85.5,
                        "I1 Instruction SE": 1.2,
                        "I1 Category": 80.0,
                        "I1 Category SE": 1.0,
                        "I1 Tool": 88.5,
                        "I1 Tool SE": 0.8,
                        "I2 Category": 82.5,
                        "I2 Category SE": 1.3,
                        "I2 Instruction": 86.0,
                        "I2 Instruction SE": 0.5,
                        "I3 Instruction": 90.0,
                        "I3 Instruction SE": 0.7,
                        "Average": 87.5,
                        "Average SE": 1.1
                        }
                    }
                    // Add more methods here...
                ],
                "SolvableWinRateScores": [
                    {
                        "Method": "Your Method Name",
                        "Scores": {
                            "I1 Instruction": 65.0,
                            "I1 Category": 68.5,
                            "I1 Tool": 66.8,
                            "I2 Category": 70.0,
                            "I2 Instruction": 69.2,
                            "I3 Instruction": 71.5,
                            "Average": 68.5
                        }
                    }
                    // Add more methods here...
                ]
            }
            ```

            Make sure your uploaded JSON file follows this structure.
        """)

    upload_component = gr.File(label="Upload JSON File")
    submit_button = gr.Button("Submit")
    submit_button.click(
        fn=process_file,
        inputs= upload_component,
        outputs=[table1, table2]
    )

    gr.Markdown(" ## If you like our project, please consider cite our work as follows: ")
    citation_text = """
        ```
        @misc{guo2024stabletoolbench,
          title={StableToolBench: Towards Stable Large-Scale Benchmarking on Tool Learning of Large Language Models}, 
          author={Zhicheng Guo and Sijie Cheng and Hao Wang and Shihao Liang and Yujia Qin and Peng Li and Zhiyuan Liu and Maosong Sun and Yang Liu},
          year={2024},
          eprint={2403.07714},
          archivePrefix={arXiv},
          primaryClass={cs.CL}
        }
        ```
        """
    gr.Markdown(citation_text)

if __name__ == "__main__":
    app.launch()
    scheduler.commit()  # Ensure all changes are committed on exit