import logging import os import subprocess import gradio as gr from apscheduler.schedulers.background import BackgroundScheduler from gradio_leaderboard import Leaderboard, SelectColumns from gradio_space_ci import enable_space_ci from src.display.about import ( INTRODUCTION_TEXT, TITLE, ) from src.display.css_html_js import custom_css from src.display.utils import ( AutoEvalColumn, fields, ) from src.envs import API, H4_TOKEN, HF_HOME, REPO_ID, RESET_JUDGEMENT_ENV from src.leaderboard.build_leaderboard import build_leadearboard_df, download_openbench os.environ["GRADIO_ANALYTICS_ENABLED"] = "false" # Configure logging logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") # Start ephemeral Spaces on PRs (see config in README.md) enable_space_ci() download_openbench() def restart_space(): API.restart_space(repo_id=REPO_ID, token=H4_TOKEN) def build_demo(): demo = gr.Blocks(title="Chatbot Arena Leaderboard", css=custom_css) leaderboard_df = build_leadearboard_df() with demo: gr.HTML(TITLE) gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") with gr.Tabs(elem_classes="tab-buttons"): with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0): Leaderboard( value=leaderboard_df, datatype=[c.type for c in fields(AutoEvalColumn)], select_columns=SelectColumns( default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default], cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden or c.dummy], label="Select Columns to Display:", ), search_columns=[ AutoEvalColumn.model.name, # AutoEvalColumn.fullname.name, # AutoEvalColumn.license.name ], ) # with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=1): # gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text") # with gr.TabItem("❗FAQ", elem_id="llm-benchmark-tab-table", id=2): # gr.Markdown(FAQ_TEXT, elem_classes="markdown-text") with gr.TabItem("🚀 Submit ", elem_id="llm-benchmark-tab-table", id=3): with gr.Row(): gr.Markdown("# ✨ Submit your model here!", elem_classes="markdown-text") with gr.Column(): model_name_textbox = gr.Textbox(label="Model name") submitter_username = gr.Textbox(label="Username") def upload_file(file): file_path = file.name.split("/")[-1] if "/" in file.name else file.name logging.info("New submition: file saved to %s", file_path) API.upload_file( path_or_fileobj=file.name, path_in_repo="model_answers/external/" + file_path, repo_id="Vikhrmodels/openbench-eval", repo_type="dataset", ) os.environ[RESET_JUDGEMENT_ENV] = "1" return file.name if model_name_textbox and submitter_username: file_output = gr.File() upload_button = gr.UploadButton( "Click to Upload & Submit Answers", file_types=["*"], file_count="single" ) upload_button.upload(upload_file, upload_button, file_output) return demo # print(os.system('cd src/gen && ../../.venv/bin/python gen_judgment.py')) # print(os.system('cd src/gen/ && python show_result.py --output')) def update_board(): need_reset = os.environ.get(RESET_JUDGEMENT_ENV) logging.info("Updating the judgement: %s", need_reset) if need_reset != "1": return os.environ[RESET_JUDGEMENT_ENV] = "0" # gen_judgement_file = os.path.join(HF_HOME, "src/gen/gen_judgement.py") # subprocess.run(["python3", gen_judgement_file], check=True) show_result_file = os.path.join(HF_HOME, "src/gen/show_result.py") subprocess.run(["python3", show_result_file, "--output"], check=True) # update the gr item with leaderboard # TODO if __name__ == "__main__": os.environ[RESET_JUDGEMENT_ENV] = "1" scheduler = BackgroundScheduler() scheduler.add_job(update_board, "interval", minutes=10) scheduler.start() demo_app = build_demo() demo_app.launch(debug=True)