import os import gradio as gr import pandas as pd from huggingface_hub import HfApi from huggingface_hub import Repository from apscheduler.schedulers.background import BackgroundScheduler from src.assets.text_content import * from src.assets.css_html_js import custom_css, get_window_url_params OPTIMUM_TOKEN = os.environ.get("OPTIMUM_TOKEN", None) LLM_PERF_LEADERBOARD_REPO = "optimum/llm-perf-leaderboard" LLM_PERF_DATASET_REPO = "optimum/llm-perf" api = HfApi() def restart_space(): api.restart_space( repo_id=LLM_PERF_LEADERBOARD_REPO, token=OPTIMUM_TOKEN ) def load_all_info_from_hub(): llm_perf_repo = None if OPTIMUM_TOKEN: llm_perf_repo = Repository( local_dir="./llm-perf/", clone_from=LLM_PERF_DATASET_REPO, token=OPTIMUM_TOKEN, repo_type="dataset", ) llm_perf_repo.git_pull() return llm_perf_repo llm_perf_repo = load_all_info_from_hub() def has_no_nan_values(df, columns): return df[columns].notna().all(axis=1) def has_nan_values(df, columns): return df[columns].isna().any(axis=1) def get_leaderboard_df(): if llm_perf_repo: llm_perf_repo.git_pull() df = pd.read_csv("./llm-perf/reports/cuda_1_100/inference_report.csv") return df original_df = get_leaderboard_df() leaderboard_df = original_df.copy() def refresh(): leaderboard_df = get_leaderboard_df() return leaderboard_df demo = gr.Blocks(css=custom_css) with demo: gr.HTML(TITLE) gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") with gr.Tabs(elem_classes="tab-buttons") as tabs: with gr.TabItem("🏅 LLM-Perf Benchmark", elem_id="llm-perf-benchmark-tab-table", id=0): leaderboard_table_lite = gr.components.Dataframe( value=leaderboard_df, headers=leaderboard_df.columns.tolist(), # datatype=TYPES_LITE, max_rows=None, elem_id="leaderboard-table-lite", ) with gr.Row(): with gr.Column(): with gr.Accordion("📙 Citation", open=False): citation_button = gr.Textbox( value=CITATION_BUTTON_TEXT, label=CITATION_BUTTON_LABEL, elem_id="citation-button", ).style(show_copy_button=True) with gr.Column(): with gr.Accordion("✨ CHANGELOG", open=False): changelog = gr.Markdown( CHANGELOG_TEXT, elem_id="changelog-text") dummy = gr.Textbox(visible=False) demo.load( dummy, tabs, _js=get_window_url_params, ) scheduler = BackgroundScheduler() scheduler.add_job(restart_space, "interval", seconds=3600) scheduler.start() demo.queue(concurrency_count=40).launch()