import gradio as gr import pandas as pd from src.display.about import ( CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, EVALUATION_QUEUE_TEXT, INTRODUCTION_TEXT, LLM_BENCHMARKS_TEXT, FAQ_TEXT, TITLE, ) from src.display.css_html_js import custom_css from src.display.utils import ( BENCHMARK_COLS, COLS, EVAL_COLS, EVAL_TYPES, NUMERIC_INTERVALS, TYPES, AutoEvalColumn, ModelType, fields, WeightType, Precision ) from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, H4_TOKEN, IS_PUBLIC, QUEUE_REPO, REPO_ID, RESULTS_REPO from PIL import Image # from src.populate import get_evaluation_queue_df, get_leaderboard_df # from src.submission.submit import add_new_eval # from src.tools.collections import update_collections # from src.tools.plots import ( # create_metric_plot_obj, # create_plot_df, # create_scores_df, # ) from dummydatagen import dummy_data_for_plot, create_metric_plot_obj_1, dummydf import copy def restart_space(): API.restart_space(repo_id=REPO_ID, token=H4_TOKEN) def add_average_col(df): always_here_cols = [ "Model", "Agent", "Opponent Model", "Opponent Agent" ] desired_col = [i for i in list(df.columns) if i not in always_here_cols] newdf = df[desired_col].mean(axis=1).round(3) return newdf gtbench_raw_data = dummydf() gtbench_raw_data["Average"] = add_average_col(gtbench_raw_data) column_to_move = "Average" # Move the column to the desired index gtbench_raw_data.insert( 4, column_to_move, gtbench_raw_data.pop(column_to_move)) models = list(set(gtbench_raw_data['Model'])) opponent_models = list(set(gtbench_raw_data['Opponent Model'])) agents = list(set(gtbench_raw_data['Agent'])) opponent_agents = list(set(gtbench_raw_data['Opponent Agent'])) # Searching and filtering def update_table( hidden_df: pd.DataFrame, columns: list, model1: list, model2: list, agent1: list, agent2: list ): filtered_df = select_columns(hidden_df, columns) filtered_df = filter_model1(filtered_df, model1) filtered_df = filter_model2(filtered_df, model2) filtered_df = filter_agent1(filtered_df, agent1) filtered_df = filter_agent2(filtered_df, agent2) return filtered_df # triggered only once at startup => read query parameter if it exists def load_query(request: gr.Request): query = request.query_params.get("query") or "" return query, query # return one for the "search_bar", one for a hidden component that triggers a reload only if value has changed def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame: return df[(df[AutoEvalColumn.dummy.name].str.contains(query, case=False))] def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame: always_here_cols = [ "Model", "Agent", "Opponent Model", "Opponent Agent" ] # We use COLS to maintain sorting all_columns = games if len(columns) == 0: filtered_df = df[ always_here_cols + [c for c in all_columns if c in df.columns] ] filtered_df["Average"] = add_average_col(filtered_df) column_to_move = "Average" current_index = filtered_df.columns.get_loc(column_to_move) # Move the column to the desired index filtered_df.insert(4, column_to_move, filtered_df.pop(column_to_move)) return filtered_df filtered_df = df[ always_here_cols + [c for c in all_columns if c in df.columns and c in columns] ] if "Average" in columns: filtered_df["Average"] = add_average_col(filtered_df) # Get the current index of the column column_to_move = "Average" current_index = filtered_df.columns.get_loc(column_to_move) # Move the column to the desired index filtered_df.insert(4, column_to_move, filtered_df.pop(column_to_move)) else: if "Average" in filtered_df.columns: # Remove the column filtered_df = filtered_df.drop(columns=["Average"]) return filtered_df def filter_model1( df: pd.DataFrame, model_query: list ) -> pd.DataFrame: # Show all models if len(model_query) == 0: return df filtered_df = df filtered_df = filtered_df[filtered_df["Model"].isin( model_query)] return filtered_df def filter_model2( df: pd.DataFrame, model_query: list ) -> pd.DataFrame: # Show all models if len(model_query) == 0: return df filtered_df = df filtered_df = filtered_df[filtered_df["Opponent Model"].isin( model_query)] return filtered_df def filter_agent1( df: pd.DataFrame, agent_query: list ) -> pd.DataFrame: # Show all models if len(agent_query) == 0: return df filtered_df = df filtered_df = filtered_df[filtered_df["Agent"].isin( agent_query)] return filtered_df def filter_agent2( df: pd.DataFrame, agent_query: list ) -> pd.DataFrame: # Show all models if len(agent_query) == 0: return df filtered_df = df filtered_df = filtered_df[filtered_df["Opponent Agent"].isin( agent_query)] return filtered_df # leaderboard_df = filter_models(leaderboard_df, [t.to_str(" : ") for t in ModelType], list(NUMERIC_INTERVALS.keys()), [i.value.name for i in Precision], False, False) class LLM_Model: def __init__(self, t_value, model_value, average_value, arc_value, hellaSwag_value, mmlu_value) -> None: self.t = t_value self.model = model_value self.average = average_value self.arc = arc_value self.hellaSwag = hellaSwag_value self.mmlu = mmlu_value games = ["Breakthrough", "Connect Four", "Blind Auction", "Kuhn Poker", "Liar's Dice", "Negotiation", "Nim", "Pig", "Iterated Prisoner's Dilemma", "Tic-Tac-Toe"] # models = ["gpt-35-turbo-1106", "gpt-4", "Llama-2-70b-chat-hf", "CodeLlama-34b-Instruct-hf", # "CodeLlama-70b-Instruct-hf", "Mistral-7B-Instruct-v01", "Mistral-7B-OpenOrca"] # agents = ["Prompt Agent", "CoT Agent", "SC-CoT Agent", # "ToT Agent", "MCTS", "Random", "TitforTat"] demo = gr.Blocks(css=custom_css) def load_image(image_path): image = Image.open(image_path) return image with demo: with gr.Row(): gr.Image("./assets/logo.png", height="200px", width="200px", scale=0.1, show_download_button=False, container=False) gr.HTML(TITLE, elem_id="title") gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") with gr.Tabs(elem_classes="tab-buttons") as tabs: with gr.TabItem("🏅 GTBench", elem_id="llm-benchmark-tab-table", id=0): with gr.Row(): with gr.Column(): with gr.Row(): shown_columns = gr.CheckboxGroup( choices=[ 'Average' ]+games, label="Select columns to show", elem_id="column-select", interactive=True, ) with gr.Column(min_width=320): # with gr.Box(elem_id="box-filter"): model1_column = gr.CheckboxGroup( label="Model", choices=models, interactive=True, elem_id="filter-columns-type", ) agent1_column = gr.CheckboxGroup( label="Agents", choices=agents, interactive=True, elem_id="filter-columns-precision", ) model2_column = gr.CheckboxGroup( label="Opponent Model", choices=opponent_models, interactive=True, elem_id="filter-columns-type", ) agent2_column = gr.CheckboxGroup( label="Opponent Agents", choices=opponent_agents, interactive=True, elem_id="filter-columns-precision", ) # filter_columns_size = gr.CheckboxGroup( # label="Model sizes (in billions of parameters)", # choices=[f'NUMERIC_INTERVALS{i}' for i in range(0, 5)], # value=[f'NUMERIC_INTERVALS{i}' for i in range(0, 5)], # interactive=True, # elem_id="filter-columns-size", # ) leaderboard_table = gr.components.Dataframe( value=gtbench_raw_data, elem_id="leaderboard-table", interactive=False, visible=True, # column_widths=["2%", "33%"] ) game_bench_df_for_search = gr.components.Dataframe( value=gtbench_raw_data, elem_id="leaderboard-table", interactive=False, visible=False, # column_widths=["2%", "33%"] ) # Dummy leaderboard for handling the case when the user uses backspace key # hidden_leaderboard_table_for_search = gr.components.Dataframe( # value=[], # headers=COLS, # datatype=TYPES, # visible=False, # ) # search_bar.submit( # update_table, # [ # # hidden_leaderboard_table_for_search, # # shown_columns, # # filter_columns_type, # # filter_columns_precision, # # filter_columns_size, # # deleted_models_visibility, # # flagged_models_visibility, # # search_bar, # ], # leaderboard_table, # ) # # Define a hidden component that will trigger a reload only if a query parameter has be set # hidden_search_bar = gr.Textbox(value="", visible=False) # hidden_search_bar.change( # update_table, # [ # hidden_leaderboard_table_for_search, # shown_columns, # filter_columns_type, # filter_columns_precision, # filter_columns_size, # deleted_models_visibility, # flagged_models_visibility, # search_bar, # ], # leaderboard_table, # ) # # Check query parameter once at startup and update search bar + hidden component # demo.load(load_query, inputs=[], outputs=[search_bar, hidden_search_bar]) for selector in [shown_columns, model1_column, model2_column, agent1_column, agent2_column]: selector.change( update_table, [ game_bench_df_for_search, shown_columns, model1_column, model2_column, agent1_column, agent2_column # filter_columns_precision, # None, # filter_columns_size, # None, # deleted_models_visibility, # None, # flagged_models_visibility, # None, # search_bar, ], leaderboard_table, queue=True, ) # with gr.TabItem("📈 Metrics through time", elem_id="llm-benchmark-tab-table", id=4): # with gr.Row(): # with gr.Column(): # chart = create_metric_plot_obj_1( # dummy_data_for_plot( # ["Metric1", "Metric2", 'Metric3']), # ["Metric1", "Metric2", "Metric3"], # title="Average of Top Scores and Human Baseline Over Time (from last update)", # ) # gr.Plot(value=chart, min_width=500) with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2): gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text") gr.Markdown(FAQ_TEXT, elem_classes="markdown-text") ''' with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3): with gr.Column(): with gr.Row(): gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text") with gr.Column(): with gr.Accordion( f"✅ Finished Evaluations ({9})", open=False, ): with gr.Row(): finished_eval_table = gr.components.Dataframe( value=None, headers=EVAL_COLS, datatype=EVAL_TYPES, row_count=5, ) with gr.Accordion( f"🔄 Running Evaluation Queue ({5})", open=False, ): with gr.Row(): running_eval_table = gr.components.Dataframe( value=None, headers=EVAL_COLS, datatype=EVAL_TYPES, row_count=5, ) with gr.Accordion( f"⏳ Pending Evaluation Queue ({7})", open=False, ): with gr.Row(): pending_eval_table = gr.components.Dataframe( value=None, headers=EVAL_COLS, datatype=EVAL_TYPES, row_count=5, ) with gr.Row(): gr.Markdown("# ✉️✨ Submit your Agent here!", elem_classes="markdown-text") with gr.Row(): with gr.Column(): model_name_textbox = gr.Textbox(label="Agent name") # revision_name_textbox = gr.Textbox( # label="Revision commit", placeholder="main") # private = gr.Checkbox( # False, label="Private", visible=not IS_PUBLIC) model_type = gr.Dropdown( choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown], label="Agent type", multiselect=False, value=ModelType.FT.to_str(" : "), interactive=True, ) # with gr.Column(): # precision = gr.Dropdown( # choices=[i.value.name for i in Precision if i != # Precision.Unknown], # label="Precision", # multiselect=False, # value="float16", # interactive=True, # ) # weight_type = gr.Dropdown( # choices=[i.value.name for i in WeightType], # label="Weights type", # multiselect=False, # value="Original", # interactive=True, # ) # base_model_name_textbox = gr.Textbox( # label="Base model (for delta or adapter weights)") submit_button = gr.Button("Submit Eval") submission_result = gr.Markdown() # submit_button.click( # add_new_eval, # [ # model_name_textbox, # base_model_name_textbox, # revision_name_textbox, # precision, # private, # weight_type, # model_type, # ], # submission_result, # ) ''' with gr.Row(): with gr.Accordion("📙 Citation", open=False): citation_button = gr.Textbox( value=CITATION_BUTTON_TEXT, label=CITATION_BUTTON_LABEL, lines=20, elem_id="citation-button", show_copy_button=True, ) # scheduler = BackgroundScheduler() # scheduler.add_job(restart_space, "interval", seconds=1800) # scheduler.start() demo.launch() # Both launches the space and its CI # configure_space_ci( # demo.queue(default_concurrency_limit=40), # trusted_authors=[], # add manually trusted authors # private="True", # ephemeral spaces will have same visibility as the main space. Otherwise, set to `True` or `False` explicitly. # variables={}, # We overwrite HF_HOME as tmp CI spaces will have no cache # secrets=["HF_TOKEN", "H4_TOKEN"], # which secret do I want to copy from the main space? Can be a `List[str]`."HF_TOKEN", "H4_TOKEN" # hardware=None, # "cpu-basic" by default. Otherwise set to "auto" to have same hardware as the main space or any valid string value. # storage=None, # no storage by default. Otherwise set to "auto" to have same storage as the main space or any valid string value. # ).launch() # notes: opponent model , opponent agent # column is games