Spaces:
Running
Running
| import gradio as gr | |
| import pandas as pd | |
| from src.about import CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, INTRODUCTION_TEXT, LLM_BENCHMARKS_TEXT, SUBMIT_TEXT, TITLE | |
| from src.display.css_html_js import custom_css | |
| from src.display.utils import COLS, TYPES, AutoEvalColumn, fields | |
| from src.envs import CRM_RESULTS_PATH | |
| from src.populate import get_leaderboard_df_crm | |
| original_df = get_leaderboard_df_crm(CRM_RESULTS_PATH, COLS) | |
| leaderboard_df = original_df.copy() | |
| # leaderboard_df = leaderboard_df.style.format({"accuracy_metric_average": "{0:.2f}"}) | |
| # Searching and filtering | |
| def update_table( | |
| hidden_df: pd.DataFrame, | |
| columns: list, | |
| framework_query: list | |
| # llm_query: list, | |
| # llm_provider_query: list, | |
| # accuracy_method_query: str, | |
| # accuracy_threshold_query: str, | |
| # use_case_area_query: list, | |
| # use_case_query: list, | |
| # use_case_type_query: list, | |
| # metric_area_query: list, | |
| ): | |
| filtered_df = filter_framework_func(hidden_df, framework_query) | |
| # filtered_df = filter_llm_func(hidden_df, llm_query) | |
| # filtered_df = filter_llm_provider_func(filtered_df, llm_provider_query) | |
| # filtered_df = filter_accuracy_method_func(filtered_df, accuracy_method_query) | |
| # filtered_df["Accuracy Threshold"] = filter_accuracy_threshold_func(filtered_df, accuracy_threshold_query) | |
| # filtered_df = filtered_df[filtered_df["Accuracy Threshold"]] | |
| # filtered_df["Use Case Area"] = filtered_df["Use Case Name"].apply(lambda x: x.split(": ")[0]) | |
| # filtered_df = filter_use_case_area_func(filtered_df, use_case_area_query) | |
| # filtered_df = filter_use_case_func(filtered_df, use_case_query) | |
| # filtered_df = filter_use_case_type_func(filtered_df, use_case_type_query) | |
| # Filtering by metric area | |
| # metric_area_maps = { | |
| # "Cost": ["Cost Band"], | |
| # "Accuracy": ["Accuracy", "Instruction Following", "Conciseness", "Completeness", "Factuality"], | |
| # "Speed (Latency)": ["Response Time (Sec)", "Mean Output Tokens"], | |
| # "Trust & Safety": ["Trust & Safety", "Safety", "Privacy", "Truthfulness", "CRM Fairness"], | |
| # } | |
| # all_metric_cols = [] | |
| # for area in metric_area_maps: | |
| # all_metric_cols = all_metric_cols + metric_area_maps[area] | |
| # columns_to_keep = list(set(columns).difference(set(all_metric_cols))) | |
| # for area in metric_area_query: | |
| # columns_to_keep = columns_to_keep + metric_area_maps[area] | |
| # columns = list(set(columns).intersection(set(columns_to_keep))) | |
| df = select_columns(filtered_df, columns) | |
| return df.style.map(highlight_cost_band_low, props="background-color: #b3d5a4") | |
| # def highlight_cols(x): | |
| # df = x.copy() | |
| # df.loc[:, :] = "color: black" | |
| # df.loc[, ["Accuracy"]] = "background-color: #b3d5a4" | |
| # return df | |
| def highlight_cost_band_low(s, props=""): | |
| return props if s == "Low" else None | |
| def init_leaderboard_df( | |
| leaderboard_df: pd.DataFrame, | |
| columns: list, | |
| llm_query: list, | |
| # llm_provider_query: list, | |
| # accuracy_method_query: str, | |
| # accuracy_threshold_query: str, | |
| # use_case_area_query: list, | |
| # use_case_query: list, | |
| # use_case_type_query: list, | |
| # metric_area_query: list, | |
| ): | |
| # Applying the style function | |
| # return df.style.apply(highlight_cols, axis=None) | |
| return update_table( | |
| leaderboard_df, | |
| columns, | |
| llm_query, | |
| # llm_provider_query, | |
| # accuracy_method_query, | |
| # accuracy_threshold_query, | |
| # use_case_area_query, | |
| # use_case_query, | |
| # use_case_type_query, | |
| # metric_area_query, | |
| ) | |
| def filter_accuracy_method_func(df: pd.DataFrame, accuracy_method_query: str) -> pd.DataFrame: | |
| return df[df["Accuracy Method"] == accuracy_method_query] | |
| def filter_accuracy_threshold_func(df: pd.DataFrame, accuracy_threshold_query: str) -> pd.DataFrame: | |
| accuracy_cols = ["Instruction Following", "Conciseness", "Completeness", "Accuracy"] | |
| return (df.loc[:, accuracy_cols] >= float(accuracy_threshold_query)).all(axis=1) | |
| def filter_use_case_area_func(df: pd.DataFrame, use_case_area_query: list) -> pd.DataFrame: | |
| return df[ | |
| df["Use Case Area"].apply( | |
| lambda x: len(set([_.strip() for _ in x.split("&")]).intersection(use_case_area_query)) | |
| ) | |
| > 0 | |
| ] | |
| def filter_use_case_func(df: pd.DataFrame, use_case_query: list) -> pd.DataFrame: | |
| return df[df["Use Case Name"].isin(use_case_query)] | |
| def filter_use_case_type_func(df: pd.DataFrame, use_case_type_query: list) -> pd.DataFrame: | |
| return df[df["Use Case Type"].isin(use_case_type_query)] | |
| def filter_llm_func(df: pd.DataFrame, llm_query: list) -> pd.DataFrame: | |
| return df[df["Model"].isin(llm_query)] | |
| def filter_framework_func(df: pd.DataFrame, framework_query: list) -> pd.DataFrame: | |
| return df[df["Agentic Framework"].isin(framework_query)] | |
| def filter_llm_provider_func(df: pd.DataFrame, llm_provider_query: list) -> pd.DataFrame: | |
| return df[df["LLM Provider"].isin(llm_provider_query)] | |
| def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame: | |
| # always_here_cols = [ | |
| # AutoEvalColumn.model.name, | |
| # ] | |
| # model_provider_col = [AutoEvalColumn.model_provider.name] if AutoEvalColumn.model_provider.name in columns else [] | |
| # We use COLS to maintain sortingx | |
| filtered_df = df[ | |
| ( | |
| [AutoEvalColumn.model.name] | |
| # + model_provider_col | |
| + [AutoEvalColumn.agentic_framework.name] | |
| + [c for c in COLS if c in df.columns and c in columns ] | |
| + [AutoEvalColumn.overall.name] | |
| ) | |
| ] | |
| return filtered_df | |
| demo = gr.Blocks(css=custom_css) | |
| with demo: | |
| gr.HTML(TITLE) | |
| gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") | |
| with gr.Tabs(elem_classes="tab-buttons") as tabs: | |
| with gr.TabItem("🏅 CRMArena Benchmark", elem_id="llm-benchmark-tab-table", id=0): | |
| with gr.Row(): | |
| with gr.Column(): | |
| filter_agentic_framework = gr.CheckboxGroup( | |
| choices=list(original_df["Agentic Framework"].unique()), | |
| value=list(original_df["Agentic Framework"].unique()), | |
| label="Agentic Frameworks", | |
| info="", | |
| interactive=True, | |
| ) | |
| with gr.Row(): | |
| shown_columns = gr.CheckboxGroup( | |
| choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden], | |
| value=[ | |
| c.name | |
| for c in fields(AutoEvalColumn) | |
| if c.displayed_by_default and not c.hidden and not c.never_hidden | |
| ], | |
| label="CRMArena Tasks", | |
| elem_id="column-select", | |
| interactive=True, | |
| ) | |
| # with gr.Column(): | |
| # filter_llm = gr.CheckboxGroup( | |
| # choices=list(original_df["Model"].unique()), | |
| # value=list(original_df["Model"].unique()), | |
| # label="Model", | |
| # info="", | |
| # interactive=True, | |
| # ) | |
| # with gr.Column(): | |
| # with gr.Row(): | |
| # filter_llm_provider = gr.CheckboxGroup( | |
| # choices=list(original_df["LLM Provider"].unique()), | |
| # value=list(original_df["LLM Provider"].unique()), | |
| # label="LLM Provider", | |
| # info="", | |
| # interactive=True, | |
| # ) | |
| # with gr.Row(): | |
| # filter_metric_area = gr.CheckboxGroup( | |
| # choices=["Accuracy", "Speed (Latency)", "Trust & Safety", "Cost"], | |
| # value=["Accuracy", "Speed (Latency)", "Trust & Safety", "Cost"], | |
| # label="Metric Area", | |
| # info="", | |
| # interactive=True, | |
| # ) | |
| # with gr.Row(): | |
| # filter_use_case = gr.CheckboxGroup( | |
| # choices=list(original_df["Use Case Name"].unique()), | |
| # value=list(original_df["Use Case Name"].unique()), | |
| # label="Use Case", | |
| # info="", | |
| # # multiselect=True, | |
| # interactive=True, | |
| # ) | |
| # with gr.Row(): | |
| # with gr.Column(): | |
| # filter_use_case_area = gr.CheckboxGroup( | |
| # choices=["Service", "Sales"], | |
| # value=["Service", "Sales"], | |
| # label="Use Case Area", | |
| # info="", | |
| # interactive=True, | |
| # ) | |
| # with gr.Column(): | |
| # filter_use_case_type = gr.CheckboxGroup( | |
| # choices=["Summary", "Generation"], | |
| # value=["Summary", "Generation"], | |
| # label="Use Case Type", | |
| # info="", | |
| # interactive=True, | |
| # ) | |
| # with gr.Column(): | |
| # filter_use_case = gr.Dropdown( | |
| # choices=list(original_df["Use Case Name"].unique()), | |
| # value=list(original_df["Use Case Name"].unique()), | |
| # label="Use Case", | |
| # info="", | |
| # multiselect=True, | |
| # interactive=True, | |
| # ) | |
| # with gr.Column(): | |
| # filter_accuracy_method = gr.Radio( | |
| # choices=["Manual", "Auto"], | |
| # value="Manual", | |
| # label="Accuracy Method", | |
| # info="", | |
| # interactive=True, | |
| # ) | |
| # with gr.Column(): | |
| # filter_accuracy_threshold = gr.Number( | |
| # value="0", | |
| # label="Accuracy Threshold", | |
| # info="Range: 0.0 to 4.0", | |
| # interactive=True, | |
| # ) | |
| leaderboard_table = gr.components.Dataframe( | |
| # value=leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value], | |
| value=init_leaderboard_df( | |
| leaderboard_df, | |
| shown_columns.value, | |
| filter_agentic_framework.value | |
| # filter_llm.value, | |
| # filter_llm_provider.value, | |
| # filter_accuracy_method.value, | |
| # filter_accuracy_threshold.value, | |
| # filter_use_case_area.value, | |
| # filter_use_case.value, | |
| # filter_use_case_type.value, | |
| # filter_metric_area.value, | |
| ), | |
| headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value, | |
| datatype=TYPES, | |
| elem_id="leaderboard-table", | |
| interactive=False, | |
| visible=True, | |
| ) | |
| # Dummy leaderboard for handling the case when the user uses backspace key | |
| hidden_leaderboard_table_for_search = gr.components.Dataframe( | |
| value=original_df[COLS], | |
| headers=COLS, | |
| datatype=TYPES, | |
| visible=False, | |
| ) | |
| for selector in [ | |
| shown_columns, | |
| filter_agentic_framework | |
| # filter_llm, | |
| # filter_llm_provider, | |
| # filter_accuracy_method, | |
| # filter_accuracy_threshold, | |
| # filter_use_case_area, | |
| # filter_use_case, | |
| # filter_use_case_type, | |
| # filter_metric_area, | |
| ]: | |
| selector.change( | |
| update_table, | |
| [ | |
| hidden_leaderboard_table_for_search, | |
| shown_columns, | |
| filter_agentic_framework, | |
| # filter_llm, | |
| # filter_llm_provider, | |
| # filter_accuracy_method, | |
| # filter_accuracy_threshold, | |
| # filter_use_case_area, | |
| # filter_use_case, | |
| # filter_use_case_type, | |
| # filter_metric_area, | |
| ], | |
| leaderboard_table, | |
| queue=True, | |
| ) | |
| with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=3): | |
| gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text") | |
| with gr.TabItem("🚀 Submit", elem_id="llm-benchmark-tab-table", id=4): | |
| gr.Markdown(SUBMIT_TEXT, elem_classes="markdown-text") | |
| with gr.Row(): | |
| with gr.Accordion("📙 Citation", open=False): | |
| citation_button = gr.Textbox( | |
| value=CITATION_BUTTON_TEXT, | |
| label=CITATION_BUTTON_LABEL, | |
| lines=20, | |
| elem_id="citation-button", | |
| show_copy_button=True, | |
| ) | |
| # scheduler = BackgroundScheduler() | |
| # scheduler.add_job(restart_space, "interval", seconds=1800) | |
| # scheduler.start() | |
| demo.queue(default_concurrency_limit=40).launch() | |