import pandas as pd import numpy as np import gradio as gr import plotly.express as px import plotly.graph_objects as go # Creating data for explanation df in about section explanation_data = { "Accuracy Scores": [ "DrugMatchQA", "MedMCQA: G2B", "MedMCQA: Original", "MedMCQA: Difference", "MedQA: G2B", "MedQA: Original", "MedQA: Difference", "Adjusted Robustness Score" ], "Description": [ "A custom MC task where the model is asked to match a brand name to its generic counterpart and vice versa. This task is designed to test the model's ability to understand drug name synonyms.", "G2B Refers to the 'Generic' to 'Brand' name swap. This is model accuracy on MedMCQA task where generic drug names are substituted with brand names.", "Model accuracy on MedMCQA task with original data. (Only includes questions that overlap with the g2b dataset)", "Difference in MedMCQA accuracy for swapped and non-swapped datasets, highlighting the impact of G2B drug name substitution on performance.", "Model accuracy on MedQA (4 options) task where generic drug names are substituted with brand names.", "Model accuracy on MedQA (4 options) task with original data. (Only includes questions that overlap with the g2b dataset)", "Difference in MedMCQA accuracy for swapped and non-swapped datasets, highlighting the impact of G2B drug name substitution on performance.", "A score given by Avg Difference / Avg G2B Accuracy. A higher score indicates a model that is more robust to drug name synonym substitution." ] } explanation_df = pd.DataFrame(explanation_data) #Loading and cleaning eval data processed by json2df.py df = pd.read_csv("data/csv/models_data.csv") df['average_g2b'] = df[['medmcqa_g2b', 'medqa_4options_g2b']].mean(axis=1).round(2) df['average_original_acc'] = df[['medmcqa_orig_filtered', 'medqa_4options_orig_filtered']].mean(axis=1).round(2) df['average_diff'] = df[['medmcqa_diff', 'medqa_diff']].mean(axis=1).round(2) df.drop(columns=['b4b'], inplace=True) #Rename columns for clarity df.rename(columns={ 'medmcqa_g2b': 'MedMCQA: G2B', 'medmcqa_orig_filtered': 'MedMCQA: Original', 'medmcqa_diff': 'MedMCQA: Difference', 'medqa_4options_g2b': 'MedQA: G2B', 'medqa_4options_orig_filtered': 'MedQA: Original', 'medqa_diff': 'MedQA: Difference', 'b4bqa': 'DrugMatchQA', 'average_g2b': 'Average G2B Accuracy', 'average_original_acc': 'Average Original Accuracy', 'average_diff': 'Average Difference' }, inplace=True) #Create adjusted robustness score that accounts for g2b accuracy and difference in accuracy df['Average Accuracy (Original and G2B)'] = (df['Average G2B Accuracy'] + df['Average Original Accuracy']) / 2 #df['Adjusted Robustness Score'] = df['Average Accuracy (Original and G2B)'] - 0.25 - df['Average Difference'].abs() #df['Adjusted Robustness Score'] = df['Adjusted Robustness Score'].round(2) #if acc is 0 in DrugMatchQA column, set it to none df['DrugMatchQA'] = df['DrugMatchQA'].apply(lambda x: None if x == 0 else x) def remove_rows_with_strings(df, column, strings): for string in strings: df = df[~df[column].str.contains(string)] return df models_to_remove = ['microsoft-phi-1', 'microsoft-phi-1_5', 'meta-llama-Llama-2-7b-hf'] non_random_df = remove_rows_with_strings(df, 'Model', models_to_remove) #Defining functions for filtering and plotting filter_mapping = { "all": "all", "🟢 Pre-trained": "🟢", "🟩 Continuously pre-trained": "🟩", "🔶 Fine-tuned on domain-specific data": "🔶", "💬 Chat-models (RLHF, DPO, IFT, ...)": "💬" } def filter_items(df, query): if query == "all": return df filter_value = filter_mapping[query] return df[df["T"].str.contains(filter_value, na=False)] def create_scatter_plot(df, x_col, y_col, title, x_title, y_title): fig = px.scatter(df, x=x_col, y=y_col, color='Model', title=title) fig.add_trace( go.Scatter( x=[0, 100], y=[0, 100], mode="lines", name="y=x line", line=dict(color='black', dash='dash') ) ) fig.update_layout( xaxis_title=x_title, yaxis_title=y_title, xaxis=dict(range=[0, 100]), yaxis=dict(range=[0, 100]), legend_title_text='Model' ) fig.update_traces(marker=dict(size=10), selector=dict(mode='markers')) return fig def create_lm_plot(df, x_col, y_col, title, x_title, y_title): fig = px.scatter(df, x=x_col, y=y_col, color='Model', title=title, trendline='ols') fig.update_layout( xaxis_title=x_title, yaxis_title=y_title, legend_title_text='Model' ) fig.update_traces(marker=dict(size=10), selector=dict(mode='markers')) return fig def create_bar_plot(df, col, title): sorted_df = df.sort_values(by=col, ascending=True) fig = px.bar(sorted_df, x=col, y='Model', orientation='h', title=title, color=col, color_continuous_scale='Aggrnyl') fig.update_layout(xaxis_title=col, yaxis_title='Model', height=600, coloraxis_showscale=False) fig.update_xaxes(range=[-20, 20]) return fig def create_bar_plot_drugmatchqa(df, col, title): clean_df = df.dropna(subset=['DrugMatchQA']) sorted_df = clean_df.sort_values(by=col, ascending=True) fig = px.bar(sorted_df, x=col, y='Model', orientation='h', title=title, color=col, color_continuous_scale='Aggrnyl') fig.update_layout(xaxis_title=col, yaxis_title='Model', height=600, coloraxis_showscale=False) return fig def create_bar_plot_adjusted(df, col, title): sorted_df = df.sort_values(by=col, ascending=True) fig = px.bar(sorted_df, x=col, y='Model', orientation='h', title=title, color=col, color_continuous_scale='Aggrnyl') fig.update_layout(xaxis_title=col, yaxis_title='Model', height=600, coloraxis_showscale=False) return fig #Create UI/Layout with gr.Blocks(css="custom.css") as demo: with gr.Column(): gr.Markdown( """

🐰 RABBITS: Robust Assessment of Biomedical Benchmarks Involving drug Term Substitutions

""" ) with gr.Row(): gr.Markdown(""" """) with gr.Row(): gr.Markdown( """

Robust language models are crucial in the medical domain and the RABBITS project tests the robustness of LLMs by evaluating their handling of synonyms, specifically brand and generic drug names. We assessed 16 open-source language models from Hugging Face using systematic synonym substitution on MedQA and MedMCQA tasks. Our results show a consistent decline in performance across all model sizes, highlighting challenges in synonym comprehension. Additionally, we discovered significant dataset contamination by identifying overlaps between MedQA, MedMCQA test sets, and the Dolma 1.6 dataset using an 8-gram analysis. This highlights the need to improve model robustness and address contamination in open-source datasets.

""" ) with gr.Row(): gr.Markdown(""" """) with gr.Row(): gr.Markdown(""" """) with gr.Row(): bar1 = gr.Plot( value=create_bar_plot(df, "MedMCQA: Difference", "Impact of Generic2Brand swap on MedMCQA Accuracy"), elem_id="bar1" ) bar2 = gr.Plot( value=create_bar_plot(df, "MedQA: Difference", "Impact of Generic2Brand swap on MedQA Accuracy"), elem_id="bar2" ) with gr.Row(): gr.Markdown(""" """) #default_visible_columns = [] with gr.Tabs(elem_classes="tab-buttons"): with gr.TabItem("🔍 Evaluation table"): with gr.Column(): with gr.Accordion("➡️ See All Columns", open=False): shown_columns = gr.CheckboxGroup( choices=df.columns.tolist(), value=df.columns.tolist(), label="Select Columns", interactive=True, ) with gr.Row(): search_bar = gr.Textbox( placeholder="🔍 Search for your model and press ENTER...", show_label=False, elem_id="search-bar" ) filter_columns = gr.Radio( label="⏚ Filter model types", choices=[ "all", "🟢 Pre-trained", "🟩 Continuously pre-trained", "🔶 Fine-tuned on domain-specific data", "💬 Chat-models (RLHF, DPO, IFT, ...)" ], value="all", elem_id="filter-columns", ) leaderboard_df = gr.Dataframe( value=df, headers="keys", datatype=["html" if col == "Model" else "str" for col in df.columns], interactive=False, elem_id="leaderboard-table" ) def update_leaderboard(search_query): filtered_df = df[df["Model"].str.contains(search_query, case=False)] return filtered_df search_bar.submit( update_leaderboard, inputs=search_bar, outputs=leaderboard_df ) def filter_update(query): filtered_df = filter_items(df, query) return filtered_df filter_columns.change( filter_update, inputs=filter_columns, outputs=leaderboard_df ) shown_columns.change( lambda cols: df[cols], inputs=shown_columns, outputs=leaderboard_df ) with gr.TabItem("📊 Evaluation Plots"): with gr.Column(): with gr.Row(): scatter1 = gr.Plot( value=create_scatter_plot(df, "MedMCQA: Original", "MedMCQA: G2B", "MedMCQA: Orig vs G2B", "MedMCQA: Original", "MedMCQA: G2B"), elem_id="scatter1" ) scatter2 = gr.Plot( value=create_scatter_plot(df, "MedQA: Original", "MedQA: G2B", "MedQA: Orig vs G2B", "MedQA: Original", "MedQA: G2B"), elem_id="scatter2" ) with gr.TabItem("📝 About"): with gr.Column(): gr.Markdown( """

About the RABBITS LLM Leaderboard

The following is an overview of the framework, along with an explanation of scores in the evaluation table.

""", elem_classes="markdown-text" ) with gr.Row(): gr.Image(value="workflow-1-2.svg", width=200, height=450) gr.Image(value="workflow-3-4.svg", width=200, height=450) with gr.Row(): gr.Dataframe( value=explanation_df, headers="keys", datatype=["str", "str"], interactive=False, label="Explanation of Scores" ) with gr.TabItem("🚀 Submit Here!"): gr.Markdown( """

Submit Your Model Results

If you have new model results that you would like to add to the leaderboard, please follow the submission guidelines below:

COMING SOON

""", elem_classes="markdown-text" ) with gr.Row(): bar3 = gr.Plot( value=create_bar_plot_drugmatchqa(df, "DrugMatchQA", "Which LLMs are best at matching brand names to generic drug names?"), elem_id="bar3" ) bar4 = gr.Plot( #remove model in model column value=create_bar_plot_adjusted(non_random_df, "Average Difference", "Which LLMs are most robust to drug name synonym substitution?"), elem_id="bar4" ) if __name__ == "__main__": demo.launch()