Spaces:
Running
Running
| import gradio as gr | |
| import os | |
| from huggingface_hub import HfApi, snapshot_download | |
| from apscheduler.schedulers.background import BackgroundScheduler | |
| from datasets import load_dataset | |
| from src.utils import load_all_data | |
| from src.md import ABOUT_TEXT, TOP_TEXT | |
| from src.plt import plot_avg_correlation | |
| from src.constants import subset_mapping, length_categories, example_counts | |
| import numpy as np | |
| api = HfApi() | |
| COLLAB_TOKEN = os.environ.get("COLLAB_TOKEN") | |
| evals_repo = "allenai/reward-bench-results" | |
| eval_set_repo = "allenai/reward-bench" | |
| repo_dir_rewardbench = "./evals/rewardbench/" | |
| def restart_space(): | |
| api.restart_space(repo_id="allenai/reward-bench", token=COLLAB_TOKEN) | |
| print("Pulling evaluation results") | |
| repo = snapshot_download( | |
| local_dir=repo_dir_rewardbench, | |
| ignore_patterns=["pref-sets-scores/*", "eval-set-scores/*"], | |
| repo_id=evals_repo, | |
| use_auth_token=COLLAB_TOKEN, | |
| tqdm_class=None, | |
| etag_timeout=30, | |
| repo_type="dataset", | |
| ) | |
| def avg_over_rewardbench(dataframe_core, dataframe_prefs): | |
| """ | |
| Averages over the subsets alpacaeval, mt-bench, llmbar, refusals, hep and returns dataframe with only these columns. | |
| We average over 4 core sections (per prompt weighting): | |
| 1. Chat: Includes the easy chat subsets (alpacaeval-easy, alpacaeval-length, alpacaeval-hard, mt-bench-easy, mt-bench-medium) | |
| 2. Chat Hard: Includes the hard chat subsets (mt-bench-hard, llmbar-natural, llmbar-adver-neighbor, llmbar-adver-GPTInst, llmbar-adver-GPTOut, llmbar-adver-manual) | |
| 3. Safety: Includes the safety subsets (refusals-dangerous, refusals-offensive, xstest-should-refuse, xstest-should-respond, do not answer) | |
| 4. Code: Includes the code subsets (hep-cpp, hep-go, hep-java, hep-js, hep-python, hep-rust) | |
| """ | |
| new_df = dataframe_core.copy() | |
| dataframe_prefs = dataframe_prefs.copy() | |
| # for main subsets, keys in subset_mapping, take the weighted avg by example_counts and store for the models | |
| for subset, sub_subsets in subset_mapping.items(): | |
| subset_cols = [col for col in new_df.columns if col in sub_subsets] | |
| sub_data = new_df[subset_cols].values # take the relevant column values | |
| sub_counts = [example_counts[s] for s in sub_subsets] # take the example counts | |
| new_df[subset] = np.round(np.average(sub_data, axis=1, weights=sub_counts), 2) # take the weighted average | |
| # new_df[subset] = np.round(np.nanmean(new_df[subset_cols].values, axis=1), 2) | |
| data_cols = list(subset_mapping.keys()) | |
| keep_columns = ["model",] + ["model_type"] + data_cols | |
| # keep_columns = ["model", "average"] + subsets | |
| new_df = new_df[keep_columns] | |
| # selected average from pref_sets | |
| pref_columns = ["anthropic_helpful", "mtbench_gpt4", "shp", "summarize"] | |
| pref_data = dataframe_prefs[pref_columns].values | |
| # add column test sets knowing the rows are not identical, take superset | |
| dataframe_prefs["Test Sets"] = np.round(np.nanmean(pref_data, axis=1), 2) | |
| # add column Test Sets empty to new_df | |
| new_df["Test Sets"] = np.nan | |
| # per row in new_df if model is in dataframe_prefs, add the value to new_df["Test Sets"] | |
| values = [] | |
| for i, row in new_df.iterrows(): | |
| model = row["model"] | |
| if model in dataframe_prefs["model"].values: | |
| values.append(dataframe_prefs[dataframe_prefs["model"] == model]["Test Sets"].values[0]) | |
| # new_df.at[i, "Test Sets"] = dataframe_prefs[dataframe_prefs["model"] == model]["Test Sets"].values[0] | |
| else: | |
| values.append(np.nan) | |
| new_df["Test Sets"] = values | |
| # add total average | |
| data_cols += ["Test Sets"] | |
| new_df["average"] = np.round(np.nanmean(new_df[data_cols].values, axis=1), 2) | |
| # make average third column | |
| keep_columns = ["model", "model_type", "average"] + data_cols | |
| new_df = new_df[keep_columns] | |
| return new_df | |
| def expand_subsets(dataframe): | |
| # TODO need to modify data/ script to do this | |
| pass | |
| def length_bias_check(dataframe): | |
| """ | |
| Takes the raw rewardbench dataframe and splits the data into new buckets according to length_categories. | |
| Then, take the average of the three buckets as "average" | |
| """ | |
| new_df = dataframe.copy() | |
| existing_subsets = new_df.columns[3:] # model, model_type, average | |
| final_subsets = ["Length Bias", "Neutral", "Terse Bias"] | |
| # new data is empty list dict for each final subset | |
| new_data = {s: [] for s in final_subsets} | |
| # now, subsets correspond to those with True, Nuetral, and False length bias | |
| # check if length_categories[subset] == "True" or "False" or "Neutral" | |
| for subset in existing_subsets: | |
| subset_data = new_df[subset].values | |
| subset_length = length_categories[subset] | |
| # route to the correct bucket | |
| if subset_length == "True": | |
| new_data["Length Bias"].append(subset_data) | |
| elif subset_length == "Neutral": | |
| new_data["Neutral"].append(subset_data) | |
| elif subset_length == "False": | |
| new_data["Terse Bias"].append(subset_data) | |
| # take average of new_data and add to new_df (removing other columns than model) | |
| for subset in final_subsets: | |
| new_df[subset] = np.round(np.nanmean(new_data[subset], axis=0), 2) | |
| keep_columns = ["model"] + final_subsets | |
| new_df = new_df[keep_columns] | |
| # recompute average | |
| # new_df["average"] = np.round(np.nanmean(new_df[final_subsets].values, axis=1), 2) | |
| return new_df | |
| rewardbench_data = load_all_data(repo_dir_rewardbench, subdir="eval-set").sort_values(by='average', ascending=False) | |
| rewardbench_data_length = length_bias_check(rewardbench_data).sort_values(by='Terse Bias', ascending=False) | |
| prefs_data = load_all_data(repo_dir_rewardbench, subdir="pref-sets").sort_values(by='average', ascending=False) | |
| # prefs_data_sub = expand_subsets(prefs_data).sort_values(by='average', ascending=False) | |
| rewardbench_data_avg = avg_over_rewardbench(rewardbench_data, prefs_data).sort_values(by='average', ascending=False) | |
| col_types_rewardbench = ["markdown"] + ["str"] + ["number"] * (len(rewardbench_data.columns) - 1) | |
| col_types_rewardbench_avg = ["markdown"]+ ["str"] + ["number"] * (len(rewardbench_data_avg.columns) - 1) | |
| cols_rewardbench_data_length = ["markdown"] + ["number"] * (len(rewardbench_data_length.columns) - 1) | |
| col_types_prefs = ["markdown"] + ["number"] * (len(prefs_data.columns) - 1) | |
| # col_types_prefs_sub = ["markdown"] + ["number"] * (len(prefs_data_sub.columns) - 1) | |
| # for showing random samples | |
| eval_set = load_dataset(eval_set_repo, use_auth_token=COLLAB_TOKEN, split="filtered") | |
| def random_sample(r: gr.Request, subset): | |
| if subset is None or subset == []: | |
| sample_index = np.random.randint(0, len(eval_set) - 1) | |
| sample = eval_set[sample_index] | |
| else: # filter by subsets (can be list) | |
| if isinstance(subset, str): | |
| subset = [subset] | |
| # filter down dataset to only include the subset(s) | |
| eval_set_filtered = eval_set.filter(lambda x: x["subset"] in subset) | |
| sample_index = np.random.randint(0, len(eval_set_filtered) - 1) | |
| sample = eval_set_filtered[sample_index] | |
| markdown_text = '\n\n'.join([f"**{key}**:\n\n{value}" for key, value in sample.items()]) | |
| return markdown_text | |
| subsets = eval_set.unique("subset") | |
| def regex_table(dataframe, regex, filter_button): | |
| """ | |
| Takes a model name as a regex, then returns only the rows that has that in it. | |
| """ | |
| # Split regex statement by comma and trim whitespace around regexes | |
| regex_list = [x.strip() for x in regex.split(",")] | |
| # Join the list into a single regex pattern with '|' acting as OR | |
| combined_regex = '|'.join(regex_list) | |
| # if filter_button, remove all rows with "ai2" in the model name | |
| if isinstance(filter_button, list) or isinstance(filter_button, str): | |
| if "AI2 Experiments" not in filter_button and ("ai2" not in regex): | |
| dataframe = dataframe[~dataframe["model"].str.contains("ai2", case=False, na=False)] | |
| if "Seq. Classifiers" not in filter_button: | |
| dataframe = dataframe[~dataframe["model_type"].str.contains("Seq. Classifier", case=False, na=False)] | |
| if "DPO" not in filter_button: | |
| dataframe = dataframe[~dataframe["model_type"].str.contains("DPO", case=False, na=False)] | |
| if "Custom Classifiers" not in filter_button: | |
| dataframe = dataframe[~dataframe["model_type"].str.contains("Custom Classifier", case=False, na=False)] | |
| # Filter the dataframe such that 'model' contains any of the regex patterns | |
| return dataframe[dataframe["model"].str.contains(combined_regex, case=False, na=False)] | |
| with gr.Blocks() as app: | |
| # create tabs for the app, moving the current table to one titled "rewardbench" and the benchmark_text to a tab called "About" | |
| with gr.Row(): | |
| with gr.Column(scale=2.2): | |
| # search = gr.Textbox(label="Model Search (delimit with , )", placeholder="Regex search for a model") | |
| # filter_button = gr.Checkbox(label="Include AI2 training runs (or type ai2 above).", interactive=True) | |
| # img = gr.Image(value="https://private-user-images.githubusercontent.com/10695622/310698241-24ed272a-0844-451f-b414-fde57478703e.png", width=500) | |
| gr.Markdown(""" | |
|  | |
| """) | |
| with gr.Column(scale=3): | |
| gr.Markdown(TOP_TEXT) | |
| with gr.Tabs(elem_classes="tab-buttons") as tabs: | |
| with gr.TabItem("🏆 RewardBench Leaderboard"): | |
| with gr.Row(): | |
| search_1 = gr.Textbox(label="Model Search (delimit with , )", placeholder="Regex search for a model") | |
| model_types_1 = gr.CheckboxGroup(["Seq. Classifiers", "DPO", "Custom Classifiers", "AI2 Experiments"], | |
| value=["Seq. Classifiers", "DPO", "Custom Classifiers"], | |
| label="Model Types", | |
| # info="Which model types to include.", | |
| ) | |
| with gr.Row(): | |
| # reference data | |
| rewardbench_table_hidden = gr.Dataframe( | |
| rewardbench_data_avg.values, | |
| datatype=col_types_rewardbench_avg, | |
| headers=rewardbench_data_avg.columns.tolist(), | |
| visible=False, | |
| ) | |
| rewardbench_table = gr.Dataframe( | |
| regex_table(rewardbench_data_avg.copy(), "", ["Seq. Classifiers", "DPO", "Custom Classifiers"]).values, | |
| datatype=col_types_rewardbench_avg, | |
| headers=rewardbench_data_avg.columns.tolist(), | |
| elem_id="rewardbench_dataframe_avg", | |
| height=1000, | |
| ) | |
| with gr.TabItem("🔍 RewardBench - Detailed"): | |
| with gr.Row(): | |
| search_2 = gr.Textbox(label="Model Search (delimit with , )", placeholder="Regex search for a model") | |
| model_types_2 = gr.CheckboxGroup(["Seq. Classifiers", "DPO", "Custom Classifiers", "AI2 Experiments"], | |
| value=["Seq. Classifiers", "DPO", "Custom Classifiers"], | |
| label="Model Types", | |
| # info="Which model types to include." | |
| ) | |
| with gr.Row(): | |
| # ref data | |
| rewardbench_table_detailed_hidden = gr.Dataframe( | |
| rewardbench_data.values, | |
| datatype=col_types_rewardbench, | |
| headers=rewardbench_data.columns.tolist(), | |
| visible=False, | |
| ) | |
| rewardbench_table_detailed = gr.Dataframe( | |
| regex_table(rewardbench_data.copy(), "", ["Seq. Classifiers", "DPO", "Custom Classifiers"]).values, | |
| datatype=col_types_rewardbench, | |
| headers=rewardbench_data.columns.tolist(), | |
| elem_id="rewardbench_dataframe", | |
| height=1000, | |
| ) | |
| # with gr.TabItem("rewardbench Eval Set - Length Bias"): | |
| # with gr.Row(): | |
| # # backup | |
| # rewardbench_table_len_hidden = gr.Dataframe( | |
| # rewardbench_data_length.values, | |
| # datatype=cols_rewardbench_data_length, | |
| # headers=rewardbench_data_length.columns.tolist(), | |
| # visible=False, | |
| # ) | |
| # rewardbench_table_len = gr.Dataframe( | |
| # regex_table(rewardbench_data_length.copy(), "", False).values, | |
| # datatype=cols_rewardbench_data_length, | |
| # headers=rewardbench_data_length.columns.tolist(), | |
| # elem_id="rewardbench_dataframe_length", | |
| # height=1000, | |
| # ) | |
| with gr.TabItem("Existing Test Sets"): | |
| with gr.Row(): | |
| search_3 = gr.Textbox(label="Model Search (delimit with , )", placeholder="Regex search for a model") | |
| model_types_3 = gr.CheckboxGroup(["Seq. Classifiers", "DPO", "Custom Classifiers", "AI2 Experiments"], | |
| value=["Seq. Classifiers", "DPO", "Custom Classifiers"], | |
| label="Model Types", | |
| # info="Which model types to include.", | |
| ) | |
| with gr.Row(): | |
| PREF_SET_TEXT = """ | |
| For more information, see the [dataset](https://huggingface.co/datasets/allenai/pref-test-sets). | |
| """ | |
| gr.Markdown(PREF_SET_TEXT) | |
| with gr.Row(): | |
| # backup | |
| pref_sets_table_hidden = gr.Dataframe( | |
| prefs_data.values, | |
| datatype=col_types_prefs, | |
| headers=prefs_data.columns.tolist(), | |
| visible=False, | |
| ) | |
| pref_sets_table = gr.Dataframe( | |
| regex_table(prefs_data.copy(), "", ["Seq. Classifiers", "DPO", "Custom Classifiers"]).values, | |
| datatype=col_types_prefs, | |
| headers=prefs_data.columns.tolist(), | |
| elem_id="prefs_dataframe", | |
| height=1000, | |
| ) | |
| with gr.TabItem("About"): | |
| with gr.Row(): | |
| gr.Markdown(ABOUT_TEXT) | |
| with gr.TabItem("Dataset Viewer"): | |
| with gr.Row(): | |
| # loads one sample | |
| gr.Markdown("## Random Dataset Sample Viewer") | |
| subset_selector = gr.Dropdown(subsets, label="Subset", value=None, multiselect=True) | |
| button = gr.Button("Show Random Sample") | |
| with gr.Row(): | |
| sample_display = gr.Markdown("{sampled data loads here}") | |
| button.click(fn=random_sample, inputs=[subset_selector], outputs=[sample_display]) | |
| # removed plot because not pretty enough | |
| # with gr.TabItem("Model Correlation"): | |
| # with gr.Row(): | |
| # plot = plot_avg_correlation(rewardbench_data_avg, prefs_data) | |
| # gr.Plot(plot) | |
| search_1.change(regex_table, inputs=[rewardbench_table_hidden, search_1, model_types_1], outputs=rewardbench_table) | |
| search_2.change(regex_table, inputs=[rewardbench_table_detailed_hidden, search_2, model_types_2], outputs=rewardbench_table_detailed) | |
| # search.change(regex_table, inputs=[rewardbench_table_len_hidden, search, filter_button], outputs=rewardbench_table_len) | |
| search_3.change(regex_table, inputs=[pref_sets_table_hidden, search_3, model_types_3], outputs=pref_sets_table) | |
| model_types_1.change(regex_table, inputs=[rewardbench_table_hidden, search_1, model_types_1], outputs=rewardbench_table) | |
| model_types_2.change(regex_table, inputs=[rewardbench_table_detailed_hidden, search_2, model_types_2], outputs=rewardbench_table_detailed) | |
| model_types_3.change(regex_table, inputs=[pref_sets_table_hidden, search_3, model_types_3], outputs=pref_sets_table) | |
| # Load data when app starts, TODO make this used somewhere... | |
| # def load_data_on_start(): | |
| # data_rewardbench = load_all_data(repo_dir_rewardbench) | |
| # rewardbench_table.update(data_rewardbench) | |
| # data_rewardbench_avg = avg_over_rewardbench(repo_dir_rewardbench) | |
| # rewardbench_table.update(data_rewardbench_avg) | |
| # data_prefs = load_all_data(repo_dir_prefs) | |
| # pref_sets_table.update(data_prefs) | |
| scheduler = BackgroundScheduler() | |
| scheduler.add_job(restart_space, "interval", seconds=10800) # restarted every 3h | |
| scheduler.start() | |
| app.launch() # had .queue() before launch before... not sure if that's necessary | |