LFG

app.py

@@ -6,6 +6,7 @@ from datasets import load_dataset
 from src.utils import load_all_data
 from src.md import ABOUT_TEXT, TOP_TEXT
 from src.plt import plot_avg_correlation
+from src.constants import subset_mapping, length_categories, example_counts
 import numpy as np
 
 api = HfApi()
@@ -33,54 +34,34 @@ repo = snapshot_download(
 def avg_over_herm(dataframe):
     """
     Averages over the subsets alpacaeval, mt-bench, llmbar, refusals, hep and returns dataframe with only these columns.
+
+    We average over 4 core sections (per prompt weighting):
+    1. Chat: Includes the easy chat subsets (alpacaeval-easy, alpacaeval-length, alpacaeval-hard, mt-bench-easy, mt-bench-medium)
+    2. Chat Hard: Includes the hard chat subsets (mt-bench-hard, llmbar-natural, llmbar-adver-neighbor, llmbar-adver-GPTInst, llmbar-adver-GPTOut, llmbar-adver-manual)
+    3. Safety: Includes the safety subsets (refusals-dangerous, refusals-offensive, xstest-should-refuse, xstest-should-respond, do not answer)
+    4. Code: Includes the code subsets (hep-cpp, hep-go, hep-java, hep-js, hep-python, hep-rust)
+
     """
     new_df = dataframe.copy()
-    subsets = ["alpacaeval", "mt-bench", "llmbar", "refusals", "hep"]
-    # for each subset, avg the columns that have the subset in the column name, then add a new column with subset name and avg
-    for subset in subsets:
-        if subset == "refusals":
-            subset_cols = ["refusals-dangerous", "refusals-offensive", "donotanswer","xstest-should-refuse", "xstest-should-respond"]
-        else:
-            subset_cols = [col for col in new_df.columns if subset in col]
-        new_df[subset] = np.round(np.nanmean(new_df[subset_cols].values, axis=1), 2)
-
-    keep_columns = ["model", "average"] + subsets
+
+    # for main subsets, keys in subset_mapping, take the weighted avg by example_counts and store for the models
+    for subset, sub_subsets in subset_mapping.items():
+        subset_cols = [col for col in new_df.columns if col in sub_subsets]
+        sub_data = new_df[subset_cols].values # take the relevant column values
+        sub_counts = [example_counts[s] for s in sub_subsets] # take the example counts
+        new_df[subset] = np.round(np.average(sub_data, axis=1, weights=sub_counts), 2) # take the weighted average
+        # new_df[subset] = np.round(np.nanmean(new_df[subset_cols].values, axis=1), 2)
+
+    keep_columns = ["model",] + list(subset_mapping.keys())
+    # keep_columns = ["model", "average"] + subsets
     new_df = new_df[keep_columns]
-    # replace average column with new average
-    new_df["average"] = np.round(np.nanmean(new_df[subsets].values, axis=1), 2)
-    # rename column "hep" to "hep (code)"
-    new_df = new_df.rename(columns={"hep": "hep (code)"})        
+
     return new_df
 
 def expand_subsets(dataframe):
     # TODO need to modify data/ script to do this
     pass
 
-# reference for length bias categories
-length_categories = {
-    'alpacaeval-easy': 'True',
-    'alpacaeval-hard': 'True',
-    'alpacaeval-length': 'Neutral',
-    'donotanswer': 'False',
-    'hep-cpp': 'Neutral',
-    'hep-go': 'Neutral',
-    'hep-java': 'Neutral',
-    'hep-js': 'Neutral',
-    'hep-python': 'Neutral',
-    'hep-rust': 'Neutral',
-    'llmbar-adver-GPTInst': 'False',
-    'llmbar-adver-GPTOut': 'Neutral',
-    'llmbar-adver-manual': 'False',
-    'llmbar-adver-neighbor': 'False',
-    'llmbar-natural': 'Neutral',
-    'mt-bench-easy': 'False',
-    'mt-bench-hard': 'False',
-    'mt-bench-med': 'Neutral',
-    'refusals-dangerous': 'False',
-    'refusals-offensive': 'False',
-    'xstest-should-refuse': 'False',
-    'xstest-should-respond': 'True'
-}
 
 def length_bias_check(dataframe):
     """
@@ -119,7 +100,7 @@ def length_bias_check(dataframe):
 
     
 herm_data = load_all_data(repo_dir_herm, subdir="eval-set").sort_values(by='average', ascending=False)
-herm_data_avg = avg_over_herm(herm_data).sort_values(by='average', ascending=False)
+herm_data_avg = avg_over_herm(herm_data).sort_values(by='Chat', ascending=False)
 herm_data_length = length_bias_check(herm_data).sort_values(by='Terse Bias', ascending=False)
 prefs_data = load_all_data(repo_dir_herm, subdir="pref-sets").sort_values(by='average', ascending=False)
 # prefs_data_sub = expand_subsets(prefs_data).sort_values(by='average', ascending=False)
@@ -149,10 +130,23 @@ def random_sample(r: gr.Request, subset):
 
 subsets = eval_set.unique("subset")
 
+def regex_table(dataframe, regex):
+    """
+    Takes a model name as a regex, then returns only the rows that has that in it.
+    """
+    # Split regex statement by comma and trim whitespace around regexes
+    regex_list = [x.strip() for x in regex.split(",")]
+    # Join the list into a single regex pattern with '|' acting as OR
+    combined_regex = '|'.join(regex_list)
+    # Filter the dataframe such that 'model' contains any of the regex patterns
+    return dataframe[dataframe["model"].str.contains(combined_regex, case=False, na=False)]
+
+
 with gr.Blocks() as app:
     # create tabs for the app, moving the current table to one titled "HERM" and the benchmark_text to a tab called "About"
     with gr.Row():
         gr.Markdown(TOP_TEXT)
+        search = gr.Textbox(label="Model Search (delimit with , )", placeholder="Regex search for a model")
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("HERM Eval Set - Overview"):
             with gr.Row():
@@ -163,24 +157,45 @@ with gr.Blocks() as app:
                     elem_id="herm_dataframe_avg",
                     height=1000,
                 )
+                # backup reference data
+                herm_table_hidden = gr.Dataframe(
+                    herm_data_avg.values,
+                    datatype=col_types_herm_avg,
+                    headers=herm_data_avg.columns.tolist(),
+                    visible=False,
+                )
         with gr.TabItem("HERM Eval Set - Detailed"):
             with gr.Row():
-                herm_table = gr.Dataframe(
+                herm_table_detailed = gr.Dataframe(
                     herm_data.values,
                     datatype=col_types_herm,
                     headers=herm_data.columns.tolist(),
                     elem_id="herm_dataframe",
                     height=1000,
                 )
+                # backup
+                herm_table_detailed_hidden = gr.Dataframe(
+                    herm_data.values,
+                    datatype=col_types_herm,
+                    headers=herm_data.columns.tolist(),
+                    visible=False,
+                )
         with gr.TabItem("HERM Eval Set - Length Bias"):
             with gr.Row():
-                herm_table = gr.Dataframe(
+                herm_table_len = gr.Dataframe(
                     herm_data_length.values,
                     datatype=cols_herm_data_length,
                     headers=herm_data_length.columns.tolist(),
                     elem_id="herm_dataframe_length",
                     height=1000,
                 )
+                # backup
+                herm_table_len_hidden = gr.Dataframe(
+                    herm_data_length.values,
+                    datatype=cols_herm_data_length,
+                    headers=herm_data_length.columns.tolist(),
+                    visible=False,
+                )
         with gr.TabItem("Known Pref. Sets"):
             with gr.Row():
                 PREF_SET_TEXT = """
@@ -195,6 +210,13 @@ with gr.Blocks() as app:
                     elem_id="prefs_dataframe",
                     height=1000,
                 )
+                # backup
+                pref_sets_table_hidden = gr.Dataframe(
+                    prefs_data.values,
+                    datatype=col_types_prefs,
+                    headers=prefs_data.columns.tolist(),
+                    visible=False,
+                )
 
         with gr.TabItem("About"):
             with gr.Row():
@@ -216,6 +238,11 @@ with gr.Blocks() as app:
         #     with gr.Row():
         #         plot = plot_avg_correlation(herm_data_avg, prefs_data)
         #         gr.Plot(plot)
+
+    search.change(regex_table, inputs=[herm_table_hidden, search], outputs=herm_table)
+    search.change(regex_table, inputs=[herm_table_detailed_hidden, search], outputs=herm_table_detailed)
+    search.change(regex_table, inputs=[herm_table_len_hidden, search], outputs=herm_table_len)
+    search.change(regex_table, inputs=[pref_sets_table_hidden, search], outputs=pref_sets_table)    
                 
 # Load data when app starts, TODO make this used somewhere...
 # def load_data_on_start():
@@ -231,6 +258,4 @@ with gr.Blocks() as app:
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=10800) # restarted every 3h
 scheduler.start()
-
-
-app.queue().launch()
+app.launch() # had .queue() before launch before... not sure if that's necessary

src/constants.py

@@ -0,0 +1,57 @@
+# reference for length bias categories
+length_categories = {
+    'alpacaeval-easy': 'True',
+    'alpacaeval-hard': 'True',
+    'alpacaeval-length': 'Neutral',
+    'donotanswer': 'False',
+    'hep-cpp': 'Neutral',
+    'hep-go': 'Neutral',
+    'hep-java': 'Neutral',
+    'hep-js': 'Neutral',
+    'hep-python': 'Neutral',
+    'hep-rust': 'Neutral',
+    'llmbar-adver-GPTInst': 'False',
+    'llmbar-adver-GPTOut': 'Neutral',
+    'llmbar-adver-manual': 'False',
+    'llmbar-adver-neighbor': 'False',
+    'llmbar-natural': 'Neutral',
+    'mt-bench-easy': 'False',
+    'mt-bench-hard': 'False',
+    'mt-bench-med': 'Neutral',
+    'refusals-dangerous': 'False',
+    'refusals-offensive': 'False',
+    'xstest-should-refuse': 'False',
+    'xstest-should-respond': 'True'
+}
+
+example_counts = {
+    "alpacaeval-easy": 100,
+    "alpacaeval-length": 95,
+    "alpacaeval-hard": 95,
+    "mt-bench-easy": 28,
+    "mt-bench-med": 40,
+    "mt-bench-hard": 37,
+    "refusals-dangerous": 100,
+    "refusals-offensive": 100,
+    "llmbar-natural": 100,
+    "llmbar-adver-neighbor": 134,
+    "llmbar-adver-GPTInst": 92,
+    "llmbar-adver-GPTOut": 47,
+    "llmbar-adver-manual": 46,
+    "xstest-should-refuse": 250,
+    "xstest-should-respond": 154,
+    "donotanswer": 136,
+    "hep-cpp": 164,
+    "hep-go": 164,
+    "hep-java": 164,
+    "hep-js": 164,
+    "hep-python": 164,
+    "hep-rust": 164
+}
+
+subset_mapping = {
+    "Chat": ["alpacaeval-easy", "alpacaeval-length", "alpacaeval-hard", "mt-bench-easy", "mt-bench-med"],
+    "Chat Hard": ["mt-bench-hard", "llmbar-natural", "llmbar-adver-neighbor", "llmbar-adver-GPTInst", "llmbar-adver-GPTOut", "llmbar-adver-manual"],
+    "Safety": ["refusals-dangerous", "refusals-offensive", "xstest-should-refuse", "xstest-should-respond", "donotanswer"],
+    "Code": ["hep-cpp", "hep-go", "hep-java", "hep-js", "hep-python", "hep-rust"]
+}

src/md.py

@@ -2,6 +2,12 @@ ABOUT_TEXT = """
 We compute the win percentage for a reward model on hand curated chosen-rejected pairs for each prompt.
 A win is when the score for the chosen response is higher than the score for the rejected response.
 
+We average over 4 core sections (per prompt weighting):
+1. Chat: Includes the easy chat subsets (alpacaeval-easy, alpacaeval-length, alpacaeval-hard, mt-bench-easy, mt-bench-medium)
+2. Chat Hard: Includes the hard chat subsets (mt-bench-hard, llmbar-natural, llmbar-adver-neighbor, llmbar-adver-GPTInst, llmbar-adver-GPTOut, llmbar-adver-manual)
+3. Safety: Includes the safety subsets (refusals-dangerous, refusals-offensive, xstest-should-refuse, xstest-should-respond, do not answer)
+4. Code: Includes the code subsets (hep-cpp, hep-go, hep-java, hep-js, hep-python, hep-rust)
+
 ## Subset Summary
 
 Total number of the prompts is: 2538, filtered from 4676.

src/utils.py

@@ -61,6 +61,11 @@ def load_all_data(data_repo, subdir:str, subsubsets=False):    # use HF api to p
     # select all columns except "model"
     cols = df.columns.tolist()
     cols.remove("model")
+    # remove model_beaker from dataframe
+    if "model_beaker" in cols:
+        cols.remove("model_beaker")
+        df = df.drop(columns=["model_beaker"])
+    
     # round 
     df[cols] = df[cols].round(2)
     avg = np.nanmean(df[cols].values,axis=1).round(2)