update

app.py

@@ -42,7 +42,7 @@ def avg_over_rewardbench(dataframe_core, dataframe_prefs):
     2. Chat Hard: Includes the hard chat subsets (mt-bench-hard, llmbar-natural, llmbar-adver-neighbor, llmbar-adver-GPTInst, llmbar-adver-GPTOut, llmbar-adver-manual)
     3. Safety: Includes the safety subsets (refusals-dangerous, refusals-offensive, xstest-should-refuse, xstest-should-respond, do not answer)
     4. Reasoning: Includes the code and math subsets (math-prm, hep-cpp, hep-go, hep-java, hep-js, hep-python, hep-rust)
-    5. Prior Sets: Includes the test sets (anthropic_helpful, mtbench_human, shp, summarize)
+    5. Prior Sets (0.5 weight): Includes the test sets (anthropic_helpful, mtbench_human, shp, summarize)
     """
     new_df = dataframe_core.copy()
     dataframe_prefs = dataframe_prefs.copy()
@@ -52,7 +52,7 @@ def avg_over_rewardbench(dataframe_core, dataframe_prefs):
         subset_cols = [col for col in new_df.columns if col in sub_subsets]
         sub_data = new_df[subset_cols].values # take the relevant column values
         sub_counts = [example_counts[s] for s in sub_subsets] # take the example counts
-        new_df[subset] = np.round(np.average(sub_data, axis=1, weights=sub_counts), 1) # take the weighted average
+        new_df[subset] = np.average(sub_data, axis=1, weights=sub_counts) # take the weighted average
         # new_df[subset] = np.round(np.nanmean(new_df[subset_cols].values, axis=1), 2)
 
     data_cols = list(subset_mapping.keys())
@@ -65,25 +65,30 @@ def avg_over_rewardbench(dataframe_core, dataframe_prefs):
     pref_data = dataframe_prefs[pref_columns].values
 
     # add column test sets knowing the rows are not identical, take superset
-    dataframe_prefs["Prior Sets"] = np.round(np.nanmean(pref_data, axis=1), 1)
+    dataframe_prefs["Prior Sets (0.5 weight)"] = np.nanmean(pref_data, axis=1)
 
     # add column Test Sets empty to new_df
-    new_df["Prior Sets"] = np.nan
-    # per row in new_df if model is in dataframe_prefs, add the value to new_df["Prior Sets"]
+    new_df["Prior Sets (0.5 weight)"] = np.nan
+    # per row in new_df if model is in dataframe_prefs, add the value to new_df["Prior Sets (0.5 weight)"]
     values = []
     for i, row in new_df.iterrows():
         model = row["model"]
         if model in dataframe_prefs["model"].values:
-            values.append(dataframe_prefs[dataframe_prefs["model"] == model]["Prior Sets"].values[0])
-            # new_df.at[i, "Prior Sets"] = dataframe_prefs[dataframe_prefs["model"] == model]["Prior Sets"].values[0]
+            values.append(dataframe_prefs[dataframe_prefs["model"] == model]["Prior Sets (0.5 weight)"].values[0])
+            # new_df.at[i, "Prior Sets (0.5 weight)"] = dataframe_prefs[dataframe_prefs["model"] == model]["Prior Sets (0.5 weight)"].values[0]
         else:
             values.append(np.nan)
     
-    new_df["Prior Sets"] = values
+    new_df["Prior Sets (0.5 weight)"] = values
 
     # add total average
-    data_cols += ["Prior Sets"]
-    new_df["average"] = np.round(np.nanmean(new_df[data_cols].values, axis=1), 1)
+    data_cols += ["Prior Sets (0.5 weight)"]
+    final_data = new_df[data_cols].values
+    masked_data = np.ma.masked_array(final_data, np.isnan(final_data))
+    weights = [2, 2, 2, 2, 1]
+    average = np.ma.average(masked_data, axis=1, weights=weights)
+    new_df["average"] = average.filled(np.nan)
+    # new_df["average"] = np.nanmean(new_df[data_cols].values, axis=1)
 
     # make average third column
     keep_columns = ["model", "model_type", "average"] + data_cols
@@ -121,7 +126,7 @@ def length_bias_check(dataframe):
 
     # take average of new_data and add to new_df (removing other columns than model)
     for subset in final_subsets:
-        new_df[subset] = np.round(np.nanmean(new_data[subset], axis=0), 2)
+        new_df[subset] = np.nanmean(new_data[subset], axis=0)
     keep_columns = ["model"] + final_subsets
     new_df = new_df[keep_columns]
     # recompute average
@@ -148,7 +153,9 @@ def prep_df(df):
 
 # add count column to all dataframes
 rewardbench_data = prep_df(rewardbench_data)
-rewardbench_data_avg = prep_df(rewardbench_data_avg)
+rewardbench_data_avg = prep_df(rewardbench_data_avg).rename(columns={"Average": "Score"})
+# adjust weight of this average to 50% for Prior Sets (0.5 weight), 1 for others
+
 rewardbench_data_length = prep_df(rewardbench_data_length)
 prefs_data = prep_df(prefs_data)
 
@@ -201,6 +208,16 @@ def regex_table(dataframe, regex, filter_button):
 
     # replace column '' with count/rank
     data[''] = np.arange(1, 1 + len(data))
+
+    # if Score exists, round to 2 decimals
+    if "Score" in data.columns:
+        data["Score"] = data["Score"].round(2)
+    if "Average" in data.columns:
+        data["Average"] = data["Average"].round(1)
+    # round all others to 1 decimal
+    for col in data.columns:
+        if col not in ["", "Model", "Model Type", "Score", "Average"]:
+            data[col] = data[col].round(1)
     return data
 
 

src/utils.py

@@ -96,9 +96,9 @@ def load_all_data(data_repo, subdir:str, subsubsets=False):    # use HF api to p
         df = df.drop(columns=["pku_safer"])
         cols.remove("pku_safer")
 
-    # round 
-    df[cols] = (df[cols]*100).round(1)
-    avg = np.nanmean(df[cols].values,axis=1).round(1)
+    # convert to score 
+    df[cols] = (df[cols]*100)
+    avg = np.nanmean(df[cols].values,axis=1)
     # add average column
     df["average"] = avg