up

src/constants.py

@@ -31,7 +31,7 @@ example_counts = {
     "mt-bench-easy": 28,
     "mt-bench-med": 40,
     "mt-bench-hard": 37,
-    "math-prm": 984, # actual length 447, upweighting to be equal to code
+    # "math-prm": 984, # actual length 447, upweighting to be equal to code
     "refusals-dangerous": 100,
     "refusals-offensive": 100,
     "llmbar-natural": 100,
@@ -54,5 +54,6 @@ subset_mapping = {
     "Chat": ["alpacaeval-easy", "alpacaeval-length", "alpacaeval-hard", "mt-bench-easy", "mt-bench-med"],
     "Chat Hard": ["mt-bench-hard", "llmbar-natural", "llmbar-adver-neighbor", "llmbar-adver-GPTInst", "llmbar-adver-GPTOut", "llmbar-adver-manual"],
     "Safety": ["refusals-dangerous", "refusals-offensive", "xstest-should-refuse", "xstest-should-respond", "donotanswer"],
-    "Reasoning": ["math-prm", "hep-cpp", "hep-go", "hep-java", "hep-js", "hep-python", "hep-rust"]
+    "Reasoning": [#"math-prm", 
+                  "hep-cpp", "hep-go", "hep-java", "hep-js", "hep-python", "hep-rust"]
 }