Repository: EleutherAI/lm-evaluation-harness
Pull request: [Refactor] Mmlu subgroups and weight avg
1. Further splits MMLU into 4 subcategories instead of just aggregating over all subtasks.
2. Add weighted averaging and stderr.

Problem statement:
Add mmlu average score in report
Currently I am using hendrycksTest-* for measuring the performance on MMLU. However, the scores are reported for each single task. It would much more convenient if there are scores for each subcategory and average score. Also, MMLU might be a more well known name compared with hendrycksTest. Is it possible to add mmlu as an alternative name for testing the results for all tasks.

Hints from issue discussion:
any progress on this average score?
@lintangsutawika is working on this!

Commit message: added stderr reprocessing for groups

Patch to review:
diff --git a/lm_eval/evaluator.py b/lm_eval/evaluator.py
index bf35097c5b..5d923b1db9 100644
--- a/lm_eval/evaluator.py
+++ b/lm_eval/evaluator.py
@@ -449,23 +449,8 @@ def evaluate(
                 group_name = None
 
             agg_fn = task.aggregation()[metric]
-            task_score = agg_fn(items)
-
-            if group_name is not None:
-                sample_metric_key = metric + "(sample agg)," + key
-                for grouping in task_to_group[task_name]:
-                    if metric_key in results[grouping]:
-                        results[grouping][metric_key].append(task_score)
-                    else:
-                        results[grouping][metric_key] = [task_score]
-
-                    if sample_metric_key in results[grouping]:
-                        results[grouping][sample_metric_key] += items
-                    else:
-                        results[grouping][sample_metric_key] = items.copy()
-                        sample_agg_fn[grouping][sample_metric_key] = agg_fn
-
-            results[task_name][metric_key] = task_score
+            results[task_name][metric_key] = agg_fn(items)
+            results[task_name]["samples"] = len(items)
 
             # hotfix: bleu, chrf, ter seem to be really expensive to bootstrap
             # so we run them less iterations. still looking for a cleaner way to do this
@@ -481,18 +466,37 @@ def evaluate(
                     results[task_name][metric + "_stderr" + "," + key] = stderr(items)
 
         if bool(results):
-            for task_or_group in results.keys():
-                for metric in results[task_or_group].keys():
-                    if type(results[task_or_group][metric]) == list:
-                        if "(sample agg)" in metric:
-                            results[task_or_group][metric] = sample_agg_fn[
-                                task_or_group
-                            ][metric](results[task_or_group][metric])
+
+            for group, task_list in reversed(task_hierarchy.items()):
+                versions[group] = "N/A"
+                task_score_dict = {}
+                total_size = 0
+                for task in task_list:
+                    metrics = results[task]
+
+                    current_size = metrics.pop("samples")
+                    # if "size" in metrics:
+                    #     current_size = metrics.pop("size")
+                    # else:
+                    #     current_size = 1
+
+                    for metric in [key for key in metrics.keys() if "_stderr" not in key]:
+
+                        stderr = "_stderr,".join(metric.split(","))
+                        stderr_score = results[task][stderr]
+                        metric_score = results[task][metric]
+
+                        if metric in results[group]:
+                            results[group][metric] = (results[group][metric]*total_size + metric_score*current_size)/(total_size+current_size)
+                            # $$s_z^2 = \frac{(n-1) s_x^2 + (m-1) s_y^2}{n+m-1} + \frac{nm(\bar x - \bar y)^2}{(n+m)(n+m-1)}.$$
+                            results[group][stderr] = ((total_size-1)*results[group][stderr]+(current_size-1)*stderr_score)/(total_size + current_size - 1) \
+                                                        + total_size*current_size/((total_size+current_size)*(total_size+current_size-1))*(results[group][metric] - metric_score)**2
                         else:
-                            results[task_or_group][metric] = np.average(
-                                results[task_or_group][metric]
-                            )
-                        versions[task_or_group] = "N/A"
+                            results[group][metric] = metric_score
+                            results[group][stderr] = stderr_score
+
+                    total_size += current_size
+                results[group]["samples"] = total_size
 
         for task_name, task in task_dict.items():
             if type(task) == tuple:
diff --git a/lm_eval/tasks/mmlu/_generate_configs.py b/lm_eval/tasks/mmlu/_generate_configs.py
index 542e11b2d5..b34a39ea96 100644
--- a/lm_eval/tasks/mmlu/_generate_configs.py
+++ b/lm_eval/tasks/mmlu/_generate_configs.py
@@ -1,5 +1,5 @@
 """
-Take in a YAML, and output all other splits with this YAML
+Take in a YAML, and output all "other" splits with this YAML
 """
 import os
 import yaml
@@ -10,65 +10,65 @@
 from lm_eval import utils
 from lm_eval.logger import eval_logger
 
-SUBJECTS = [
-    "abstract_algebra",
-    "anatomy",
-    "astronomy",
-    "business_ethics",
-    "clinical_knowledge",
-    "college_biology",
-    "college_chemistry",
-    "college_computer_science",
-    "college_mathematics",
-    "college_medicine",
-    "college_physics",
-    "computer_security",
-    "conceptual_physics",
-    "econometrics",
-    "electrical_engineering",
-    "elementary_mathematics",
-    "formal_logic",
-    "global_facts",
-    "high_school_biology",
-    "high_school_chemistry",
-    "high_school_computer_science",
-    "high_school_european_history",
-    "high_school_geography",
-    "high_school_government_and_politics",
-    "high_school_macroeconomics",
-    "high_school_mathematics",
-    "high_school_microeconomics",
-    "high_school_physics",
-    "high_school_psychology",
-    "high_school_statistics",
-    "high_school_us_history",
-    "high_school_world_history",
-    "human_aging",
-    "human_sexuality",
-    "international_law",
-    "jurisprudence",
-    "logical_fallacies",
-    "machine_learning",
-    "management",
-    "marketing",
-    "medical_genetics",
-    "miscellaneous",
-    "moral_disputes",
-    "moral_scenarios",
-    "nutrition",
-    "philosophy",
-    "prehistory",
-    "professional_accounting",
-    "professional_law",
-    "professional_medicine",
-    "professional_psychology",
-    "public_relations",
-    "security_studies",
-    "sociology",
-    "us_foreign_policy",
-    "virology",
-    "world_religions",
-]
+SUBJECTS = {
+    "abstract_algebra": "stem",
+    "anatomy": "stem",
+    "astronomy": "stem",
+    "business_ethics": "other",
+    "clinical_knowledge": "other",
+    "college_biology": "stem",
+    "college_chemistry": "stem",
+    "college_computer_science": "stem",
+    "college_mathematics": "stem",
+    "college_medicine": "other",
+    "college_physics": "stem",
+    "computer_security": "stem",
+    "conceptual_physics": "stem",
+    "econometrics": "social_sciences",
+    "electrical_engineering": "stem",
+    "elementary_mathematics": "stem",
+    "formal_logic": "humanities",
+    "global_facts": "other",
+    "high_school_biology": "stem",
+    "high_school_chemistry": "stem",
+    "high_school_computer_science": "stem",
+    "high_school_european_history": "humanities",
+    "high_school_geography": "social_sciences",
+    "high_school_government_and_politics": "social_sciences",
+    "high_school_macroeconomics": "social_sciences",
+    "high_school_mathematics": "stem",
+    "high_school_microeconomics": "social_sciences",
+    "high_school_physics": "stem",
+    "high_school_psychology": "social_sciences",
+    "high_school_statistics": "stem",
+    "high_school_us_history": "humanities",
+    "high_school_world_history": "humanities",
+    "human_aging": "other",
+    "human_sexuality": "social_sciences",
+    "international_law": "humanities",
+    "jurisprudence": "humanities",
+    "logical_fallacies": "humanities",
+    "machine_learning": "stem",
+    "management": "other",
+    "marketing": "other",
+    "medical_genetics": "other",
+    "miscellaneous": "other",
+    "moral_disputes": "humanities",
+    "moral_scenarios": "humanities",
+    "nutrition": "other",
+    "philosophy": "humanities",
+    "prehistory": "humanities",
+    "professional_accounting": "other",
+    "professional_law": "humanities",
+    "professional_medicine": "other",
+    "professional_psychology": "social_sciences",
+    "public_relations": "social_sciences",
+    "security_studies": "social_sciences",
+    "sociology": "social_sciences",
+    "us_foreign_policy": "social_sciences",
+    "virology": "other",
+    "world_religions": "humanities",
+}
 
 
 def parse_args():
@@ -77,6 +77,7 @@ def parse_args():
     parser.add_argument("--save_prefix_path", default="flan")
     parser.add_argument("--cot_prompt_path", default=None)
     parser.add_argument("--task_prefix", default="")
+    parser.add_argument("--group_prefix", default="")
     return parser.parse_args()
 
 
@@ -84,7 +85,7 @@ def parse_args():
 
     args = parse_args()
 
-    # get filename of base_yaml so we can `"include": ` it in our other YAMLs.
+    # get filename of base_yaml so we can `"include": ` it in our "other" YAMLs.
     base_yaml_name = os.path.split(args.base_yaml_path)[-1]
     with open(args.base_yaml_path) as f:
         base_yaml = yaml.full_load(f)
@@ -95,7 +96,12 @@ def parse_args():
         with open(args.cot_prompt_path) as f:
             cot_file = json.load(f)
 
-    for subject in tqdm(SUBJECTS):
+    ALL_CATEGORIES = []
+    for subject, category in tqdm(SUBJECTS.items()):
+
+        if category not in ALL_CATEGORIES:
+            ALL_CATEGORIES.append(category)
+
         if args.cot_prompt_path is not None:
             description = cot_file[subject]
         else:
@@ -103,6 +109,7 @@ def parse_args():
 
         yaml_dict = {
             "include": base_yaml_name,
+            "group": f"mmlu_{category}",
             "task": f"mmlu_{args.task_prefix}_{subject}"
             if args.task_prefix != ""
             else f"mmlu_{subject}",
@@ -120,3 +127,18 @@ def parse_args():
                 allow_unicode=True,
                 default_style='"',
             )
+
+    if args.group_prefix == "":
+        file_save_path = args.save_prefix_path + ".yaml"
+    else:
+        file_save_path = args.save_prefix_path + f"_{args.group_prefix}.yaml"
+    eval_logger.info(f"Saving benchmark config to {file_save_path}")
+    with open(file_save_path, "w") as yaml_file:
+        yaml.dump(
+            {
+                "group": f"mmlu_{args.group_prefix}",
+                "task": [f"mmlu_{category}" for category in ALL_CATEGORIES]
+                },
+            yaml_file,
+            default_flow_style=False
+        )
diff --git a/lm_eval/tasks/mmlu/default/_default_template_yaml b/lm_eval/tasks/mmlu/default/_default_template_yaml
index bd989c408a..5eb1c06998 100644
--- a/lm_eval/tasks/mmlu/default/_default_template_yaml
+++ b/lm_eval/tasks/mmlu/default/_default_template_yaml
@@ -1,4 +1,3 @@
-group: mmlu
 dataset_path: cais/mmlu
 test_split: test
 fewshot_split: dev
@@ -12,6 +11,3 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
-  - metric: acc_norm
-    aggregation: mean
-    higher_is_better: true
diff --git a/lm_eval/tasks/mmlu/default/mmlu.yaml b/lm_eval/tasks/mmlu/default/mmlu.yaml
new file mode 100644
index 0000000000..584de02993
--- /dev/null
+++ b/lm_eval/tasks/mmlu/default/mmlu.yaml
@@ -0,0 +1,6 @@
+group: mmlu
+task:
+  - mmlu_stem
+  - mmlu_other
+  - mmlu_social_sciences
+  - mmlu_humanities
diff --git a/lm_eval/tasks/mmlu/default/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/default/mmlu_abstract_algebra.yaml
index b6d595d33b..bb786cc880 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_abstract_algebra.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_abstract_algebra.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "abstract_algebra"
 "description": "The following are multiple choice questions (with answers) about abstract algebra.\n\n"
+"group": "mmlu_stem"
 "include": "_default_template_yaml"
 "task": "mmlu_abstract_algebra"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/default/mmlu_anatomy.yaml
index 6459cb41c2..22eaa7fdd4 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_anatomy.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_anatomy.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "anatomy"
 "description": "The following are multiple choice questions (with answers) about anatomy.\n\n"
+"group": "mmlu_stem"
 "include": "_default_template_yaml"
 "task": "mmlu_anatomy"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/default/mmlu_astronomy.yaml
index 573dedd751..64f20d204d 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_astronomy.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_astronomy.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "astronomy"
 "description": "The following are multiple choice questions (with answers) about astronomy.\n\n"
+"group": "mmlu_stem"
 "include": "_default_template_yaml"
 "task": "mmlu_astronomy"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/default/mmlu_business_ethics.yaml
index 4b20b795a1..49330917a2 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_business_ethics.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_business_ethics.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "business_ethics"
 "description": "The following are multiple choice questions (with answers) about business ethics.\n\n"
+"group": "mmlu_other"
 "include": "_default_template_yaml"
 "task": "mmlu_business_ethics"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/default/mmlu_clinical_knowledge.yaml
index f758e66db4..547aeccf19 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_clinical_knowledge.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_clinical_knowledge.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "clinical_knowledge"
 "description": "The following are multiple choice questions (with answers) about clinical knowledge.\n\n"
+"group": "mmlu_other"
 "include": "_default_template_yaml"
 "task": "mmlu_clinical_knowledge"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/default/mmlu_college_biology.yaml
index f806900785..6982639747 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_college_biology.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_college_biology.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "college_biology"
 "description": "The following are multiple choice questions (with answers) about college biology.\n\n"
+"group": "mmlu_stem"
 "include": "_default_template_yaml"
 "task": "mmlu_college_biology"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/default/mmlu_college_chemistry.yaml
index e03fbccd41..b91c07f673 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_college_chemistry.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_college_chemistry.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "college_chemistry"
 "description": "The following are multiple choice questions (with answers) about college chemistry.\n\n"
+"group": "mmlu_stem"
 "include": "_default_template_yaml"
 "task": "mmlu_college_chemistry"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/default/mmlu_college_computer_science.yaml
index a9d4a6f288..a89a46aa9a 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_college_computer_science.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_college_computer_science.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "college_computer_science"
 "description": "The following are multiple choice questions (with answers) about college computer science.\n\n"
+"group": "mmlu_stem"
 "include": "_default_template_yaml"
 "task": "mmlu_college_computer_science"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/default/mmlu_college_mathematics.yaml
index f6a86179c7..c452ff9751 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_college_mathematics.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_college_mathematics.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "college_mathematics"
 "description": "The following are multiple choice questions (with answers) about college mathematics.\n\n"
+"group": "mmlu_stem"
 "include": "_default_template_yaml"
 "task": "mmlu_college_mathematics"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/default/mmlu_college_medicine.yaml
index 0ea75fb31f..d696a40d98 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_college_medicine.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_college_medicine.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "college_medicine"
 "description": "The following are multiple choice questions (with answers) about college medicine.\n\n"
+"group": "mmlu_other"
 "include": "_default_template_yaml"
 "task": "mmlu_college_medicine"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/default/mmlu_college_physics.yaml
index 82f13e4030..16046e53f3 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_college_physics.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_college_physics.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "college_physics"
 "description": "The following are multiple choice questions (with answers) about college physics.\n\n"
+"group": "mmlu_stem"
 "include": "_default_template_yaml"
 "task": "mmlu_college_physics"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/default/mmlu_computer_security.yaml
index e9e06de2fb..923967ae41 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_computer_security.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_computer_security.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "computer_security"
 "description": "The following are multiple choice questions (with answers) about computer security.\n\n"
+"group": "mmlu_stem"
 "include": "_default_template_yaml"
 "task": "mmlu_computer_security"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/default/mmlu_conceptual_physics.yaml
index 30ca6efe8c..88096f3b4f 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_conceptual_physics.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_conceptual_physics.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "conceptual_physics"
 "description": "The following are multiple choice questions (with answers) about conceptual physics.\n\n"
+"group": "mmlu_stem"
 "include": "_default_template_yaml"
 "task": "mmlu_conceptual_physics"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/default/mmlu_econometrics.yaml
index 680cc507b1..4c43a5c8fd 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_econometrics.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_econometrics.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "econometrics"
 "description": "The following are multiple choice questions (with answers) about econometrics.\n\n"
+"group": "mmlu_social_sciences"
 "include": "_default_template_yaml"
 "task": "mmlu_econometrics"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/default/mmlu_electrical_engineering.yaml
index 8dd63b337d..27ab482821 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_electrical_engineering.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_electrical_engineering.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "electrical_engineering"
 "description": "The following are multiple choice questions (with answers) about electrical engineering.\n\n"
+"group": "mmlu_stem"
 "include": "_default_template_yaml"
 "task": "mmlu_electrical_engineering"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/default/mmlu_elementary_mathematics.yaml
index 4979ee303c..bd7106e474 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_elementary_mathematics.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_elementary_mathematics.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "elementary_mathematics"
 "description": "The following are multiple choice questions (with answers) about elementary mathematics.\n\n"
+"group": "mmlu_stem"
 "include": "_default_template_yaml"
 "task": "mmlu_elementary_mathematics"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/default/mmlu_formal_logic.yaml
index 9b73509b24..98486ebe7d 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_formal_logic.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_formal_logic.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "formal_logic"
 "description": "The following are multiple choice questions (with answers) about formal logic.\n\n"
+"group": "mmlu_humanities"
 "include": "_default_template_yaml"
 "task": "mmlu_formal_logic"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/default/mmlu_global_facts.yaml
index 8c43a6c9e1..9db3f3d3c7 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_global_facts.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_global_facts.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "global_facts"
 "description": "The following are multiple choice questions (with answers) about global facts.\n\n"
+"group": "mmlu_other"
 "include": "_default_template_yaml"
 "task": "mmlu_global_facts"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_biology.yaml
index 453d30333c..0ed8c0e7f2 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_high_school_biology.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_biology.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "high_school_biology"
 "description": "The following are multiple choice questions (with answers) about high school biology.\n\n"
+"group": "mmlu_stem"
 "include": "_default_template_yaml"
 "task": "mmlu_high_school_biology"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_chemistry.yaml
index 714ee0e538..7aa6037d83 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_high_school_chemistry.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_chemistry.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "high_school_chemistry"
 "description": "The following are multiple choice questions (with answers) about high school chemistry.\n\n"
+"group": "mmlu_stem"
 "include": "_default_template_yaml"
 "task": "mmlu_high_school_chemistry"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_computer_science.yaml
index 9326e25988..9cf212af5a 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_high_school_computer_science.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_computer_science.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "high_school_computer_science"
 "description": "The following are multiple choice questions (with answers) about high school computer science.\n\n"
+"group": "mmlu_stem"
 "include": "_default_template_yaml"
 "task": "mmlu_high_school_computer_science"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_european_history.yaml
index e212cd224b..e9189bd90e 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_high_school_european_history.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_european_history.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "high_school_european_history"
 "description": "The following are multiple choice questions (with answers) about high school european history.\n\n"
+"group": "mmlu_humanities"
 "include": "_default_template_yaml"
 "task": "mmlu_high_school_european_history"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_geography.yaml
index a7fffc2522..7573c8c2e7 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_high_school_geography.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_geography.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "high_school_geography"
 "description": "The following are multiple choice questions (with answers) about high school geography.\n\n"
+"group": "mmlu_social_sciences"
 "include": "_default_template_yaml"
 "task": "mmlu_high_school_geography"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_government_and_politics.yaml
index 7255d60fd7..83d7d49830 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_high_school_government_and_politics.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_government_and_politics.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "high_school_government_and_politics"
 "description": "The following are multiple choice questions (with answers) about high school government and politics.\n\n"
+"group": "mmlu_social_sciences"
 "include": "_default_template_yaml"
 "task": "mmlu_high_school_government_and_politics"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_macroeconomics.yaml
index 29d9ddd73a..4e8269b3a1 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_high_school_macroeconomics.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_macroeconomics.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "high_school_macroeconomics"
 "description": "The following are multiple choice questions (with answers) about high school macroeconomics.\n\n"
+"group": "mmlu_social_sciences"
 "include": "_default_template_yaml"
 "task": "mmlu_high_school_macroeconomics"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_mathematics.yaml
index 035e7a12ec..2b9d321691 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_high_school_mathematics.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_mathematics.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "high_school_mathematics"
 "description": "The following are multiple choice questions (with answers) about high school mathematics.\n\n"
+"group": "mmlu_stem"
 "include": "_default_template_yaml"
 "task": "mmlu_high_school_mathematics"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_microeconomics.yaml
index 72b1c8cf04..3206bfdf35 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_high_school_microeconomics.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_microeconomics.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "high_school_microeconomics"
 "description": "The following are multiple choice questions (with answers) about high school microeconomics.\n\n"
+"group": "mmlu_social_sciences"
 "include": "_default_template_yaml"
 "task": "mmlu_high_school_microeconomics"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_physics.yaml
index ef8f6ca518..27a1e51ae9 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_high_school_physics.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_physics.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "high_school_physics"
 "description": "The following are multiple choice questions (with answers) about high school physics.\n\n"
+"group": "mmlu_stem"
 "include": "_default_template_yaml"
 "task": "mmlu_high_school_physics"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_psychology.yaml
index 5c4cce750a..1e0b1628f5 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_high_school_psychology.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_psychology.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "high_school_psychology"
 "description": "The following are multiple choice questions (with answers) about high school psychology.\n\n"
+"group": "mmlu_social_sciences"
 "include": "_default_template_yaml"
 "task": "mmlu_high_school_psychology"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_statistics.yaml
index 20ed42ecfb..4244ea8f39 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_high_school_statistics.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_statistics.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "high_school_statistics"
 "description": "The following are multiple choice questions (with answers) about high school statistics.\n\n"
+"group": "mmlu_stem"
 "include": "_default_template_yaml"
 "task": "mmlu_high_school_statistics"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_us_history.yaml
index 18cd48da93..a6f085ec57 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_high_school_us_history.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_us_history.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "high_school_us_history"
 "description": "The following are multiple choice questions (with answers) about high school us history.\n\n"
+"group": "mmlu_humanities"
 "include": "_default_template_yaml"
 "task": "mmlu_high_school_us_history"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_world_history.yaml
index b17daac60f..c23d93c822 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_high_school_world_history.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_world_history.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "high_school_world_history"
 "description": "The following are multiple choice questions (with answers) about high school world history.\n\n"
+"group": "mmlu_humanities"
 "include": "_default_template_yaml"
 "task": "mmlu_high_school_world_history"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/default/mmlu_human_aging.yaml
index 080b267606..1478d3ddcf 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_human_aging.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_human_aging.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "human_aging"
 "description": "The following are multiple choice questions (with answers) about human aging.\n\n"
+"group": "mmlu_other"
 "include": "_default_template_yaml"
 "task": "mmlu_human_aging"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/default/mmlu_human_sexuality.yaml
index ca3389fe69..ab56a03502 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_human_sexuality.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_human_sexuality.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "human_sexuality"
 "description": "The following are multiple choice questions (with answers) about human sexuality.\n\n"
+"group": "mmlu_social_sciences"
 "include": "_default_template_yaml"
 "task": "mmlu_human_sexuality"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/default/mmlu_international_law.yaml
index a3d443e01d..7a35270168 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_international_law.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_international_law.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "international_law"
 "description": "The following are multiple choice questions (with answers) about international law.\n\n"
+"group": "mmlu_humanities"
 "include": "_default_template_yaml"
 "task": "mmlu_international_law"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/default/mmlu_jurisprudence.yaml
index 4ba00a2ac6..af29fae31c 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_jurisprudence.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_jurisprudence.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "jurisprudence"
 "description": "The following are multiple choice questions (with answers) about jurisprudence.\n\n"
+"group": "mmlu_humanities"
 "include": "_default_template_yaml"
 "task": "mmlu_jurisprudence"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/default/mmlu_logical_fallacies.yaml
index ea45a4f302..570fd3dd02 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_logical_fallacies.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_logical_fallacies.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "logical_fallacies"
 "description": "The following are multiple choice questions (with answers) about logical fallacies.\n\n"
+"group": "mmlu_humanities"
 "include": "_default_template_yaml"
 "task": "mmlu_logical_fallacies"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/default/mmlu_machine_learning.yaml
index 2ba6d16244..11166e2f24 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_machine_learning.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_machine_learning.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "machine_learning"
 "description": "The following are multiple choice questions (with answers) about machine learning.\n\n"
+"group": "mmlu_stem"
 "include": "_default_template_yaml"
 "task": "mmlu_machine_learning"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_management.yaml b/lm_eval/tasks/mmlu/default/mmlu_management.yaml
index b4ea6da942..745ac7629e 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_management.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_management.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "management"
 "description": "The following are multiple choice questions (with answers) about management.\n\n"
+"group": "mmlu_other"
 "include": "_default_template_yaml"
 "task": "mmlu_management"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/default/mmlu_marketing.yaml
index afa30a0cd4..38401dc85c 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_marketing.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_marketing.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "marketing"
 "description": "The following are multiple choice questions (with answers) about marketing.\n\n"
+"group": "mmlu_other"
 "include": "_default_template_yaml"
 "task": "mmlu_marketing"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/default/mmlu_medical_genetics.yaml
index 92095635c4..2e4fbbd865 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_medical_genetics.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_medical_genetics.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "medical_genetics"
 "description": "The following are multiple choice questions (with answers) about medical genetics.\n\n"
+"group": "mmlu_other"
 "include": "_default_template_yaml"
 "task": "mmlu_medical_genetics"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/default/mmlu_miscellaneous.yaml
index 94ebd1b02e..aa67418026 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_miscellaneous.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_miscellaneous.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "miscellaneous"
 "description": "The following are multiple choice questions (with answers) about miscellaneous.\n\n"
+"group": "mmlu_other"
 "include": "_default_template_yaml"
 "task": "mmlu_miscellaneous"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/default/mmlu_moral_disputes.yaml
index 8bea0a1f22..ac8bbdb9ec 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_moral_disputes.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_moral_disputes.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "moral_disputes"
 "description": "The following are multiple choice questions (with answers) about moral disputes.\n\n"
+"group": "mmlu_humanities"
 "include": "_default_template_yaml"
 "task": "mmlu_moral_disputes"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/default/mmlu_moral_scenarios.yaml
index 71dcc69360..33a249c20c 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_moral_scenarios.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_moral_scenarios.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "moral_scenarios"
 "description": "The following are multiple choice questions (with answers) about moral scenarios.\n\n"
+"group": "mmlu_humanities"
 "include": "_default_template_yaml"
 "task": "mmlu_moral_scenarios"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/default/mmlu_nutrition.yaml
index e6b4cbcd98..44b799cd9a 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_nutrition.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_nutrition.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "nutrition"
 "description": "The following are multiple choice questions (with answers) about nutrition.\n\n"
+"group": "mmlu_other"
 "include": "_default_template_yaml"
 "task": "mmlu_nutrition"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/default/mmlu_philosophy.yaml
index b9a0b2c5d1..5a703cc44c 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_philosophy.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_philosophy.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "philosophy"
 "description": "The following are multiple choice questions (with answers) about philosophy.\n\n"
+"group": "mmlu_humanities"
 "include": "_default_template_yaml"
 "task": "mmlu_philosophy"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/default/mmlu_prehistory.yaml
index 7f71bd54f5..dc8e65b8b3 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_prehistory.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_prehistory.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "prehistory"
 "description": "The following are multiple choice questions (with answers) about prehistory.\n\n"
+"group": "mmlu_humanities"
 "include": "_default_template_yaml"
 "task": "mmlu_prehistory"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/default/mmlu_professional_accounting.yaml
index 94ca6e6e93..c59ccffd22 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_professional_accounting.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_professional_accounting.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "professional_accounting"
 "description": "The following are multiple choice questions (with answers) about professional accounting.\n\n"
+"group": "mmlu_other"
 "include": "_default_template_yaml"
 "task": "mmlu_professional_accounting"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/default/mmlu_professional_law.yaml
index 074c34e699..46a3ebbde2 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_professional_law.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_professional_law.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "professional_law"
 "description": "The following are multiple choice questions (with answers) about professional law.\n\n"
+"group": "mmlu_humanities"
 "include": "_default_template_yaml"
 "task": "mmlu_professional_law"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/default/mmlu_professional_medicine.yaml
index 2f99c316ec..fe52278dfe 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_professional_medicine.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_professional_medicine.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "professional_medicine"
 "description": "The following are multiple choice questions (with answers) about professional medicine.\n\n"
+"group": "mmlu_other"
 "include": "_default_template_yaml"
 "task": "mmlu_professional_medicine"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/default/mmlu_professional_psychology.yaml
index 015658487f..ff7bb1f72d 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_professional_psychology.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_professional_psychology.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "professional_psychology"
 "description": "The following are multiple choice questions (with answers) about professional psychology.\n\n"
+"group": "mmlu_social_sciences"
 "include": "_default_template_yaml"
 "task": "mmlu_professional_psychology"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/default/mmlu_public_relations.yaml
index 0d46c66ec8..290a85f5ac 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_public_relations.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_public_relations.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "public_relations"
 "description": "The following are multiple choice questions (with answers) about public relations.\n\n"
+"group": "mmlu_social_sciences"
 "include": "_default_template_yaml"
 "task": "mmlu_public_relations"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/default/mmlu_security_studies.yaml
index f30dffdefc..d1a4187194 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_security_studies.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_security_studies.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "security_studies"
 "description": "The following are multiple choice questions (with answers) about security studies.\n\n"
+"group": "mmlu_social_sciences"
 "include": "_default_template_yaml"
 "task": "mmlu_security_studies"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/default/mmlu_sociology.yaml
index c36bd40370..be1e46f5ad 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_sociology.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_sociology.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "sociology"
 "description": "The following are multiple choice questions (with answers) about sociology.\n\n"
+"group": "mmlu_social_sciences"
 "include": "_default_template_yaml"
 "task": "mmlu_sociology"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/default/mmlu_us_foreign_policy.yaml
index fe8c68d8fb..f94e8bc0e9 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_us_foreign_policy.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_us_foreign_policy.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "us_foreign_policy"
 "description": "The following are multiple choice questions (with answers) about us foreign policy.\n\n"
+"group": "mmlu_social_sciences"
 "include": "_default_template_yaml"
 "task": "mmlu_us_foreign_policy"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_virology.yaml b/lm_eval/tasks/mmlu/default/mmlu_virology.yaml
index 4cbd095998..4fdc1bf6b5 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_virology.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_virology.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "virology"
 "description": "The following are multiple choice questions (with answers) about virology.\n\n"
+"group": "mmlu_other"
 "include": "_default_template_yaml"
 "task": "mmlu_virology"
diff --git a/lm_eval/tasks/mmlu/default/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/default/mmlu_world_religions.yaml
index 375efbae5d..870ea78df7 100644
--- a/lm_eval/tasks/mmlu/default/mmlu_world_religions.yaml
+++ b/lm_eval/tasks/mmlu/default/mmlu_world_religions.yaml
@@ -1,4 +1,5 @@
 "dataset_name": "world_religions"
 "description": "The following are multiple choice questions (with answers) about world religions.\n\n"
+"group": "mmlu_humanities"
 "include": "_default_template_yaml"
 "task": "mmlu_world_religions"


Code review comments:
diff_hunk: @@ -481,18 +466,37 @@ def evaluate(
                     results[task_name][metric + "_stderr" + "," + key] = stderr(items)
 
         if bool(results):
-            for task_or_group in results.keys():
-                for metric in results[task_or_group].keys():
-                    if type(results[task_or_group][metric]) == list:
-                        if "(sample agg)" in metric:
-                            results[task_or_group][metric] = sample_agg_fn[
-                                task_or_group
-                            ][metric](results[task_or_group][metric])
+
+            for group, task_list in reversed(task_hierarchy.items()):
+                versions[group] = "N/A"
+                task_score_dict = {}
+                total_size = 0
+                for task in task_list:
+                    metrics = results[task]
+
+                    current_size = metrics.pop("samples")
+                    # if "size" in metrics:
+                    #     current_size = metrics.pop("size")
+                    # else:
+                    #     current_size = 1
+
+                    for metric in [key for key in metrics.keys() if "_stderr" not in key]:
+
+                        stderr = "_stderr,".join(metric.split(","))
+                        stderr_score = results[task][stderr]
+                        metric_score = results[task][metric]
+
+                        if metric in results[group]:
+                            results[group][metric] = (results[group][metric]*total_size + metric_score*current_size)/(total_size+current_size)
+                            # $$s_z^2 = \frac{(n-1) s_x^2 + (m-1) s_y^2}{n+m-1} + \frac{nm(\bar x - \bar y)^2}{(n+m)(n+m-1)}.$$
+                            results[group][stderr] = ((total_size-1)*results[group][stderr]+(current_size-1)*stderr_score)/(total_size + current_size - 1) \
+                                                        + total_size*current_size/((total_size+current_size)*(total_size+current_size-1))*(results[group][metric] - metric_score)**2
path: lm_eval/evaluator.py
text: @author:
@user1 added the weighted averaging of subtasks and combining standard error as in 
![image](https://github.com/EleutherAI/lm-evaluation-harness/assets/5774558/5664cacd-e1f8-4ff5-911c-b68b1baf9f7f)

Does the final value need to be square-rooted?

@user1:
Yes, s_x is the standard deviation. s^2_x is the variance.