Repository: EleutherAI/lm-evaluation-harness
Issue: The number of samples in MMLU
Root cause: Documentation Deficiency

I obtained the following raw data through 'hendrycksTest-*' as below:

{"results": {"hendrycksTest-professional_psychology": {"acc": 0.45098039215686275, "acc_stderr": 0.020130388312904528, "acc_norm": 0.45098039215686275, "acc_norm_stderr": 0.020130388312904528}, "hendrycksTest-college_mathematics": {"acc": 0.33, "acc_stderr": 0.04725815626252606, "acc_norm": 0.33, "acc_norm_stderr": 0.04725815626252606}, "hendrycksTest-high_school_psychology": {"acc": 0.6201834862385321, "acc_stderr": 0.020808825617866244, "acc_norm": 0.6201834862385321, "acc_norm_stderr": 0.020808825617866244}, "hendrycksTest-professional_accounting": {"acc": 0.3475177304964539, "acc_stderr": 0.028406627809590954, "acc_norm": 0.3475177304964539, "acc_norm_stderr": 0.028406627809590954}, "hendrycksTest-management": {"acc": 0.5436893203883495, "acc_stderr": 0.04931801994220416, "acc_norm": 0.5436893203883495, "acc_norm_stderr": 0.04931801994220416}, "hendrycksTest-college_chemistry": {"acc": 0.38, "acc_stderr": 0.04878317312145632, "acc_norm": 0.38, "acc_norm_stderr": 0.04878317312145632}, "hendrycksTest-human_aging": {"acc": 0.5605381165919282, "acc_stderr": 0.03331092511038179, "acc_norm": 0.5605381165919282, "acc_norm_stderr": 0.03331092511038179}, "hendrycksTest-college_computer_science": {"acc": 0.35, "acc_stderr": 0.047937248544110196, "acc_norm": 0.35, "acc_norm_stderr": 0.047937248544110196}, "hendrycksTest-college_biology": {"acc": 0.4583333333333333, "acc_stderr": 0.04166666666666665, "acc_norm": 0.4583333333333333, "acc_norm_stderr": 0.04166666666666665}, "hendrycksTest-international_law": {"acc": 0.6528925619834711, "acc_stderr": 0.04345724570292534, "acc_norm": 0.6528925619834711, "acc_norm_stderr": 0.04345724570292534}, "hendrycksTest-public_relations": {"acc": 0.5363636363636364, "acc_stderr": 0.04776449162396197, "acc_norm": 0.5363636363636364, "acc_norm_stderr": 0.04776449162396197}, "hendrycksTest-high_school_biology": {"acc": 0.49032258064516127, "acc_stderr": 0.028438677998909558, "acc_norm": 0.49032258064516127, "acc_norm_stderr": 0.028438677998909558}, "hendrycksTest-virology": {"acc": 0.40963855421686746, "acc_stderr": 0.03828401115079022, "acc_norm": 0.40963855421686746, "acc_norm_stderr": 0.03828401115079022}, "hendrycksTest-professional_medicine": {"acc": 0.5036764705882353, "acc_stderr": 0.030372015885428195, "acc_norm": 0.5036764705882353, "acc_norm_stderr": 0.030372015885428195}, "hendrycksTest-astronomy": {"acc": 0.40131578947368424, "acc_stderr": 0.03988903703336284, "acc_norm": 0.40131578947368424, "acc_norm_stderr": 0.03988903703336284}, "hendrycksTest-high_school_mathematics": {"acc": 0.2962962962962963, "acc_stderr": 0.02784081149587192, "acc_norm": 0.2962962962962963, "acc_norm_stderr": 0.02784081149587192}, "hendrycksTest-prehistory": {"acc": 0.5185185185185185, "acc_stderr": 0.02780165621232366, "acc_norm": 0.5185185185185185, "acc_norm_stderr": 0.02780165621232366}, "hendrycksTest-clinical_knowledge": {"acc": 0.45660377358490567, "acc_stderr": 0.030656748696739438, "acc_norm": 0.45660377358490567, "acc_norm_stderr": 0.030656748696739438}, "hendrycksTest-high_school_statistics": {"acc": 0.2777777777777778, "acc_stderr": 0.030546745264953178, "acc_norm": 0.2777777777777778, "acc_norm_stderr": 0.030546745264953178}, "hendrycksTest-computer_security": {"acc": 0.61, "acc_stderr": 0.04902071300001975, "acc_norm": 0.61, "acc_norm_stderr": 0.04902071300001975}, "hendrycksTest-miscellaneous": {"acc": 0.6411238825031929, "acc_stderr": 0.017152991797501342, "acc_norm": 0.6411238825031929, "acc_norm_stderr": 0.017152991797501342}, "hendrycksTest-college_medicine": {"acc": 0.41040462427745666, "acc_stderr": 0.03750757044895537, "acc_norm": 0.41040462427745666, "acc_norm_stderr": 0.03750757044895537}, "hendrycksTest-high_school_geography": {"acc": 0.4797979797979798, "acc_stderr": 0.035594435655639196, "acc_norm": 0.4797979797979798, "acc_norm_stderr": 0.035594435655639196}, "hendrycksTest-business_ethics": {"acc": 0.48, "acc_stderr": 0.050211673156867795, "acc_norm": 0.48, "acc_norm_stderr": 0.050211673156867795}, "hendrycksTest-high_school_microeconomics": {"acc": 0.42436974789915966, "acc_stderr": 0.03210479051015776, "acc_norm": 0.42436974789915966, "acc_norm_stderr": 0.03210479051015776}, "hendrycksTest-high_school_european_history": {"acc": 0.5878787878787879, "acc_stderr": 0.03843566993588717, "acc_norm": 0.5878787878787879, "acc_norm_stderr": 0.03843566993588717}, "hendrycksTest-professional_law": {"acc": 0.3767926988265971, "acc_stderr": 0.012376459593894402, "acc_norm": 0.3767926988265971, "acc_norm_stderr": 0.012376459593894402}, "hendrycksTest-sociology": {"acc": 0.6268656716417911, "acc_stderr": 0.03419832608176008, "acc_norm": 0.6268656716417911, "acc_norm_stderr": 0.03419832608176008}, "hendrycksTest-jurisprudence": {"acc": 0.5277777777777778, "acc_stderr": 0.048262172941398944, "acc_norm": 0.5277777777777778, "acc_norm_stderr": 0.048262172941398944}, "hendrycksTest-abstract_algebra": {"acc": 0.31, "acc_stderr": 0.04648231987117316, "acc_norm": 0.31, "acc_norm_stderr": 0.04648231987117316}, "hendrycksTest-human_sexuality": {"acc": 0.5572519083969466, "acc_stderr": 0.043564472026650695, "acc_norm": 0.5572519083969466, "acc_norm_stderr": 0.043564472026650695}, "hendrycksTest-world_religions": {"acc": 0.7134502923976608, "acc_stderr": 0.03467826685703826, "acc_norm": 0.7134502923976608, "acc_norm_stderr": 0.03467826685703826}, "hendrycksTest-high_school_macroeconomics": {"acc": 0.4461538461538462, "acc_stderr": 0.02520357177302833, "acc_norm": 0.4461538461538462, "acc_norm_stderr": 0.02520357177302833}, "hendrycksTest-econometrics": {"acc": 0.2894736842105263, "acc_stderr": 0.042663394431593935, "acc_norm": 0.2894736842105263, "acc_norm_stderr": 0.042663394431593935}, "hendrycksTest-marketing": {"acc": 0.7008547008547008, "acc_stderr": 0.029996951858349472, "acc_norm": 0.7008547008547008, "acc_norm_stderr": 0.029996951858349472}, "hendrycksTest-medical_genetics": {"acc": 0.51, "acc_stderr": 0.05024183937956913, "acc_norm": 0.51, "acc_norm_stderr": 0.05024183937956913}, "hendrycksTest-formal_logic": {"acc": 0.2857142857142857, "acc_stderr": 0.0404061017820884, "acc_norm": 0.2857142857142857, "acc_norm_stderr": 0.0404061017820884}, "hendrycksTest-high_school_world_history": {"acc": 0.6244725738396625, "acc_stderr": 0.03152256243091156, "acc_norm": 0.6244725738396625, "acc_norm_stderr": 0.03152256243091156}, "hendrycksTest-high_school_us_history": {"acc": 0.5294117647058824, "acc_stderr": 0.03503235296367993, "acc_norm": 0.5294117647058824, "acc_norm_stderr": 0.03503235296367993}, "hendrycksTest-high_school_physics": {"acc": 0.32450331125827814, "acc_stderr": 0.03822746937658753, "acc_norm": 0.32450331125827814, "acc_norm_stderr": 0.03822746937658753}, "hendrycksTest-high_school_chemistry": {"acc": 0.35960591133004927, "acc_stderr": 0.03376458246509566, "acc_norm": 0.35960591133004927, "acc_norm_stderr": 0.03376458246509566}, "hendrycksTest-electrical_engineering": {"acc": 0.45517241379310347, "acc_stderr": 0.04149886942192117, "acc_norm": 0.45517241379310347, "acc_norm_stderr": 0.04149886942192117}, "hendrycksTest-moral_scenarios": {"acc": 0.23910614525139665, "acc_stderr": 0.014265554192331144, "acc_norm": 0.23910614525139665, "acc_norm_stderr": 0.014265554192331144}, "hendrycksTest-conceptual_physics": {"acc": 0.425531914893617, "acc_stderr": 0.03232146916224468, "acc_norm": 0.425531914893617, "acc_norm_stderr": 0.03232146916224468}, "hendrycksTest-security_studies": {"acc": 0.4857142857142857, "acc_stderr": 0.03199615232806287, "acc_norm": 0.4857142857142857, "acc_norm_stderr": 0.03199615232806287}, "hendrycksTest-elementary_mathematics": {"acc": 0.24867724867724866, "acc_stderr": 0.02226181769240017, "acc_norm": 0.24867724867724866, "acc_norm_stderr": 0.02226181769240017}, "hendrycksTest-logical_fallacies": {"acc": 0.5214723926380368, "acc_stderr": 0.03924746876751129, "acc_norm": 0.5214723926380368, "acc_norm_stderr": 0.03924746876751129}, "hendrycksTest-machine_learning": {"acc": 0.375, "acc_stderr": 0.04595091388086298, "acc_norm": 0.375, "acc_norm_stderr": 0.04595091388086298}, "hendrycksTest-college_physics": {"acc": 0.23529411764705882, "acc_stderr": 0.04220773659171453, "acc_norm": 0.23529411764705882, "acc_norm_stderr": 0.04220773659171453}, "hendrycksTest-moral_disputes": {"acc": 0.49421965317919075, "acc_stderr": 0.026917296179149116, "acc_norm": 0.49421965317919075, "acc_norm_stderr": 0.026917296179149116}, "hendrycksTest-high_school_computer_science": {"acc": 0.39, "acc_stderr": 0.04902071300001974, "acc_norm": 0.39, "acc_norm_stderr": 0.04902071300001974}, "hendrycksTest-global_facts": {"acc": 0.32, "acc_stderr": 0.04688261722621503, "acc_norm": 0.32, "acc_norm_stderr": 0.04688261722621503}, "hendrycksTest-anatomy": {"acc": 0.4666666666666667, "acc_stderr": 0.043097329010363554, "acc_norm": 0.4666666666666667, "acc_norm_stderr": 0.043097329010363554}, "hendrycksTest-philosophy": {"acc": 0.5884244372990354, "acc_stderr": 0.027950481494401266, "acc_norm": 0.5884244372990354, "acc_norm_stderr": 0.027950481494401266}, "hendrycksTest-high_school_government_and_politics": {"acc": 0.6787564766839378, "acc_stderr": 0.033699508685490674, "acc_norm": 0.6787564766839378, "acc_norm_stderr": 0.033699508685490674}, "hendrycksTest-us_foreign_policy": {"acc": 0.64, "acc_stderr": 0.04824181513244218, "acc_norm": 0.64, "acc_norm_stderr": 0.04824181513244218}, "hendrycksTest-nutrition": {"acc": 0.4869281045751634, "acc_stderr": 0.028620130800700246, "acc_norm": 0.4869281045751634, "acc_norm_stderr": 0.028620130800700246}}, "versions": {"hendrycksTest-professional_psychology": 1, "hendrycksTest-college_mathematics": 1, "hendrycksTest-high_school_psychology": 1, "hendrycksTest-professional_accounting": 1, "hendrycksTest-management": 1, "hendrycksTest-college_chemistry": 1, "hendrycksTest-human_aging": 1, "hendrycksTest-college_computer_science": 1, "hendrycksTest-college_biology": 1, "hendrycksTest-international_law": 1, "hendrycksTest-public_relations": 1, "hendrycksTest-high_school_biology": 1, "hendrycksTest-virology": 1, "hendrycksTest-professional_medicine": 1, "hendrycksTest-astronomy": 1, "hendrycksTest-high_school_mathematics": 1, "hendrycksTest-prehistory": 1, "hendrycksTest-clinical_knowledge": 1, "hendrycksTest-high_school_statistics": 1, "hendrycksTest-computer_security": 1, "hendrycksTest-miscellaneous": 1, "hendrycksTest-college_medicine": 1, "hendrycksTest-high_school_geography": 1, "hendrycksTest-business_ethics": 1, "hendrycksTest-high_school_microeconomics": 1, "hendrycksTest-high_school_european_history": 1, "hendrycksTest-professional_law": 1, "hendrycksTest-sociology": 1, "hendrycksTest-jurisprudence": 1, "hendrycksTest-abstract_algebra": 1, "hendrycksTest-human_sexuality": 1, "hendrycksTest-world_religions": 1, "hendrycksTest-high_school_macroeconomics": 1, "hendrycksTest-econometrics": 1, "hendrycksTest-marketing": 1, "hendrycksTest-medical_genetics": 1, "hendrycksTest-formal_logic": 1, "hendrycksTest-high_school_world_history": 1, "hendrycksTest-high_school_us_history": 1, "hendrycksTest-high_school_physics": 1, "hendrycksTest-high_school_chemistry": 1, "hendrycksTest-electrical_engineering": 1, "hendrycksTest-moral_scenarios": 1, "hendrycksTest-conceptual_physics": 1, "hendrycksTest-security_studies": 1, "hendrycksTest-elementary_mathematics": 1, "hendrycksTest-logical_fallacies": 1, "hendrycksTest-machine_learning": 1, "hendrycksTest-college_physics": 1, "hendrycksTest-moral_disputes": 1, "hendrycksTest-high_school_computer_science": 1, "hendrycksTest-global_facts": 1, "hendrycksTest-anatomy": 1, "hendrycksTest-philosophy": 1, "hendrycksTest-high_school_government_and_politics": 1, "hendrycksTest-us_foreign_policy": 1, "hendrycksTest-nutrition": 1}, "config": {"model": "Llama-2-7b-hf", "batch_size": 1, "device": "cuda:0", "num_fewshot": 5, "limit": null, "bootstrap_iters": 100000, "no_cache": true}}

Here, I want to calculate the average for four subjects (STEM, Humanities, Social Science, Others) and the overall average. From looking at the code, it seems that to compute the average sum form these data, I need to know the number of samples for each task. Is it possible to check the n_samples for each task?