Upload README.md with huggingface_hub

README.md

@@ -2097,6 +2097,3258 @@ model-index:
           Vulnerability Tsx async abort:      Not affected
 
 
+          Versions of relevant libraries:
+
+          [pip3] numpy==1.24.1
+
+          [pip3] torch==2.1.2
+
+          [pip3] torchaudio==2.0.2+cu118
+
+          [pip3] torchvision==0.15.2+cu118
+
+          [pip3] triton==2.1.0
+
+          [conda] Could not collect'
+        transformers_version: 4.42.4
+  - task:
+      type: mmlu
+    dataset:
+      name: mmlu
+      type: public-dataset
+    metrics:
+    - type: acc
+      value: '0.595'
+      args:
+        results:
+          mmlu:
+            acc,none: 0.5817547357926222
+            acc_stderr,none: 0.0039373066351597085
+            alias: mmlu
+          mmlu_humanities:
+            alias: ' - humanities'
+            acc,none: 0.5247608926673751
+            acc_stderr,none: 0.006839745323517898
+          mmlu_formal_logic:
+            alias: '  - formal_logic'
+            acc,none: 0.35714285714285715
+            acc_stderr,none: 0.042857142857142816
+          mmlu_high_school_european_history:
+            alias: '  - high_school_european_history'
+            acc,none: 0.696969696969697
+            acc_stderr,none: 0.035886248000917075
+          mmlu_high_school_us_history:
+            alias: '  - high_school_us_history'
+            acc,none: 0.7745098039215687
+            acc_stderr,none: 0.02933116229425172
+          mmlu_high_school_world_history:
+            alias: '  - high_school_world_history'
+            acc,none: 0.7974683544303798
+            acc_stderr,none: 0.026160568246601453
+          mmlu_international_law:
+            alias: '  - international_law'
+            acc,none: 0.7107438016528925
+            acc_stderr,none: 0.041391127276354626
+          mmlu_jurisprudence:
+            alias: '  - jurisprudence'
+            acc,none: 0.7037037037037037
+            acc_stderr,none: 0.04414343666854932
+          mmlu_logical_fallacies:
+            alias: '  - logical_fallacies'
+            acc,none: 0.7055214723926381
+            acc_stderr,none: 0.03581165790474082
+          mmlu_moral_disputes:
+            alias: '  - moral_disputes'
+            acc,none: 0.615606936416185
+            acc_stderr,none: 0.026189666966272028
+          mmlu_moral_scenarios:
+            alias: '  - moral_scenarios'
+            acc,none: 0.2837988826815642
+            acc_stderr,none: 0.01507835897075178
+          mmlu_philosophy:
+            alias: '  - philosophy'
+            acc,none: 0.6591639871382636
+            acc_stderr,none: 0.02692084126077615
+          mmlu_prehistory:
+            alias: '  - prehistory'
+            acc,none: 0.6666666666666666
+            acc_stderr,none: 0.026229649178821163
+          mmlu_professional_law:
+            alias: '  - professional_law'
+            acc,none: 0.4348109517601043
+            acc_stderr,none: 0.012661233805616292
+          mmlu_world_religions:
+            alias: '  - world_religions'
+            acc,none: 0.7602339181286549
+            acc_stderr,none: 0.03274485211946956
+          mmlu_other:
+            alias: ' - other'
+            acc,none: 0.6678467975539105
+            acc_stderr,none: 0.008199669520892388
+          mmlu_business_ethics:
+            alias: '  - business_ethics'
+            acc,none: 0.6
+            acc_stderr,none: 0.049236596391733084
+          mmlu_clinical_knowledge:
+            alias: '  - clinical_knowledge'
+            acc,none: 0.6943396226415094
+            acc_stderr,none: 0.028353298073322663
+          mmlu_college_medicine:
+            alias: '  - college_medicine'
+            acc,none: 0.5780346820809249
+            acc_stderr,none: 0.03765746693865151
+          mmlu_global_facts:
+            alias: '  - global_facts'
+            acc,none: 0.41
+            acc_stderr,none: 0.04943110704237102
+          mmlu_human_aging:
+            alias: '  - human_aging'
+            acc,none: 0.6681614349775785
+            acc_stderr,none: 0.03160295143776679
+          mmlu_management:
+            alias: '  - management'
+            acc,none: 0.7766990291262136
+            acc_stderr,none: 0.04123553189891431
+          mmlu_marketing:
+            alias: '  - marketing'
+            acc,none: 0.8076923076923077
+            acc_stderr,none: 0.025819233256483706
+          mmlu_medical_genetics:
+            alias: '  - medical_genetics'
+            acc,none: 0.7
+            acc_stderr,none: 0.046056618647183814
+          mmlu_miscellaneous:
+            alias: '  - miscellaneous'
+            acc,none: 0.7879948914431673
+            acc_stderr,none: 0.014616099385833688
+          mmlu_nutrition:
+            alias: '  - nutrition'
+            acc,none: 0.6503267973856209
+            acc_stderr,none: 0.027305308076274695
+          mmlu_professional_accounting:
+            alias: '  - professional_accounting'
+            acc,none: 0.46808510638297873
+            acc_stderr,none: 0.02976667507587387
+          mmlu_professional_medicine:
+            alias: '  - professional_medicine'
+            acc,none: 0.6360294117647058
+            acc_stderr,none: 0.029227192460032032
+          mmlu_virology:
+            alias: '  - virology'
+            acc,none: 0.4879518072289157
+            acc_stderr,none: 0.038913644958358196
+          mmlu_social_sciences:
+            alias: ' - social_sciences'
+            acc,none: 0.6785830354241144
+            acc_stderr,none: 0.00821975248078532
+          mmlu_econometrics:
+            alias: '  - econometrics'
+            acc,none: 0.43859649122807015
+            acc_stderr,none: 0.04668000738510455
+          mmlu_high_school_geography:
+            alias: '  - high_school_geography'
+            acc,none: 0.6868686868686869
+            acc_stderr,none: 0.03304205087813652
+          mmlu_high_school_government_and_politics:
+            alias: '  - high_school_government_and_politics'
+            acc,none: 0.8031088082901554
+            acc_stderr,none: 0.028697873971860702
+          mmlu_high_school_macroeconomics:
+            alias: '  - high_school_macroeconomics'
+            acc,none: 0.5153846153846153
+            acc_stderr,none: 0.025339003010106515
+          mmlu_high_school_microeconomics:
+            alias: '  - high_school_microeconomics'
+            acc,none: 0.6512605042016807
+            acc_stderr,none: 0.030956636328566548
+          mmlu_high_school_psychology:
+            alias: '  - high_school_psychology'
+            acc,none: 0.7669724770642202
+            acc_stderr,none: 0.0181256691808615
+          mmlu_human_sexuality:
+            alias: '  - human_sexuality'
+            acc,none: 0.7099236641221374
+            acc_stderr,none: 0.03980066246467765
+          mmlu_professional_psychology:
+            alias: '  - professional_psychology'
+            acc,none: 0.619281045751634
+            acc_stderr,none: 0.019643801557924806
+          mmlu_public_relations:
+            alias: '  - public_relations'
+            acc,none: 0.6727272727272727
+            acc_stderr,none: 0.0449429086625209
+          mmlu_security_studies:
+            alias: '  - security_studies'
+            acc,none: 0.726530612244898
+            acc_stderr,none: 0.028535560337128445
+          mmlu_sociology:
+            alias: '  - sociology'
+            acc,none: 0.8208955223880597
+            acc_stderr,none: 0.027113286753111837
+          mmlu_us_foreign_policy:
+            alias: '  - us_foreign_policy'
+            acc,none: 0.84
+            acc_stderr,none: 0.03684529491774708
+          mmlu_stem:
+            alias: ' - stem'
+            acc,none: 0.4874722486520774
+            acc_stderr,none: 0.008583025767956746
+          mmlu_abstract_algebra:
+            alias: '  - abstract_algebra'
+            acc,none: 0.31
+            acc_stderr,none: 0.04648231987117316
+          mmlu_anatomy:
+            alias: '  - anatomy'
+            acc,none: 0.5481481481481482
+            acc_stderr,none: 0.04299268905480864
+          mmlu_astronomy:
+            alias: '  - astronomy'
+            acc,none: 0.6118421052631579
+            acc_stderr,none: 0.03965842097512744
+          mmlu_college_biology:
+            alias: '  - college_biology'
+            acc,none: 0.7569444444444444
+            acc_stderr,none: 0.03586879280080341
+          mmlu_college_chemistry:
+            alias: '  - college_chemistry'
+            acc,none: 0.38
+            acc_stderr,none: 0.04878317312145633
+          mmlu_college_computer_science:
+            alias: '  - college_computer_science'
+            acc,none: 0.4
+            acc_stderr,none: 0.049236596391733084
+          mmlu_college_mathematics:
+            alias: '  - college_mathematics'
+            acc,none: 0.35
+            acc_stderr,none: 0.04793724854411019
+          mmlu_college_physics:
+            alias: '  - college_physics'
+            acc,none: 0.37254901960784315
+            acc_stderr,none: 0.04810840148082633
+          mmlu_computer_security:
+            alias: '  - computer_security'
+            acc,none: 0.67
+            acc_stderr,none: 0.04725815626252609
+          mmlu_conceptual_physics:
+            alias: '  - conceptual_physics'
+            acc,none: 0.5234042553191489
+            acc_stderr,none: 0.032650194750335815
+          mmlu_electrical_engineering:
+            alias: '  - electrical_engineering'
+            acc,none: 0.5172413793103449
+            acc_stderr,none: 0.04164188720169375
+          mmlu_elementary_mathematics:
+            alias: '  - elementary_mathematics'
+            acc,none: 0.373015873015873
+            acc_stderr,none: 0.02490699045899257
+          mmlu_high_school_biology:
+            alias: '  - high_school_biology'
+            acc,none: 0.7225806451612903
+            acc_stderr,none: 0.02547019683590005
+          mmlu_high_school_chemistry:
+            alias: '  - high_school_chemistry'
+            acc,none: 0.4630541871921182
+            acc_stderr,none: 0.035083705204426656
+          mmlu_high_school_computer_science:
+            alias: '  - high_school_computer_science'
+            acc,none: 0.62
+            acc_stderr,none: 0.048783173121456316
+          mmlu_high_school_mathematics:
+            alias: '  - high_school_mathematics'
+            acc,none: 0.32222222222222224
+            acc_stderr,none: 0.028493465091028593
+          mmlu_high_school_physics:
+            alias: '  - high_school_physics'
+            acc,none: 0.3576158940397351
+            acc_stderr,none: 0.03913453431177258
+          mmlu_high_school_statistics:
+            alias: '  - high_school_statistics'
+            acc,none: 0.4398148148148148
+            acc_stderr,none: 0.033851779760448106
+          mmlu_machine_learning:
+            alias: '  - machine_learning'
+            acc,none: 0.5089285714285714
+            acc_stderr,none: 0.04745033255489123
+        groups:
+          mmlu:
+            acc,none: 0.5817547357926222
+            acc_stderr,none: 0.0039373066351597085
+            alias: mmlu
+          mmlu_humanities:
+            alias: ' - humanities'
+            acc,none: 0.5247608926673751
+            acc_stderr,none: 0.006839745323517898
+          mmlu_other:
+            alias: ' - other'
+            acc,none: 0.6678467975539105
+            acc_stderr,none: 0.008199669520892388
+          mmlu_social_sciences:
+            alias: ' - social_sciences'
+            acc,none: 0.6785830354241144
+            acc_stderr,none: 0.00821975248078532
+          mmlu_stem:
+            alias: ' - stem'
+            acc,none: 0.4874722486520774
+            acc_stderr,none: 0.008583025767956746
+        group_subtasks:
+          mmlu_stem:
+          - mmlu_college_computer_science
+          - mmlu_college_chemistry
+          - mmlu_college_biology
+          - mmlu_astronomy
+          - mmlu_anatomy
+          - mmlu_abstract_algebra
+          - mmlu_machine_learning
+          - mmlu_high_school_statistics
+          - mmlu_high_school_physics
+          - mmlu_high_school_mathematics
+          - mmlu_high_school_computer_science
+          - mmlu_high_school_chemistry
+          - mmlu_high_school_biology
+          - mmlu_elementary_mathematics
+          - mmlu_electrical_engineering
+          - mmlu_conceptual_physics
+          - mmlu_computer_security
+          - mmlu_college_physics
+          - mmlu_college_mathematics
+          mmlu_other:
+          - mmlu_clinical_knowledge
+          - mmlu_business_ethics
+          - mmlu_virology
+          - mmlu_professional_medicine
+          - mmlu_professional_accounting
+          - mmlu_nutrition
+          - mmlu_miscellaneous
+          - mmlu_medical_genetics
+          - mmlu_marketing
+          - mmlu_management
+          - mmlu_human_aging
+          - mmlu_global_facts
+          - mmlu_college_medicine
+          mmlu_social_sciences:
+          - mmlu_us_foreign_policy
+          - mmlu_sociology
+          - mmlu_security_studies
+          - mmlu_public_relations
+          - mmlu_professional_psychology
+          - mmlu_human_sexuality
+          - mmlu_high_school_psychology
+          - mmlu_high_school_microeconomics
+          - mmlu_high_school_macroeconomics
+          - mmlu_high_school_government_and_politics
+          - mmlu_high_school_geography
+          - mmlu_econometrics
+          mmlu_humanities:
+          - mmlu_world_religions
+          - mmlu_professional_law
+          - mmlu_prehistory
+          - mmlu_philosophy
+          - mmlu_moral_scenarios
+          - mmlu_moral_disputes
+          - mmlu_logical_fallacies
+          - mmlu_jurisprudence
+          - mmlu_international_law
+          - mmlu_high_school_world_history
+          - mmlu_high_school_us_history
+          - mmlu_high_school_european_history
+          - mmlu_formal_logic
+          mmlu:
+          - mmlu_humanities
+          - mmlu_social_sciences
+          - mmlu_other
+          - mmlu_stem
+        configs:
+          mmlu_abstract_algebra:
+            task: mmlu_abstract_algebra
+            task_alias: abstract_algebra
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: abstract_algebra
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about abstract algebra.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_anatomy:
+            task: mmlu_anatomy
+            task_alias: anatomy
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: anatomy
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about anatomy.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_astronomy:
+            task: mmlu_astronomy
+            task_alias: astronomy
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: astronomy
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about astronomy.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_business_ethics:
+            task: mmlu_business_ethics
+            task_alias: business_ethics
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: business_ethics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about business ethics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_clinical_knowledge:
+            task: mmlu_clinical_knowledge
+            task_alias: clinical_knowledge
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: clinical_knowledge
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about clinical knowledge.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_college_biology:
+            task: mmlu_college_biology
+            task_alias: college_biology
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: college_biology
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about college biology.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_college_chemistry:
+            task: mmlu_college_chemistry
+            task_alias: college_chemistry
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: college_chemistry
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about college chemistry.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_college_computer_science:
+            task: mmlu_college_computer_science
+            task_alias: college_computer_science
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: college_computer_science
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about college computer science.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_college_mathematics:
+            task: mmlu_college_mathematics
+            task_alias: college_mathematics
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: college_mathematics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about college mathematics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_college_medicine:
+            task: mmlu_college_medicine
+            task_alias: college_medicine
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: college_medicine
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about college medicine.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_college_physics:
+            task: mmlu_college_physics
+            task_alias: college_physics
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: college_physics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about college physics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_computer_security:
+            task: mmlu_computer_security
+            task_alias: computer_security
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: computer_security
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about computer security.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_conceptual_physics:
+            task: mmlu_conceptual_physics
+            task_alias: conceptual_physics
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: conceptual_physics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about conceptual physics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_econometrics:
+            task: mmlu_econometrics
+            task_alias: econometrics
+            group: mmlu_social_sciences
+            group_alias: social_sciences
+            dataset_path: hails/mmlu_no_train
+            dataset_name: econometrics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about econometrics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_electrical_engineering:
+            task: mmlu_electrical_engineering
+            task_alias: electrical_engineering
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: electrical_engineering
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about electrical engineering.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_elementary_mathematics:
+            task: mmlu_elementary_mathematics
+            task_alias: elementary_mathematics
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: elementary_mathematics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about elementary mathematics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_formal_logic:
+            task: mmlu_formal_logic
+            task_alias: formal_logic
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: formal_logic
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about formal logic.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_global_facts:
+            task: mmlu_global_facts
+            task_alias: global_facts
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: global_facts
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about global facts.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_biology:
+            task: mmlu_high_school_biology
+            task_alias: high_school_biology
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_biology
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school biology.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_chemistry:
+            task: mmlu_high_school_chemistry
+            task_alias: high_school_chemistry
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_chemistry
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school chemistry.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_computer_science:
+            task: mmlu_high_school_computer_science
+            task_alias: high_school_computer_science
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_computer_science
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school computer science.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_european_history:
+            task: mmlu_high_school_european_history
+            task_alias: high_school_european_history
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_european_history
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school european history.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_geography:
+            task: mmlu_high_school_geography
+            task_alias: high_school_geography
+            group: mmlu_social_sciences
+            group_alias: social_sciences
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_geography
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school geography.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_government_and_politics:
+            task: mmlu_high_school_government_and_politics
+            task_alias: high_school_government_and_politics
+            group: mmlu_social_sciences
+            group_alias: social_sciences
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_government_and_politics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school government and politics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_macroeconomics:
+            task: mmlu_high_school_macroeconomics
+            task_alias: high_school_macroeconomics
+            group: mmlu_social_sciences
+            group_alias: social_sciences
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_macroeconomics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school macroeconomics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_mathematics:
+            task: mmlu_high_school_mathematics
+            task_alias: high_school_mathematics
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_mathematics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school mathematics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_microeconomics:
+            task: mmlu_high_school_microeconomics
+            task_alias: high_school_microeconomics
+            group: mmlu_social_sciences
+            group_alias: social_sciences
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_microeconomics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school microeconomics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_physics:
+            task: mmlu_high_school_physics
+            task_alias: high_school_physics
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_physics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school physics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_psychology:
+            task: mmlu_high_school_psychology
+            task_alias: high_school_psychology
+            group: mmlu_social_sciences
+            group_alias: social_sciences
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_psychology
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school psychology.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_statistics:
+            task: mmlu_high_school_statistics
+            task_alias: high_school_statistics
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_statistics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school statistics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_us_history:
+            task: mmlu_high_school_us_history
+            task_alias: high_school_us_history
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_us_history
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school us history.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_world_history:
+            task: mmlu_high_school_world_history
+            task_alias: high_school_world_history
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_world_history
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school world history.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_human_aging:
+            task: mmlu_human_aging
+            task_alias: human_aging
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: human_aging
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about human aging.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_human_sexuality:
+            task: mmlu_human_sexuality
+            task_alias: human_sexuality
+            group: mmlu_social_sciences
+            group_alias: social_sciences
+            dataset_path: hails/mmlu_no_train
+            dataset_name: human_sexuality
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about human sexuality.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_international_law:
+            task: mmlu_international_law
+            task_alias: international_law
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: international_law
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about international law.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_jurisprudence:
+            task: mmlu_jurisprudence
+            task_alias: jurisprudence
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: jurisprudence
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about jurisprudence.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_logical_fallacies:
+            task: mmlu_logical_fallacies
+            task_alias: logical_fallacies
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: logical_fallacies
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about logical fallacies.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_machine_learning:
+            task: mmlu_machine_learning
+            task_alias: machine_learning
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: machine_learning
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about machine learning.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_management:
+            task: mmlu_management
+            task_alias: management
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: management
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about management.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_marketing:
+            task: mmlu_marketing
+            task_alias: marketing
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: marketing
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about marketing.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_medical_genetics:
+            task: mmlu_medical_genetics
+            task_alias: medical_genetics
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: medical_genetics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about medical genetics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_miscellaneous:
+            task: mmlu_miscellaneous
+            task_alias: miscellaneous
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: miscellaneous
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about miscellaneous.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_moral_disputes:
+            task: mmlu_moral_disputes
+            task_alias: moral_disputes
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: moral_disputes
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about moral disputes.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_moral_scenarios:
+            task: mmlu_moral_scenarios
+            task_alias: moral_scenarios
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: moral_scenarios
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about moral scenarios.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_nutrition:
+            task: mmlu_nutrition
+            task_alias: nutrition
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: nutrition
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about nutrition.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_philosophy:
+            task: mmlu_philosophy
+            task_alias: philosophy
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: philosophy
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about philosophy.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_prehistory:
+            task: mmlu_prehistory
+            task_alias: prehistory
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: prehistory
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about prehistory.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_professional_accounting:
+            task: mmlu_professional_accounting
+            task_alias: professional_accounting
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: professional_accounting
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about professional accounting.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_professional_law:
+            task: mmlu_professional_law
+            task_alias: professional_law
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: professional_law
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about professional law.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_professional_medicine:
+            task: mmlu_professional_medicine
+            task_alias: professional_medicine
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: professional_medicine
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about professional medicine.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_professional_psychology:
+            task: mmlu_professional_psychology
+            task_alias: professional_psychology
+            group: mmlu_social_sciences
+            group_alias: social_sciences
+            dataset_path: hails/mmlu_no_train
+            dataset_name: professional_psychology
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about professional psychology.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_public_relations:
+            task: mmlu_public_relations
+            task_alias: public_relations
+            group: mmlu_social_sciences
+            group_alias: social_sciences
+            dataset_path: hails/mmlu_no_train
+            dataset_name: public_relations
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about public relations.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_security_studies:
+            task: mmlu_security_studies
+            task_alias: security_studies
+            group: mmlu_social_sciences
+            group_alias: social_sciences
+            dataset_path: hails/mmlu_no_train
+            dataset_name: security_studies
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about security studies.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_sociology:
+            task: mmlu_sociology
+            task_alias: sociology
+            group: mmlu_social_sciences
+            group_alias: social_sciences
+            dataset_path: hails/mmlu_no_train
+            dataset_name: sociology
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about sociology.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_us_foreign_policy:
+            task: mmlu_us_foreign_policy
+            task_alias: us_foreign_policy
+            group: mmlu_social_sciences
+            group_alias: social_sciences
+            dataset_path: hails/mmlu_no_train
+            dataset_name: us_foreign_policy
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about us foreign policy.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_virology:
+            task: mmlu_virology
+            task_alias: virology
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: virology
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about virology.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_world_religions:
+            task: mmlu_world_religions
+            task_alias: world_religions
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: world_religions
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about world religions.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+        versions:
+          mmlu_abstract_algebra: 0.0
+          mmlu_anatomy: 0.0
+          mmlu_astronomy: 0.0
+          mmlu_business_ethics: 0.0
+          mmlu_clinical_knowledge: 0.0
+          mmlu_college_biology: 0.0
+          mmlu_college_chemistry: 0.0
+          mmlu_college_computer_science: 0.0
+          mmlu_college_mathematics: 0.0
+          mmlu_college_medicine: 0.0
+          mmlu_college_physics: 0.0
+          mmlu_computer_security: 0.0
+          mmlu_conceptual_physics: 0.0
+          mmlu_econometrics: 0.0
+          mmlu_electrical_engineering: 0.0
+          mmlu_elementary_mathematics: 0.0
+          mmlu_formal_logic: 0.0
+          mmlu_global_facts: 0.0
+          mmlu_high_school_biology: 0.0
+          mmlu_high_school_chemistry: 0.0
+          mmlu_high_school_computer_science: 0.0
+          mmlu_high_school_european_history: 0.0
+          mmlu_high_school_geography: 0.0
+          mmlu_high_school_government_and_politics: 0.0
+          mmlu_high_school_macroeconomics: 0.0
+          mmlu_high_school_mathematics: 0.0
+          mmlu_high_school_microeconomics: 0.0
+          mmlu_high_school_physics: 0.0
+          mmlu_high_school_psychology: 0.0
+          mmlu_high_school_statistics: 0.0
+          mmlu_high_school_us_history: 0.0
+          mmlu_high_school_world_history: 0.0
+          mmlu_human_aging: 0.0
+          mmlu_human_sexuality: 0.0
+          mmlu_international_law: 0.0
+          mmlu_jurisprudence: 0.0
+          mmlu_logical_fallacies: 0.0
+          mmlu_machine_learning: 0.0
+          mmlu_management: 0.0
+          mmlu_marketing: 0.0
+          mmlu_medical_genetics: 0.0
+          mmlu_miscellaneous: 0.0
+          mmlu_moral_disputes: 0.0
+          mmlu_moral_scenarios: 0.0
+          mmlu_nutrition: 0.0
+          mmlu_philosophy: 0.0
+          mmlu_prehistory: 0.0
+          mmlu_professional_accounting: 0.0
+          mmlu_professional_law: 0.0
+          mmlu_professional_medicine: 0.0
+          mmlu_professional_psychology: 0.0
+          mmlu_public_relations: 0.0
+          mmlu_security_studies: 0.0
+          mmlu_sociology: 0.0
+          mmlu_us_foreign_policy: 0.0
+          mmlu_virology: 0.0
+          mmlu_world_religions: 0.0
+        n-shot:
+          mmlu: 0
+        config:
+          model: vllm
+          model_args: pretrained=DiscoResearch/Llama3-DiscoLeo-Instruct-8B-v0.1,tensor_parallel_size=1,dtype=auto,gpu_memory_utilization=0.8,max_model_len=2048,trust_remote_code=True
+          batch_size: auto
+          batch_sizes: []
+          bootstrap_iters: 100000
+        git_hash: cddf85d
+        pretty_env_info: 'PyTorch version: 2.1.2+cu121
+
+          Is debug build: False
+
+          CUDA used to build PyTorch: 12.1
+
+          ROCM used to build PyTorch: N/A
+
+
+          OS: Ubuntu 22.04.3 LTS (x86_64)
+
+          GCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0
+
+          Clang version: Could not collect
+
+          CMake version: version 3.25.0
+
+          Libc version: glibc-2.35
+
+
+          Python version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit
+          runtime)
+
+          Python platform: Linux-6.5.0-35-generic-x86_64-with-glibc2.35
+
+          Is CUDA available: True
+
+          CUDA runtime version: 11.8.89
+
+          CUDA_MODULE_LOADING set to: LAZY
+
+          GPU models and configuration: GPU 0: NVIDIA GeForce RTX 4090
+
+          Nvidia driver version: 550.54.15
+
+          cuDNN version: Could not collect
+
+          HIP runtime version: N/A
+
+          MIOpen runtime version: N/A
+
+          Is XNNPACK available: True
+
+
+          CPU:
+
+          Architecture:                       x86_64
+
+          CPU op-mode(s):                     32-bit, 64-bit
+
+          Address sizes:                      52 bits physical, 57 bits virtual
+
+          Byte Order:                         Little Endian
+
+          CPU(s):                             64
+
+          On-line CPU(s) list:                0-63
+
+          Vendor ID:                          AuthenticAMD
+
+          Model name:                         AMD EPYC 9354 32-Core Processor
+
+          CPU family:                         25
+
+          Model:                              17
+
+          Thread(s) per core:                 2
+
+          Core(s) per socket:                 32
+
+          Socket(s):                          1
+
+          Stepping:                           1
+
+          Frequency boost:                    enabled
+
+          CPU max MHz:                        3799.0720
+
+          CPU min MHz:                        1500.0000
+
+          BogoMIPS:                           6499.74
+
+          Flags:                              fpu vme de pse tsc msr pae mce cx8 apic
+          sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx
+          mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl
+          nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3
+          fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand
+          lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch
+          osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc
+          mwaitx cpb cat_l3 cdp_l3 invpcid_single hw_pstate ssbd mba perfmon_v2 ibrs
+          ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid
+          cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd
+          sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc
+          cqm_mbm_total cqm_mbm_local avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd
+          amd_ppin cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid
+          decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl
+          vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni
+          avx512_bitalg avx512_vpopcntdq la57 rdpid overflow_recov succor smca fsrm
+          flush_l1d
+
+          Virtualization:                     AMD-V
+
+          L1d cache:                          1 MiB (32 instances)
+
+          L1i cache:                          1 MiB (32 instances)
+
+          L2 cache:                           32 MiB (32 instances)
+
+          L3 cache:                           256 MiB (8 instances)
+
+          NUMA node(s):                       1
+
+          NUMA node0 CPU(s):                  0-63
+
+          Vulnerability Gather data sampling: Not affected
+
+          Vulnerability Itlb multihit:        Not affected
+
+          Vulnerability L1tf:                 Not affected
+
+          Vulnerability Mds:                  Not affected
+
+          Vulnerability Meltdown:             Not affected
+
+          Vulnerability Mmio stale data:      Not affected
+
+          Vulnerability Retbleed:             Not affected
+
+          Vulnerability Spec rstack overflow: Mitigation; Safe RET
+
+          Vulnerability Spec store bypass:    Mitigation; Speculative Store Bypass
+          disabled via prctl
+
+          Vulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers
+          and __user pointer sanitization
+
+          Vulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS;
+          IBPB conditional; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected;
+          BHI Not affected
+
+          Vulnerability Srbds:                Not affected
+
+          Vulnerability Tsx async abort:      Not affected
+
+
           Versions of relevant libraries:
 
           [pip3] numpy==1.24.1