Upload README.md with huggingface_hub

README.md

@@ -2148,6 +2148,3258 @@ model-index:
           Vulnerability Tsx async abort:      Not affected
 
 
+          Versions of relevant libraries:
+
+          [pip3] numpy==1.24.1
+
+          [pip3] torch==2.1.2
+
+          [pip3] torchaudio==2.0.2+cu118
+
+          [pip3] torchvision==0.15.2+cu118
+
+          [pip3] triton==2.1.0
+
+          [conda] Could not collect'
+        transformers_version: 4.42.4
+  - task:
+      type: mmlu
+    dataset:
+      name: mmlu
+      type: public-dataset
+    metrics:
+    - type: acc
+      value: '0.634'
+      args:
+        results:
+          mmlu:
+            acc,none: 0.6240564022219057
+            acc_stderr,none: 0.0038572036515963077
+            alias: mmlu
+          mmlu_humanities:
+            alias: ' - humanities'
+            acc,none: 0.5704569606801275
+            acc_stderr,none: 0.00680518705216219
+          mmlu_formal_logic:
+            alias: '  - formal_logic'
+            acc,none: 0.42857142857142855
+            acc_stderr,none: 0.0442626668137991
+          mmlu_high_school_european_history:
+            alias: '  - high_school_european_history'
+            acc,none: 0.7393939393939394
+            acc_stderr,none: 0.034277431758165236
+          mmlu_high_school_us_history:
+            alias: '  - high_school_us_history'
+            acc,none: 0.8235294117647058
+            acc_stderr,none: 0.02675640153807895
+          mmlu_high_school_world_history:
+            alias: '  - high_school_world_history'
+            acc,none: 0.8354430379746836
+            acc_stderr,none: 0.024135736240566946
+          mmlu_international_law:
+            alias: '  - international_law'
+            acc,none: 0.71900826446281
+            acc_stderr,none: 0.04103203830514512
+          mmlu_jurisprudence:
+            alias: '  - jurisprudence'
+            acc,none: 0.7592592592592593
+            acc_stderr,none: 0.04133119440243839
+          mmlu_logical_fallacies:
+            alias: '  - logical_fallacies'
+            acc,none: 0.7668711656441718
+            acc_stderr,none: 0.0332201579577674
+          mmlu_moral_disputes:
+            alias: '  - moral_disputes'
+            acc,none: 0.6502890173410405
+            acc_stderr,none: 0.02567428145653102
+          mmlu_moral_scenarios:
+            alias: '  - moral_scenarios'
+            acc,none: 0.35307262569832404
+            acc_stderr,none: 0.01598420454526857
+          mmlu_philosophy:
+            alias: '  - philosophy'
+            acc,none: 0.7009646302250804
+            acc_stderr,none: 0.026003301117885142
+          mmlu_prehistory:
+            alias: '  - prehistory'
+            acc,none: 0.7160493827160493
+            acc_stderr,none: 0.02508947852376513
+          mmlu_professional_law:
+            alias: '  - professional_law'
+            acc,none: 0.470013037809648
+            acc_stderr,none: 0.012747248967079062
+          mmlu_world_religions:
+            alias: '  - world_religions'
+            acc,none: 0.7953216374269005
+            acc_stderr,none: 0.030944459778533204
+          mmlu_other:
+            alias: ' - other'
+            acc,none: 0.7151593176697779
+            acc_stderr,none: 0.00781329664246705
+          mmlu_business_ethics:
+            alias: '  - business_ethics'
+            acc,none: 0.61
+            acc_stderr,none: 0.04902071300001974
+          mmlu_clinical_knowledge:
+            alias: '  - clinical_knowledge'
+            acc,none: 0.7584905660377359
+            acc_stderr,none: 0.026341480371118355
+          mmlu_college_medicine:
+            alias: '  - college_medicine'
+            acc,none: 0.6589595375722543
+            acc_stderr,none: 0.036146654241808254
+          mmlu_global_facts:
+            alias: '  - global_facts'
+            acc,none: 0.41
+            acc_stderr,none: 0.04943110704237102
+          mmlu_human_aging:
+            alias: '  - human_aging'
+            acc,none: 0.6860986547085202
+            acc_stderr,none: 0.031146796482972465
+          mmlu_management:
+            alias: '  - management'
+            acc,none: 0.8543689320388349
+            acc_stderr,none: 0.03492606476623789
+          mmlu_marketing:
+            alias: '  - marketing'
+            acc,none: 0.8717948717948718
+            acc_stderr,none: 0.02190190511507333
+          mmlu_medical_genetics:
+            alias: '  - medical_genetics'
+            acc,none: 0.75
+            acc_stderr,none: 0.04351941398892446
+          mmlu_miscellaneous:
+            alias: '  - miscellaneous'
+            acc,none: 0.8263090676883781
+            acc_stderr,none: 0.013547415658662264
+          mmlu_nutrition:
+            alias: '  - nutrition'
+            acc,none: 0.7091503267973857
+            acc_stderr,none: 0.02600480036395213
+          mmlu_professional_accounting:
+            alias: '  - professional_accounting'
+            acc,none: 0.5212765957446809
+            acc_stderr,none: 0.029800481645628693
+          mmlu_professional_medicine:
+            alias: '  - professional_medicine'
+            acc,none: 0.6875
+            acc_stderr,none: 0.02815637344037142
+          mmlu_virology:
+            alias: '  - virology'
+            acc,none: 0.5240963855421686
+            acc_stderr,none: 0.038879718495972646
+          mmlu_social_sciences:
+            alias: ' - social_sciences'
+            acc,none: 0.7221319467013325
+            acc_stderr,none: 0.007909660127989188
+          mmlu_econometrics:
+            alias: '  - econometrics'
+            acc,none: 0.5
+            acc_stderr,none: 0.047036043419179864
+          mmlu_high_school_geography:
+            alias: '  - high_school_geography'
+            acc,none: 0.7575757575757576
+            acc_stderr,none: 0.030532892233932026
+          mmlu_high_school_government_and_politics:
+            alias: '  - high_school_government_and_politics'
+            acc,none: 0.8652849740932642
+            acc_stderr,none: 0.024639789097709437
+          mmlu_high_school_macroeconomics:
+            alias: '  - high_school_macroeconomics'
+            acc,none: 0.5923076923076923
+            acc_stderr,none: 0.024915243985987847
+          mmlu_high_school_microeconomics:
+            alias: '  - high_school_microeconomics'
+            acc,none: 0.6932773109243697
+            acc_stderr,none: 0.02995382389188703
+          mmlu_high_school_psychology:
+            alias: '  - high_school_psychology'
+            acc,none: 0.7963302752293578
+            acc_stderr,none: 0.017266742087630797
+          mmlu_human_sexuality:
+            alias: '  - human_sexuality'
+            acc,none: 0.7862595419847328
+            acc_stderr,none: 0.035954616117746904
+          mmlu_professional_psychology:
+            alias: '  - professional_psychology'
+            acc,none: 0.6683006535947712
+            acc_stderr,none: 0.01904748523936038
+          mmlu_public_relations:
+            alias: '  - public_relations'
+            acc,none: 0.6545454545454545
+            acc_stderr,none: 0.04554619617541054
+          mmlu_security_studies:
+            alias: '  - security_studies'
+            acc,none: 0.726530612244898
+            acc_stderr,none: 0.02853556033712844
+          mmlu_sociology:
+            alias: '  - sociology'
+            acc,none: 0.845771144278607
+            acc_stderr,none: 0.025538433368578337
+          mmlu_us_foreign_policy:
+            alias: '  - us_foreign_policy'
+            acc,none: 0.86
+            acc_stderr,none: 0.03487350880197769
+          mmlu_stem:
+            alias: ' - stem'
+            acc,none: 0.5185537583254044
+            acc_stderr,none: 0.008550177348592522
+          mmlu_abstract_algebra:
+            alias: '  - abstract_algebra'
+            acc,none: 0.36
+            acc_stderr,none: 0.04824181513244218
+          mmlu_anatomy:
+            alias: '  - anatomy'
+            acc,none: 0.6074074074074074
+            acc_stderr,none: 0.04218506215368879
+          mmlu_astronomy:
+            alias: '  - astronomy'
+            acc,none: 0.6973684210526315
+            acc_stderr,none: 0.03738520676119668
+          mmlu_college_biology:
+            alias: '  - college_biology'
+            acc,none: 0.7916666666666666
+            acc_stderr,none: 0.033961162058453336
+          mmlu_college_chemistry:
+            alias: '  - college_chemistry'
+            acc,none: 0.4
+            acc_stderr,none: 0.04923659639173309
+          mmlu_college_computer_science:
+            alias: '  - college_computer_science'
+            acc,none: 0.42
+            acc_stderr,none: 0.049604496374885836
+          mmlu_college_mathematics:
+            alias: '  - college_mathematics'
+            acc,none: 0.33
+            acc_stderr,none: 0.047258156262526045
+          mmlu_college_physics:
+            alias: '  - college_physics'
+            acc,none: 0.35294117647058826
+            acc_stderr,none: 0.047551296160629475
+          mmlu_computer_security:
+            alias: '  - computer_security'
+            acc,none: 0.76
+            acc_stderr,none: 0.042923469599092816
+          mmlu_conceptual_physics:
+            alias: '  - conceptual_physics'
+            acc,none: 0.5531914893617021
+            acc_stderr,none: 0.032500536843658404
+          mmlu_electrical_engineering:
+            alias: '  - electrical_engineering'
+            acc,none: 0.5172413793103449
+            acc_stderr,none: 0.04164188720169375
+          mmlu_elementary_mathematics:
+            alias: '  - elementary_mathematics'
+            acc,none: 0.42328042328042326
+            acc_stderr,none: 0.025446365634406772
+          mmlu_high_school_biology:
+            alias: '  - high_school_biology'
+            acc,none: 0.7451612903225806
+            acc_stderr,none: 0.0247901184593322
+          mmlu_high_school_chemistry:
+            alias: '  - high_school_chemistry'
+            acc,none: 0.4827586206896552
+            acc_stderr,none: 0.035158955511657
+          mmlu_high_school_computer_science:
+            alias: '  - high_school_computer_science'
+            acc,none: 0.65
+            acc_stderr,none: 0.0479372485441102
+          mmlu_high_school_mathematics:
+            alias: '  - high_school_mathematics'
+            acc,none: 0.37407407407407406
+            acc_stderr,none: 0.029502861128955286
+          mmlu_high_school_physics:
+            alias: '  - high_school_physics'
+            acc,none: 0.3841059602649007
+            acc_stderr,none: 0.03971301814719197
+          mmlu_high_school_statistics:
+            alias: '  - high_school_statistics'
+            acc,none: 0.4722222222222222
+            acc_stderr,none: 0.0340470532865388
+          mmlu_machine_learning:
+            alias: '  - machine_learning'
+            acc,none: 0.44642857142857145
+            acc_stderr,none: 0.04718471485219588
+        groups:
+          mmlu:
+            acc,none: 0.6240564022219057
+            acc_stderr,none: 0.0038572036515963077
+            alias: mmlu
+          mmlu_humanities:
+            alias: ' - humanities'
+            acc,none: 0.5704569606801275
+            acc_stderr,none: 0.00680518705216219
+          mmlu_other:
+            alias: ' - other'
+            acc,none: 0.7151593176697779
+            acc_stderr,none: 0.00781329664246705
+          mmlu_social_sciences:
+            alias: ' - social_sciences'
+            acc,none: 0.7221319467013325
+            acc_stderr,none: 0.007909660127989188
+          mmlu_stem:
+            alias: ' - stem'
+            acc,none: 0.5185537583254044
+            acc_stderr,none: 0.008550177348592522
+        group_subtasks:
+          mmlu_stem:
+          - mmlu_college_computer_science
+          - mmlu_college_chemistry
+          - mmlu_college_biology
+          - mmlu_astronomy
+          - mmlu_anatomy
+          - mmlu_abstract_algebra
+          - mmlu_machine_learning
+          - mmlu_high_school_statistics
+          - mmlu_high_school_physics
+          - mmlu_high_school_mathematics
+          - mmlu_high_school_computer_science
+          - mmlu_high_school_chemistry
+          - mmlu_high_school_biology
+          - mmlu_elementary_mathematics
+          - mmlu_electrical_engineering
+          - mmlu_conceptual_physics
+          - mmlu_computer_security
+          - mmlu_college_physics
+          - mmlu_college_mathematics
+          mmlu_other:
+          - mmlu_clinical_knowledge
+          - mmlu_business_ethics
+          - mmlu_virology
+          - mmlu_professional_medicine
+          - mmlu_professional_accounting
+          - mmlu_nutrition
+          - mmlu_miscellaneous
+          - mmlu_medical_genetics
+          - mmlu_marketing
+          - mmlu_management
+          - mmlu_human_aging
+          - mmlu_global_facts
+          - mmlu_college_medicine
+          mmlu_social_sciences:
+          - mmlu_us_foreign_policy
+          - mmlu_sociology
+          - mmlu_security_studies
+          - mmlu_public_relations
+          - mmlu_professional_psychology
+          - mmlu_human_sexuality
+          - mmlu_high_school_psychology
+          - mmlu_high_school_microeconomics
+          - mmlu_high_school_macroeconomics
+          - mmlu_high_school_government_and_politics
+          - mmlu_high_school_geography
+          - mmlu_econometrics
+          mmlu_humanities:
+          - mmlu_world_religions
+          - mmlu_professional_law
+          - mmlu_prehistory
+          - mmlu_philosophy
+          - mmlu_moral_scenarios
+          - mmlu_moral_disputes
+          - mmlu_logical_fallacies
+          - mmlu_jurisprudence
+          - mmlu_international_law
+          - mmlu_high_school_world_history
+          - mmlu_high_school_us_history
+          - mmlu_high_school_european_history
+          - mmlu_formal_logic
+          mmlu:
+          - mmlu_humanities
+          - mmlu_social_sciences
+          - mmlu_other
+          - mmlu_stem
+        configs:
+          mmlu_abstract_algebra:
+            task: mmlu_abstract_algebra
+            task_alias: abstract_algebra
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: abstract_algebra
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about abstract algebra.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_anatomy:
+            task: mmlu_anatomy
+            task_alias: anatomy
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: anatomy
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about anatomy.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_astronomy:
+            task: mmlu_astronomy
+            task_alias: astronomy
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: astronomy
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about astronomy.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_business_ethics:
+            task: mmlu_business_ethics
+            task_alias: business_ethics
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: business_ethics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about business ethics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_clinical_knowledge:
+            task: mmlu_clinical_knowledge
+            task_alias: clinical_knowledge
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: clinical_knowledge
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about clinical knowledge.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_college_biology:
+            task: mmlu_college_biology
+            task_alias: college_biology
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: college_biology
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about college biology.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_college_chemistry:
+            task: mmlu_college_chemistry
+            task_alias: college_chemistry
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: college_chemistry
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about college chemistry.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_college_computer_science:
+            task: mmlu_college_computer_science
+            task_alias: college_computer_science
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: college_computer_science
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about college computer science.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_college_mathematics:
+            task: mmlu_college_mathematics
+            task_alias: college_mathematics
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: college_mathematics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about college mathematics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_college_medicine:
+            task: mmlu_college_medicine
+            task_alias: college_medicine
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: college_medicine
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about college medicine.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_college_physics:
+            task: mmlu_college_physics
+            task_alias: college_physics
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: college_physics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about college physics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_computer_security:
+            task: mmlu_computer_security
+            task_alias: computer_security
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: computer_security
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about computer security.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_conceptual_physics:
+            task: mmlu_conceptual_physics
+            task_alias: conceptual_physics
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: conceptual_physics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about conceptual physics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_econometrics:
+            task: mmlu_econometrics
+            task_alias: econometrics
+            group: mmlu_social_sciences
+            group_alias: social_sciences
+            dataset_path: hails/mmlu_no_train
+            dataset_name: econometrics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about econometrics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_electrical_engineering:
+            task: mmlu_electrical_engineering
+            task_alias: electrical_engineering
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: electrical_engineering
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about electrical engineering.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_elementary_mathematics:
+            task: mmlu_elementary_mathematics
+            task_alias: elementary_mathematics
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: elementary_mathematics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about elementary mathematics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_formal_logic:
+            task: mmlu_formal_logic
+            task_alias: formal_logic
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: formal_logic
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about formal logic.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_global_facts:
+            task: mmlu_global_facts
+            task_alias: global_facts
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: global_facts
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about global facts.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_biology:
+            task: mmlu_high_school_biology
+            task_alias: high_school_biology
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_biology
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school biology.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_chemistry:
+            task: mmlu_high_school_chemistry
+            task_alias: high_school_chemistry
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_chemistry
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school chemistry.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_computer_science:
+            task: mmlu_high_school_computer_science
+            task_alias: high_school_computer_science
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_computer_science
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school computer science.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_european_history:
+            task: mmlu_high_school_european_history
+            task_alias: high_school_european_history
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_european_history
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school european history.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_geography:
+            task: mmlu_high_school_geography
+            task_alias: high_school_geography
+            group: mmlu_social_sciences
+            group_alias: social_sciences
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_geography
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school geography.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_government_and_politics:
+            task: mmlu_high_school_government_and_politics
+            task_alias: high_school_government_and_politics
+            group: mmlu_social_sciences
+            group_alias: social_sciences
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_government_and_politics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school government and politics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_macroeconomics:
+            task: mmlu_high_school_macroeconomics
+            task_alias: high_school_macroeconomics
+            group: mmlu_social_sciences
+            group_alias: social_sciences
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_macroeconomics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school macroeconomics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_mathematics:
+            task: mmlu_high_school_mathematics
+            task_alias: high_school_mathematics
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_mathematics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school mathematics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_microeconomics:
+            task: mmlu_high_school_microeconomics
+            task_alias: high_school_microeconomics
+            group: mmlu_social_sciences
+            group_alias: social_sciences
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_microeconomics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school microeconomics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_physics:
+            task: mmlu_high_school_physics
+            task_alias: high_school_physics
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_physics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school physics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_psychology:
+            task: mmlu_high_school_psychology
+            task_alias: high_school_psychology
+            group: mmlu_social_sciences
+            group_alias: social_sciences
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_psychology
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school psychology.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_statistics:
+            task: mmlu_high_school_statistics
+            task_alias: high_school_statistics
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_statistics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school statistics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_us_history:
+            task: mmlu_high_school_us_history
+            task_alias: high_school_us_history
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_us_history
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school us history.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_high_school_world_history:
+            task: mmlu_high_school_world_history
+            task_alias: high_school_world_history
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: high_school_world_history
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about high school world history.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_human_aging:
+            task: mmlu_human_aging
+            task_alias: human_aging
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: human_aging
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about human aging.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_human_sexuality:
+            task: mmlu_human_sexuality
+            task_alias: human_sexuality
+            group: mmlu_social_sciences
+            group_alias: social_sciences
+            dataset_path: hails/mmlu_no_train
+            dataset_name: human_sexuality
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about human sexuality.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_international_law:
+            task: mmlu_international_law
+            task_alias: international_law
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: international_law
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about international law.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_jurisprudence:
+            task: mmlu_jurisprudence
+            task_alias: jurisprudence
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: jurisprudence
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about jurisprudence.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_logical_fallacies:
+            task: mmlu_logical_fallacies
+            task_alias: logical_fallacies
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: logical_fallacies
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about logical fallacies.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_machine_learning:
+            task: mmlu_machine_learning
+            task_alias: machine_learning
+            group: mmlu_stem
+            group_alias: stem
+            dataset_path: hails/mmlu_no_train
+            dataset_name: machine_learning
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about machine learning.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_management:
+            task: mmlu_management
+            task_alias: management
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: management
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about management.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_marketing:
+            task: mmlu_marketing
+            task_alias: marketing
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: marketing
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about marketing.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_medical_genetics:
+            task: mmlu_medical_genetics
+            task_alias: medical_genetics
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: medical_genetics
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about medical genetics.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_miscellaneous:
+            task: mmlu_miscellaneous
+            task_alias: miscellaneous
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: miscellaneous
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about miscellaneous.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_moral_disputes:
+            task: mmlu_moral_disputes
+            task_alias: moral_disputes
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: moral_disputes
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about moral disputes.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_moral_scenarios:
+            task: mmlu_moral_scenarios
+            task_alias: moral_scenarios
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: moral_scenarios
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about moral scenarios.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_nutrition:
+            task: mmlu_nutrition
+            task_alias: nutrition
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: nutrition
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about nutrition.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_philosophy:
+            task: mmlu_philosophy
+            task_alias: philosophy
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: philosophy
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about philosophy.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_prehistory:
+            task: mmlu_prehistory
+            task_alias: prehistory
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: prehistory
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about prehistory.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_professional_accounting:
+            task: mmlu_professional_accounting
+            task_alias: professional_accounting
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: professional_accounting
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about professional accounting.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_professional_law:
+            task: mmlu_professional_law
+            task_alias: professional_law
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: professional_law
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about professional law.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_professional_medicine:
+            task: mmlu_professional_medicine
+            task_alias: professional_medicine
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: professional_medicine
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about professional medicine.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_professional_psychology:
+            task: mmlu_professional_psychology
+            task_alias: professional_psychology
+            group: mmlu_social_sciences
+            group_alias: social_sciences
+            dataset_path: hails/mmlu_no_train
+            dataset_name: professional_psychology
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about professional psychology.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_public_relations:
+            task: mmlu_public_relations
+            task_alias: public_relations
+            group: mmlu_social_sciences
+            group_alias: social_sciences
+            dataset_path: hails/mmlu_no_train
+            dataset_name: public_relations
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about public relations.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_security_studies:
+            task: mmlu_security_studies
+            task_alias: security_studies
+            group: mmlu_social_sciences
+            group_alias: social_sciences
+            dataset_path: hails/mmlu_no_train
+            dataset_name: security_studies
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about security studies.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_sociology:
+            task: mmlu_sociology
+            task_alias: sociology
+            group: mmlu_social_sciences
+            group_alias: social_sciences
+            dataset_path: hails/mmlu_no_train
+            dataset_name: sociology
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about sociology.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_us_foreign_policy:
+            task: mmlu_us_foreign_policy
+            task_alias: us_foreign_policy
+            group: mmlu_social_sciences
+            group_alias: social_sciences
+            dataset_path: hails/mmlu_no_train
+            dataset_name: us_foreign_policy
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about us foreign policy.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_virology:
+            task: mmlu_virology
+            task_alias: virology
+            group: mmlu_other
+            group_alias: other
+            dataset_path: hails/mmlu_no_train
+            dataset_name: virology
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about virology.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+          mmlu_world_religions:
+            task: mmlu_world_religions
+            task_alias: world_religions
+            group: mmlu_humanities
+            group_alias: humanities
+            dataset_path: hails/mmlu_no_train
+            dataset_name: world_religions
+            test_split: test
+            fewshot_split: dev
+            doc_to_text: '{{question.strip()}}
+
+              A. {{choices[0]}}
+
+              B. {{choices[1]}}
+
+              C. {{choices[2]}}
+
+              D. {{choices[3]}}
+
+              Answer:'
+            doc_to_target: answer
+            doc_to_choice:
+            - A
+            - B
+            - C
+            - D
+            description: 'The following are multiple choice questions (with answers)
+              about world religions.
+
+
+              '
+            target_delimiter: ' '
+            fewshot_delimiter: '
+
+
+              '
+            fewshot_config:
+              sampler: first_n
+            metric_list:
+            - metric: acc
+              aggregation: mean
+              higher_is_better: true
+            output_type: multiple_choice
+            repeats: 1
+            should_decontaminate: false
+            metadata:
+              version: 0.0
+        versions:
+          mmlu_abstract_algebra: 0.0
+          mmlu_anatomy: 0.0
+          mmlu_astronomy: 0.0
+          mmlu_business_ethics: 0.0
+          mmlu_clinical_knowledge: 0.0
+          mmlu_college_biology: 0.0
+          mmlu_college_chemistry: 0.0
+          mmlu_college_computer_science: 0.0
+          mmlu_college_mathematics: 0.0
+          mmlu_college_medicine: 0.0
+          mmlu_college_physics: 0.0
+          mmlu_computer_security: 0.0
+          mmlu_conceptual_physics: 0.0
+          mmlu_econometrics: 0.0
+          mmlu_electrical_engineering: 0.0
+          mmlu_elementary_mathematics: 0.0
+          mmlu_formal_logic: 0.0
+          mmlu_global_facts: 0.0
+          mmlu_high_school_biology: 0.0
+          mmlu_high_school_chemistry: 0.0
+          mmlu_high_school_computer_science: 0.0
+          mmlu_high_school_european_history: 0.0
+          mmlu_high_school_geography: 0.0
+          mmlu_high_school_government_and_politics: 0.0
+          mmlu_high_school_macroeconomics: 0.0
+          mmlu_high_school_mathematics: 0.0
+          mmlu_high_school_microeconomics: 0.0
+          mmlu_high_school_physics: 0.0
+          mmlu_high_school_psychology: 0.0
+          mmlu_high_school_statistics: 0.0
+          mmlu_high_school_us_history: 0.0
+          mmlu_high_school_world_history: 0.0
+          mmlu_human_aging: 0.0
+          mmlu_human_sexuality: 0.0
+          mmlu_international_law: 0.0
+          mmlu_jurisprudence: 0.0
+          mmlu_logical_fallacies: 0.0
+          mmlu_machine_learning: 0.0
+          mmlu_management: 0.0
+          mmlu_marketing: 0.0
+          mmlu_medical_genetics: 0.0
+          mmlu_miscellaneous: 0.0
+          mmlu_moral_disputes: 0.0
+          mmlu_moral_scenarios: 0.0
+          mmlu_nutrition: 0.0
+          mmlu_philosophy: 0.0
+          mmlu_prehistory: 0.0
+          mmlu_professional_accounting: 0.0
+          mmlu_professional_law: 0.0
+          mmlu_professional_medicine: 0.0
+          mmlu_professional_psychology: 0.0
+          mmlu_public_relations: 0.0
+          mmlu_security_studies: 0.0
+          mmlu_sociology: 0.0
+          mmlu_us_foreign_policy: 0.0
+          mmlu_virology: 0.0
+          mmlu_world_religions: 0.0
+        n-shot:
+          mmlu: 0
+        config:
+          model: vllm
+          model_args: pretrained=DataGuard/Llama-disco-pali-merged,tensor_parallel_size=1,dtype=auto,gpu_memory_utilization=0.8,max_model_len=2048,trust_remote_code=True
+          batch_size: auto
+          batch_sizes: []
+          bootstrap_iters: 100000
+        git_hash: cddf85d
+        pretty_env_info: 'PyTorch version: 2.1.2+cu121
+
+          Is debug build: False
+
+          CUDA used to build PyTorch: 12.1
+
+          ROCM used to build PyTorch: N/A
+
+
+          OS: Ubuntu 22.04.3 LTS (x86_64)
+
+          GCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0
+
+          Clang version: Could not collect
+
+          CMake version: version 3.25.0
+
+          Libc version: glibc-2.35
+
+
+          Python version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit
+          runtime)
+
+          Python platform: Linux-6.5.0-35-generic-x86_64-with-glibc2.35
+
+          Is CUDA available: True
+
+          CUDA runtime version: 11.8.89
+
+          CUDA_MODULE_LOADING set to: LAZY
+
+          GPU models and configuration: GPU 0: NVIDIA GeForce RTX 4090
+
+          Nvidia driver version: 550.54.15
+
+          cuDNN version: Could not collect
+
+          HIP runtime version: N/A
+
+          MIOpen runtime version: N/A
+
+          Is XNNPACK available: True
+
+
+          CPU:
+
+          Architecture:                       x86_64
+
+          CPU op-mode(s):                     32-bit, 64-bit
+
+          Address sizes:                      52 bits physical, 57 bits virtual
+
+          Byte Order:                         Little Endian
+
+          CPU(s):                             64
+
+          On-line CPU(s) list:                0-63
+
+          Vendor ID:                          AuthenticAMD
+
+          Model name:                         AMD EPYC 9354 32-Core Processor
+
+          CPU family:                         25
+
+          Model:                              17
+
+          Thread(s) per core:                 2
+
+          Core(s) per socket:                 32
+
+          Socket(s):                          1
+
+          Stepping:                           1
+
+          Frequency boost:                    enabled
+
+          CPU max MHz:                        3799.0720
+
+          CPU min MHz:                        1500.0000
+
+          BogoMIPS:                           6499.74
+
+          Flags:                              fpu vme de pse tsc msr pae mce cx8 apic
+          sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx
+          mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl
+          nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3
+          fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand
+          lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch
+          osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc
+          mwaitx cpb cat_l3 cdp_l3 invpcid_single hw_pstate ssbd mba perfmon_v2 ibrs
+          ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid
+          cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd
+          sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc
+          cqm_mbm_total cqm_mbm_local avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd
+          amd_ppin cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid
+          decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl
+          vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni
+          avx512_bitalg avx512_vpopcntdq la57 rdpid overflow_recov succor smca fsrm
+          flush_l1d
+
+          Virtualization:                     AMD-V
+
+          L1d cache:                          1 MiB (32 instances)
+
+          L1i cache:                          1 MiB (32 instances)
+
+          L2 cache:                           32 MiB (32 instances)
+
+          L3 cache:                           256 MiB (8 instances)
+
+          NUMA node(s):                       1
+
+          NUMA node0 CPU(s):                  0-63
+
+          Vulnerability Gather data sampling: Not affected
+
+          Vulnerability Itlb multihit:        Not affected
+
+          Vulnerability L1tf:                 Not affected
+
+          Vulnerability Mds:                  Not affected
+
+          Vulnerability Meltdown:             Not affected
+
+          Vulnerability Mmio stale data:      Not affected
+
+          Vulnerability Retbleed:             Not affected
+
+          Vulnerability Spec rstack overflow: Mitigation; Safe RET
+
+          Vulnerability Spec store bypass:    Mitigation; Speculative Store Bypass
+          disabled via prctl
+
+          Vulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers
+          and __user pointer sanitization
+
+          Vulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS;
+          IBPB conditional; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected;
+          BHI Not affected
+
+          Vulnerability Srbds:                Not affected
+
+          Vulnerability Tsx async abort:      Not affected
+
+
           Versions of relevant libraries:
 
           [pip3] numpy==1.24.1