{ "results": { "blimp": { "acc,none": 0.6665074626865671, "acc_stderr,none": 0.2001790545943022, "alias": "blimp" }, "blimp_adjunct_island": { "acc,none": 0.668, "acc_stderr,none": 0.014899597242811476, "alias": " - blimp_adjunct_island" }, "blimp_anaphor_gender_agreement": { "acc,none": 0.67, "acc_stderr,none": 0.014876872027456741, "alias": " - blimp_anaphor_gender_agreement" }, "blimp_anaphor_number_agreement": { "acc,none": 0.863, "acc_stderr,none": 0.01087884871433333, "alias": " - blimp_anaphor_number_agreement" }, "blimp_animate_subject_passive": { "acc,none": 0.74, "acc_stderr,none": 0.013877773329774166, "alias": " - blimp_animate_subject_passive" }, "blimp_animate_subject_trans": { "acc,none": 0.692, "acc_stderr,none": 0.01460648312734276, "alias": " - blimp_animate_subject_trans" }, "blimp_causative": { "acc,none": 0.613, "acc_stderr,none": 0.015410011955493935, "alias": " - blimp_causative" }, "blimp_complex_NP_island": { "acc,none": 0.539, "acc_stderr,none": 0.01577110420128319, "alias": " - blimp_complex_NP_island" }, "blimp_coordinate_structure_constraint_complex_left_branch": { "acc,none": 0.378, "acc_stderr,none": 0.015341165254026649, "alias": " - blimp_coordinate_structure_constraint_complex_left_branch" }, "blimp_coordinate_structure_constraint_object_extraction": { "acc,none": 0.559, "acc_stderr,none": 0.015708779894242676, "alias": " - blimp_coordinate_structure_constraint_object_extraction" }, "blimp_determiner_noun_agreement_1": { "acc,none": 0.912, "acc_stderr,none": 0.008963053962592086, "alias": " - blimp_determiner_noun_agreement_1" }, "blimp_determiner_noun_agreement_2": { "acc,none": 0.932, "acc_stderr,none": 0.007964887911291605, "alias": " - blimp_determiner_noun_agreement_2" }, "blimp_determiner_noun_agreement_irregular_1": { "acc,none": 0.738, "acc_stderr,none": 0.013912208651021345, "alias": " - blimp_determiner_noun_agreement_irregular_1" }, "blimp_determiner_noun_agreement_irregular_2": { "acc,none": 0.849, "acc_stderr,none": 0.011328165223341681, "alias": " - blimp_determiner_noun_agreement_irregular_2" }, "blimp_determiner_noun_agreement_with_adj_2": { "acc,none": 0.885, "acc_stderr,none": 0.010093407594904624, "alias": " - blimp_determiner_noun_agreement_with_adj_2" }, "blimp_determiner_noun_agreement_with_adj_irregular_1": { "acc,none": 0.787, "acc_stderr,none": 0.01295371756673723, "alias": " - blimp_determiner_noun_agreement_with_adj_irregular_1" }, "blimp_determiner_noun_agreement_with_adj_irregular_2": { "acc,none": 0.826, "acc_stderr,none": 0.011994493230973438, "alias": " - blimp_determiner_noun_agreement_with_adj_irregular_2" }, "blimp_determiner_noun_agreement_with_adjective_1": { "acc,none": 0.888, "acc_stderr,none": 0.00997775303139724, "alias": " - blimp_determiner_noun_agreement_with_adjective_1" }, "blimp_distractor_agreement_relational_noun": { "acc,none": 0.446, "acc_stderr,none": 0.015726771166750357, "alias": " - blimp_distractor_agreement_relational_noun" }, "blimp_distractor_agreement_relative_clause": { "acc,none": 0.412, "acc_stderr,none": 0.01557236329201509, "alias": " - blimp_distractor_agreement_relative_clause" }, "blimp_drop_argument": { "acc,none": 0.69, "acc_stderr,none": 0.014632638658632902, "alias": " - blimp_drop_argument" }, "blimp_ellipsis_n_bar_1": { "acc,none": 0.599, "acc_stderr,none": 0.015506109745498316, "alias": " - blimp_ellipsis_n_bar_1" }, "blimp_ellipsis_n_bar_2": { "acc,none": 0.483, "acc_stderr,none": 0.015810153729833434, "alias": " - blimp_ellipsis_n_bar_2" }, "blimp_existential_there_object_raising": { "acc,none": 0.696, "acc_stderr,none": 0.014553205687950427, "alias": " - blimp_existential_there_object_raising" }, "blimp_existential_there_quantifiers_1": { "acc,none": 0.936, "acc_stderr,none": 0.007743640226919291, "alias": " - blimp_existential_there_quantifiers_1" }, "blimp_existential_there_quantifiers_2": { "acc,none": 0.488, "acc_stderr,none": 0.015814743314581818, "alias": " - blimp_existential_there_quantifiers_2" }, "blimp_existential_there_subject_raising": { "acc,none": 0.766, "acc_stderr,none": 0.013394902889660006, "alias": " - blimp_existential_there_subject_raising" }, "blimp_expletive_it_object_raising": { "acc,none": 0.661, "acc_stderr,none": 0.014976758771620344, "alias": " - blimp_expletive_it_object_raising" }, "blimp_inchoative": { "acc,none": 0.51, "acc_stderr,none": 0.015816135752773203, "alias": " - blimp_inchoative" }, "blimp_intransitive": { "acc,none": 0.648, "acc_stderr,none": 0.015110404505648675, "alias": " - blimp_intransitive" }, "blimp_irregular_past_participle_adjectives": { "acc,none": 0.811, "acc_stderr,none": 0.012386784588117709, "alias": " - blimp_irregular_past_participle_adjectives" }, "blimp_irregular_past_participle_verbs": { "acc,none": 0.819, "acc_stderr,none": 0.01218143617917792, "alias": " - blimp_irregular_past_participle_verbs" }, "blimp_irregular_plural_subject_verb_agreement_1": { "acc,none": 0.698, "acc_stderr,none": 0.014526080235459543, "alias": " - blimp_irregular_plural_subject_verb_agreement_1" }, "blimp_irregular_plural_subject_verb_agreement_2": { "acc,none": 0.779, "acc_stderr,none": 0.01312750285969623, "alias": " - blimp_irregular_plural_subject_verb_agreement_2" }, "blimp_left_branch_island_echo_question": { "acc,none": 0.542, "acc_stderr,none": 0.015763390640483703, "alias": " - blimp_left_branch_island_echo_question" }, "blimp_left_branch_island_simple_question": { "acc,none": 0.496, "acc_stderr,none": 0.01581879370351089, "alias": " - blimp_left_branch_island_simple_question" }, "blimp_matrix_question_npi_licensor_present": { "acc,none": 0.053, "acc_stderr,none": 0.007088105617246444, "alias": " - blimp_matrix_question_npi_licensor_present" }, "blimp_npi_present_1": { "acc,none": 0.327, "acc_stderr,none": 0.014842213153411247, "alias": " - blimp_npi_present_1" }, "blimp_npi_present_2": { "acc,none": 0.477, "acc_stderr,none": 0.015802554246726098, "alias": " - blimp_npi_present_2" }, "blimp_only_npi_licensor_present": { "acc,none": 0.805, "acc_stderr,none": 0.012535235623319332, "alias": " - blimp_only_npi_licensor_present" }, "blimp_only_npi_scope": { "acc,none": 0.569, "acc_stderr,none": 0.015667944488173505, "alias": " - blimp_only_npi_scope" }, "blimp_passive_1": { "acc,none": 0.832, "acc_stderr,none": 0.011828605831454259, "alias": " - blimp_passive_1" }, "blimp_passive_2": { "acc,none": 0.799, "acc_stderr,none": 0.012679107214617331, "alias": " - blimp_passive_2" }, "blimp_principle_A_c_command": { "acc,none": 0.573, "acc_stderr,none": 0.015649789644462214, "alias": " - blimp_principle_A_c_command" }, "blimp_principle_A_case_1": { "acc,none": 1.0, "acc_stderr,none": 0.0, "alias": " - blimp_principle_A_case_1" }, "blimp_principle_A_case_2": { "acc,none": 0.819, "acc_stderr,none": 0.012181436179177912, "alias": " - blimp_principle_A_case_2" }, "blimp_principle_A_domain_1": { "acc,none": 0.967, "acc_stderr,none": 0.005651808820452374, "alias": " - blimp_principle_A_domain_1" }, "blimp_principle_A_domain_2": { "acc,none": 0.66, "acc_stderr,none": 0.014987482264363937, "alias": " - blimp_principle_A_domain_2" }, "blimp_principle_A_domain_3": { "acc,none": 0.513, "acc_stderr,none": 0.015813952101896626, "alias": " - blimp_principle_A_domain_3" }, "blimp_principle_A_reconstruction": { "acc,none": 0.365, "acc_stderr,none": 0.015231776226264891, "alias": " - blimp_principle_A_reconstruction" }, "blimp_regular_plural_subject_verb_agreement_1": { "acc,none": 0.818, "acc_stderr,none": 0.012207580637662164, "alias": " - blimp_regular_plural_subject_verb_agreement_1" }, "blimp_regular_plural_subject_verb_agreement_2": { "acc,none": 0.693, "acc_stderr,none": 0.01459328489285263, "alias": " - blimp_regular_plural_subject_verb_agreement_2" }, "blimp_sentential_negation_npi_licensor_present": { "acc,none": 0.991, "acc_stderr,none": 0.002987963843142665, "alias": " - blimp_sentential_negation_npi_licensor_present" }, "blimp_sentential_negation_npi_scope": { "acc,none": 0.426, "acc_stderr,none": 0.015645087688113814, "alias": " - blimp_sentential_negation_npi_scope" }, "blimp_sentential_subject_island": { "acc,none": 0.451, "acc_stderr,none": 0.015743152379585533, "alias": " - blimp_sentential_subject_island" }, "blimp_superlative_quantifiers_1": { "acc,none": 0.834, "acc_stderr,none": 0.011772110370812194, "alias": " - blimp_superlative_quantifiers_1" }, "blimp_superlative_quantifiers_2": { "acc,none": 0.682, "acc_stderr,none": 0.014734079309311901, "alias": " - blimp_superlative_quantifiers_2" }, "blimp_tough_vs_raising_1": { "acc,none": 0.348, "acc_stderr,none": 0.01507060460376841, "alias": " - blimp_tough_vs_raising_1" }, "blimp_tough_vs_raising_2": { "acc,none": 0.806, "acc_stderr,none": 0.012510816141264359, "alias": " - blimp_tough_vs_raising_2" }, "blimp_transitive": { "acc,none": 0.721, "acc_stderr,none": 0.014190150117612045, "alias": " - blimp_transitive" }, "blimp_wh_island": { "acc,none": 0.498, "acc_stderr,none": 0.015819173374302706, "alias": " - blimp_wh_island" }, "blimp_wh_questions_object_gap": { "acc,none": 0.588, "acc_stderr,none": 0.015572363292015095, "alias": " - blimp_wh_questions_object_gap" }, "blimp_wh_questions_subject_gap": { "acc,none": 0.872, "acc_stderr,none": 0.010570133761108663, "alias": " - blimp_wh_questions_subject_gap" }, "blimp_wh_questions_subject_gap_long_distance": { "acc,none": 0.923, "acc_stderr,none": 0.008434580140240648, "alias": " - blimp_wh_questions_subject_gap_long_distance" }, "blimp_wh_vs_that_no_gap": { "acc,none": 0.92, "acc_stderr,none": 0.00858333697775365, "alias": " - blimp_wh_vs_that_no_gap" }, "blimp_wh_vs_that_no_gap_long_distance": { "acc,none": 0.962, "acc_stderr,none": 0.0060491811505849384, "alias": " - blimp_wh_vs_that_no_gap_long_distance" }, "blimp_wh_vs_that_with_gap": { "acc,none": 0.26, "acc_stderr,none": 0.013877773329774164, "alias": " - blimp_wh_vs_that_with_gap" }, "blimp_wh_vs_that_with_gap_long_distance": { "acc,none": 0.085, "acc_stderr,none": 0.008823426366942288, "alias": " - blimp_wh_vs_that_with_gap_long_distance" } }, "groups": { "blimp": { "acc,none": 0.6665074626865671, "acc_stderr,none": 0.2001790545943022, "alias": "blimp" } }, "configs": { "blimp_adjunct_island": { "task": "blimp_adjunct_island", "group": "blimp", "dataset_path": "blimp", "dataset_name": "adjunct_island", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_anaphor_gender_agreement": { "task": "blimp_anaphor_gender_agreement", "group": "blimp", "dataset_path": "blimp", "dataset_name": "anaphor_gender_agreement", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_anaphor_number_agreement": { "task": "blimp_anaphor_number_agreement", "group": "blimp", "dataset_path": "blimp", "dataset_name": "anaphor_number_agreement", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_animate_subject_passive": { "task": "blimp_animate_subject_passive", "group": "blimp", "dataset_path": "blimp", "dataset_name": "animate_subject_passive", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_animate_subject_trans": { "task": "blimp_animate_subject_trans", "group": "blimp", "dataset_path": "blimp", "dataset_name": "animate_subject_trans", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_causative": { "task": "blimp_causative", "group": "blimp", "dataset_path": "blimp", "dataset_name": "causative", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_complex_NP_island": { "task": "blimp_complex_NP_island", "group": "blimp", "dataset_path": "blimp", "dataset_name": "complex_NP_island", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_coordinate_structure_constraint_complex_left_branch": { "task": "blimp_coordinate_structure_constraint_complex_left_branch", "group": "blimp", "dataset_path": "blimp", "dataset_name": "coordinate_structure_constraint_complex_left_branch", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_coordinate_structure_constraint_object_extraction": { "task": "blimp_coordinate_structure_constraint_object_extraction", "group": "blimp", "dataset_path": "blimp", "dataset_name": "coordinate_structure_constraint_object_extraction", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_determiner_noun_agreement_1": { "task": "blimp_determiner_noun_agreement_1", "group": "blimp", "dataset_path": "blimp", "dataset_name": "determiner_noun_agreement_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_determiner_noun_agreement_2": { "task": "blimp_determiner_noun_agreement_2", "group": "blimp", "dataset_path": "blimp", "dataset_name": "determiner_noun_agreement_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_determiner_noun_agreement_irregular_1": { "task": "blimp_determiner_noun_agreement_irregular_1", "group": "blimp", "dataset_path": "blimp", "dataset_name": "determiner_noun_agreement_irregular_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_determiner_noun_agreement_irregular_2": { "task": "blimp_determiner_noun_agreement_irregular_2", "group": "blimp", "dataset_path": "blimp", "dataset_name": "determiner_noun_agreement_irregular_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_determiner_noun_agreement_with_adj_2": { "task": "blimp_determiner_noun_agreement_with_adj_2", "group": "blimp", "dataset_path": "blimp", "dataset_name": "determiner_noun_agreement_with_adj_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_determiner_noun_agreement_with_adj_irregular_1": { "task": "blimp_determiner_noun_agreement_with_adj_irregular_1", "group": "blimp", "dataset_path": "blimp", "dataset_name": "determiner_noun_agreement_with_adj_irregular_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_determiner_noun_agreement_with_adj_irregular_2": { "task": "blimp_determiner_noun_agreement_with_adj_irregular_2", "group": "blimp", "dataset_path": "blimp", "dataset_name": "determiner_noun_agreement_with_adj_irregular_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_determiner_noun_agreement_with_adjective_1": { "task": "blimp_determiner_noun_agreement_with_adjective_1", "group": "blimp", "dataset_path": "blimp", "dataset_name": "determiner_noun_agreement_with_adjective_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_distractor_agreement_relational_noun": { "task": "blimp_distractor_agreement_relational_noun", "group": "blimp", "dataset_path": "blimp", "dataset_name": "distractor_agreement_relational_noun", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_distractor_agreement_relative_clause": { "task": "blimp_distractor_agreement_relative_clause", "group": "blimp", "dataset_path": "blimp", "dataset_name": "distractor_agreement_relative_clause", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_drop_argument": { "task": "blimp_drop_argument", "group": "blimp", "dataset_path": "blimp", "dataset_name": "drop_argument", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_ellipsis_n_bar_1": { "task": "blimp_ellipsis_n_bar_1", "group": "blimp", "dataset_path": "blimp", "dataset_name": "ellipsis_n_bar_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_ellipsis_n_bar_2": { "task": "blimp_ellipsis_n_bar_2", "group": "blimp", "dataset_path": "blimp", "dataset_name": "ellipsis_n_bar_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_existential_there_object_raising": { "task": "blimp_existential_there_object_raising", "group": "blimp", "dataset_path": "blimp", "dataset_name": "existential_there_object_raising", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_existential_there_quantifiers_1": { "task": "blimp_existential_there_quantifiers_1", "group": "blimp", "dataset_path": "blimp", "dataset_name": "existential_there_quantifiers_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_existential_there_quantifiers_2": { "task": "blimp_existential_there_quantifiers_2", "group": "blimp", "dataset_path": "blimp", "dataset_name": "existential_there_quantifiers_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_existential_there_subject_raising": { "task": "blimp_existential_there_subject_raising", "group": "blimp", "dataset_path": "blimp", "dataset_name": "existential_there_subject_raising", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_expletive_it_object_raising": { "task": "blimp_expletive_it_object_raising", "group": "blimp", "dataset_path": "blimp", "dataset_name": "expletive_it_object_raising", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_inchoative": { "task": "blimp_inchoative", "group": "blimp", "dataset_path": "blimp", "dataset_name": "inchoative", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_intransitive": { "task": "blimp_intransitive", "group": "blimp", "dataset_path": "blimp", "dataset_name": "intransitive", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_irregular_past_participle_adjectives": { "task": "blimp_irregular_past_participle_adjectives", "group": "blimp", "dataset_path": "blimp", "dataset_name": "irregular_past_participle_adjectives", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_irregular_past_participle_verbs": { "task": "blimp_irregular_past_participle_verbs", "group": "blimp", "dataset_path": "blimp", "dataset_name": "irregular_past_participle_verbs", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_irregular_plural_subject_verb_agreement_1": { "task": "blimp_irregular_plural_subject_verb_agreement_1", "group": "blimp", "dataset_path": "blimp", "dataset_name": "irregular_plural_subject_verb_agreement_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_irregular_plural_subject_verb_agreement_2": { "task": "blimp_irregular_plural_subject_verb_agreement_2", "group": "blimp", "dataset_path": "blimp", "dataset_name": "irregular_plural_subject_verb_agreement_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_left_branch_island_echo_question": { "task": "blimp_left_branch_island_echo_question", "group": "blimp", "dataset_path": "blimp", "dataset_name": "left_branch_island_echo_question", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_left_branch_island_simple_question": { "task": "blimp_left_branch_island_simple_question", "group": "blimp", "dataset_path": "blimp", "dataset_name": "left_branch_island_simple_question", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_matrix_question_npi_licensor_present": { "task": "blimp_matrix_question_npi_licensor_present", "group": "blimp", "dataset_path": "blimp", "dataset_name": "matrix_question_npi_licensor_present", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_npi_present_1": { "task": "blimp_npi_present_1", "group": "blimp", "dataset_path": "blimp", "dataset_name": "npi_present_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_npi_present_2": { "task": "blimp_npi_present_2", "group": "blimp", "dataset_path": "blimp", "dataset_name": "npi_present_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_only_npi_licensor_present": { "task": "blimp_only_npi_licensor_present", "group": "blimp", "dataset_path": "blimp", "dataset_name": "only_npi_licensor_present", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_only_npi_scope": { "task": "blimp_only_npi_scope", "group": "blimp", "dataset_path": "blimp", "dataset_name": "only_npi_scope", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_passive_1": { "task": "blimp_passive_1", "group": "blimp", "dataset_path": "blimp", "dataset_name": "passive_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_passive_2": { "task": "blimp_passive_2", "group": "blimp", "dataset_path": "blimp", "dataset_name": "passive_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_principle_A_c_command": { "task": "blimp_principle_A_c_command", "group": "blimp", "dataset_path": "blimp", "dataset_name": "principle_A_c_command", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_principle_A_case_1": { "task": "blimp_principle_A_case_1", "group": "blimp", "dataset_path": "blimp", "dataset_name": "principle_A_case_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_principle_A_case_2": { "task": "blimp_principle_A_case_2", "group": "blimp", "dataset_path": "blimp", "dataset_name": "principle_A_case_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_principle_A_domain_1": { "task": "blimp_principle_A_domain_1", "group": "blimp", "dataset_path": "blimp", "dataset_name": "principle_A_domain_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_principle_A_domain_2": { "task": "blimp_principle_A_domain_2", "group": "blimp", "dataset_path": "blimp", "dataset_name": "principle_A_domain_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_principle_A_domain_3": { "task": "blimp_principle_A_domain_3", "group": "blimp", "dataset_path": "blimp", "dataset_name": "principle_A_domain_3", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_principle_A_reconstruction": { "task": "blimp_principle_A_reconstruction", "group": "blimp", "dataset_path": "blimp", "dataset_name": "principle_A_reconstruction", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_regular_plural_subject_verb_agreement_1": { "task": "blimp_regular_plural_subject_verb_agreement_1", "group": "blimp", "dataset_path": "blimp", "dataset_name": "regular_plural_subject_verb_agreement_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_regular_plural_subject_verb_agreement_2": { "task": "blimp_regular_plural_subject_verb_agreement_2", "group": "blimp", "dataset_path": "blimp", "dataset_name": "regular_plural_subject_verb_agreement_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_sentential_negation_npi_licensor_present": { "task": "blimp_sentential_negation_npi_licensor_present", "group": "blimp", "dataset_path": "blimp", "dataset_name": "sentential_negation_npi_licensor_present", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_sentential_negation_npi_scope": { "task": "blimp_sentential_negation_npi_scope", "group": "blimp", "dataset_path": "blimp", "dataset_name": "sentential_negation_npi_scope", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_sentential_subject_island": { "task": "blimp_sentential_subject_island", "group": "blimp", "dataset_path": "blimp", "dataset_name": "sentential_subject_island", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_superlative_quantifiers_1": { "task": "blimp_superlative_quantifiers_1", "group": "blimp", "dataset_path": "blimp", "dataset_name": "superlative_quantifiers_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_superlative_quantifiers_2": { "task": "blimp_superlative_quantifiers_2", "group": "blimp", "dataset_path": "blimp", "dataset_name": "superlative_quantifiers_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_tough_vs_raising_1": { "task": "blimp_tough_vs_raising_1", "group": "blimp", "dataset_path": "blimp", "dataset_name": "tough_vs_raising_1", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_tough_vs_raising_2": { "task": "blimp_tough_vs_raising_2", "group": "blimp", "dataset_path": "blimp", "dataset_name": "tough_vs_raising_2", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_transitive": { "task": "blimp_transitive", "group": "blimp", "dataset_path": "blimp", "dataset_name": "transitive", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_wh_island": { "task": "blimp_wh_island", "group": "blimp", "dataset_path": "blimp", "dataset_name": "wh_island", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_wh_questions_object_gap": { "task": "blimp_wh_questions_object_gap", "group": "blimp", "dataset_path": "blimp", "dataset_name": "wh_questions_object_gap", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_wh_questions_subject_gap": { "task": "blimp_wh_questions_subject_gap", "group": "blimp", "dataset_path": "blimp", "dataset_name": "wh_questions_subject_gap", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_wh_questions_subject_gap_long_distance": { "task": "blimp_wh_questions_subject_gap_long_distance", "group": "blimp", "dataset_path": "blimp", "dataset_name": "wh_questions_subject_gap_long_distance", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_wh_vs_that_no_gap": { "task": "blimp_wh_vs_that_no_gap", "group": "blimp", "dataset_path": "blimp", "dataset_name": "wh_vs_that_no_gap", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_wh_vs_that_no_gap_long_distance": { "task": "blimp_wh_vs_that_no_gap_long_distance", "group": "blimp", "dataset_path": "blimp", "dataset_name": "wh_vs_that_no_gap_long_distance", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_wh_vs_that_with_gap": { "task": "blimp_wh_vs_that_with_gap", "group": "blimp", "dataset_path": "blimp", "dataset_name": "wh_vs_that_with_gap", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } }, "blimp_wh_vs_that_with_gap_long_distance": { "task": "blimp_wh_vs_that_with_gap_long_distance", "group": "blimp", "dataset_path": "blimp", "dataset_name": "wh_vs_that_with_gap_long_distance", "validation_split": "train", "doc_to_text": "", "doc_to_target": 0, "doc_to_choice": "{{[sentence_good, sentence_bad]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc" } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}", "metadata": { "version": 1.0 } } }, "versions": { "blimp": "N/A", "blimp_adjunct_island": 1.0, "blimp_anaphor_gender_agreement": 1.0, "blimp_anaphor_number_agreement": 1.0, "blimp_animate_subject_passive": 1.0, "blimp_animate_subject_trans": 1.0, "blimp_causative": 1.0, "blimp_complex_NP_island": 1.0, "blimp_coordinate_structure_constraint_complex_left_branch": 1.0, "blimp_coordinate_structure_constraint_object_extraction": 1.0, "blimp_determiner_noun_agreement_1": 1.0, "blimp_determiner_noun_agreement_2": 1.0, "blimp_determiner_noun_agreement_irregular_1": 1.0, "blimp_determiner_noun_agreement_irregular_2": 1.0, "blimp_determiner_noun_agreement_with_adj_2": 1.0, "blimp_determiner_noun_agreement_with_adj_irregular_1": 1.0, "blimp_determiner_noun_agreement_with_adj_irregular_2": 1.0, "blimp_determiner_noun_agreement_with_adjective_1": 1.0, "blimp_distractor_agreement_relational_noun": 1.0, "blimp_distractor_agreement_relative_clause": 1.0, "blimp_drop_argument": 1.0, "blimp_ellipsis_n_bar_1": 1.0, "blimp_ellipsis_n_bar_2": 1.0, "blimp_existential_there_object_raising": 1.0, "blimp_existential_there_quantifiers_1": 1.0, "blimp_existential_there_quantifiers_2": 1.0, "blimp_existential_there_subject_raising": 1.0, "blimp_expletive_it_object_raising": 1.0, "blimp_inchoative": 1.0, "blimp_intransitive": 1.0, "blimp_irregular_past_participle_adjectives": 1.0, "blimp_irregular_past_participle_verbs": 1.0, "blimp_irregular_plural_subject_verb_agreement_1": 1.0, "blimp_irregular_plural_subject_verb_agreement_2": 1.0, "blimp_left_branch_island_echo_question": 1.0, "blimp_left_branch_island_simple_question": 1.0, "blimp_matrix_question_npi_licensor_present": 1.0, "blimp_npi_present_1": 1.0, "blimp_npi_present_2": 1.0, "blimp_only_npi_licensor_present": 1.0, "blimp_only_npi_scope": 1.0, "blimp_passive_1": 1.0, "blimp_passive_2": 1.0, "blimp_principle_A_c_command": 1.0, "blimp_principle_A_case_1": 1.0, "blimp_principle_A_case_2": 1.0, "blimp_principle_A_domain_1": 1.0, "blimp_principle_A_domain_2": 1.0, "blimp_principle_A_domain_3": 1.0, "blimp_principle_A_reconstruction": 1.0, "blimp_regular_plural_subject_verb_agreement_1": 1.0, "blimp_regular_plural_subject_verb_agreement_2": 1.0, "blimp_sentential_negation_npi_licensor_present": 1.0, "blimp_sentential_negation_npi_scope": 1.0, "blimp_sentential_subject_island": 1.0, "blimp_superlative_quantifiers_1": 1.0, "blimp_superlative_quantifiers_2": 1.0, "blimp_tough_vs_raising_1": 1.0, "blimp_tough_vs_raising_2": 1.0, "blimp_transitive": 1.0, "blimp_wh_island": 1.0, "blimp_wh_questions_object_gap": 1.0, "blimp_wh_questions_subject_gap": 1.0, "blimp_wh_questions_subject_gap_long_distance": 1.0, "blimp_wh_vs_that_no_gap": 1.0, "blimp_wh_vs_that_no_gap_long_distance": 1.0, "blimp_wh_vs_that_with_gap": 1.0, "blimp_wh_vs_that_with_gap_long_distance": 1.0 }, "n-shot": { "blimp": 0, "blimp_adjunct_island": 0, "blimp_anaphor_gender_agreement": 0, "blimp_anaphor_number_agreement": 0, "blimp_animate_subject_passive": 0, "blimp_animate_subject_trans": 0, "blimp_causative": 0, "blimp_complex_NP_island": 0, "blimp_coordinate_structure_constraint_complex_left_branch": 0, "blimp_coordinate_structure_constraint_object_extraction": 0, "blimp_determiner_noun_agreement_1": 0, "blimp_determiner_noun_agreement_2": 0, "blimp_determiner_noun_agreement_irregular_1": 0, "blimp_determiner_noun_agreement_irregular_2": 0, "blimp_determiner_noun_agreement_with_adj_2": 0, "blimp_determiner_noun_agreement_with_adj_irregular_1": 0, "blimp_determiner_noun_agreement_with_adj_irregular_2": 0, "blimp_determiner_noun_agreement_with_adjective_1": 0, "blimp_distractor_agreement_relational_noun": 0, "blimp_distractor_agreement_relative_clause": 0, "blimp_drop_argument": 0, "blimp_ellipsis_n_bar_1": 0, "blimp_ellipsis_n_bar_2": 0, "blimp_existential_there_object_raising": 0, "blimp_existential_there_quantifiers_1": 0, "blimp_existential_there_quantifiers_2": 0, "blimp_existential_there_subject_raising": 0, "blimp_expletive_it_object_raising": 0, "blimp_inchoative": 0, "blimp_intransitive": 0, "blimp_irregular_past_participle_adjectives": 0, "blimp_irregular_past_participle_verbs": 0, "blimp_irregular_plural_subject_verb_agreement_1": 0, "blimp_irregular_plural_subject_verb_agreement_2": 0, "blimp_left_branch_island_echo_question": 0, "blimp_left_branch_island_simple_question": 0, "blimp_matrix_question_npi_licensor_present": 0, "blimp_npi_present_1": 0, "blimp_npi_present_2": 0, "blimp_only_npi_licensor_present": 0, "blimp_only_npi_scope": 0, "blimp_passive_1": 0, "blimp_passive_2": 0, "blimp_principle_A_c_command": 0, "blimp_principle_A_case_1": 0, "blimp_principle_A_case_2": 0, "blimp_principle_A_domain_1": 0, "blimp_principle_A_domain_2": 0, "blimp_principle_A_domain_3": 0, "blimp_principle_A_reconstruction": 0, "blimp_regular_plural_subject_verb_agreement_1": 0, "blimp_regular_plural_subject_verb_agreement_2": 0, "blimp_sentential_negation_npi_licensor_present": 0, "blimp_sentential_negation_npi_scope": 0, "blimp_sentential_subject_island": 0, "blimp_superlative_quantifiers_1": 0, "blimp_superlative_quantifiers_2": 0, "blimp_tough_vs_raising_1": 0, "blimp_tough_vs_raising_2": 0, "blimp_transitive": 0, "blimp_wh_island": 0, "blimp_wh_questions_object_gap": 0, "blimp_wh_questions_subject_gap": 0, "blimp_wh_questions_subject_gap_long_distance": 0, "blimp_wh_vs_that_no_gap": 0, "blimp_wh_vs_that_no_gap_long_distance": 0, "blimp_wh_vs_that_with_gap": 0, "blimp_wh_vs_that_with_gap_long_distance": 0 }, "config": { "model": "hf", "model_args": "pretrained=/home/bastian/Dokumente/baby_llamas/models/final_20", "batch_size": 1, "batch_sizes": [], "device": "cuda", "use_cache": null, "limit": null, "bootstrap_iters": 100000, "gen_kwargs": null }, "git_hash": null }