#!/bin/bash #SBATCH --job-name=evaluate_t0 #SBATCH --nodes=1 #SBATCH --ntasks-per-node=1 # crucial - only 1 task per dist per node! #SBATCH --cpus-per-task=32 # number of cores per tasks #SBATCH --hint=nomultithread # we get physical cores not logical #SBATCH --gpus-per-node=mi250:1 #SBATCH --time 5:00:00 # maximum execution time (HH:MM:SS) #SBATCH --output=logs/%j.out # output file name #SBATCH -e logs/%j.err #SBATCH --account=project_462000119 #SBATCH -p pilot #SBATCH --exclusive=user # VALIDATION: # --array=0-168 # L1 # --array=0-169 # L2 # --array=0-84 # MT L1 # --array=0-69 # MT L2 # --array=0-89 # XNLIMTHT: # --array=0-79 set -x -e # source $six_ALL_CCFRWORK/start-py38-pt111 # conda activate thomas_t_zero_evaluation source /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/bin/activate # CHECKPOINT_PATH=/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3/bloom-6b3 # CHECKPOINT_PATH=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-mtf/global_step1000 # CHECKPOINT_PATH=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-mtf/bloomz7b1 CHECKPOINT_PATH=bloomz-560m CHECKPOINT_PATH=bloomz-7b1-500m-ru CHECKPOINT_PATH=bloomz-7b1-100m-ru CHECKPOINT_PATH=bloomz-7b1-xp3ru CHECKPOINT_PATH=bloomz-7b1-4b-ru # WORKDIR=/gpfswork/rech/six/commun/code/tr13f-6B3-ml-t0 # pushd $WORKDIR export TRANSFORMERS_OFFLINE=1 export HF_TRANSFORMERS_OFFLINE=1 export HF_DATASETS_OFFLINE=1 export HF_DATASETS_CACHE=/scratch/project_462000119/ds_cache OUTPUT_DIR=$CHECKPOINT_PATH/evaluation mkdir -p $OUTPUT_DIR # Validation DATASETS_AND_CONFIGS_VAL=( head_qa,en,en,"multiple_choice_q_and_a_index_with_context_en",validation head_qa,en,en,"multiple_choice_q_and_a_en",validation head_qa,en,en,"multiple_choice_q_and_a_index_en",validation head_qa,en,en,"multiple_choice_a_and_q_with_context_en",validation head_qa,en,en,"multiple_choice_a_and_q_en",validation head_qa,es,en,"multiple_choice_q_and_a_index_with_context_en",validation head_qa,es,en,"multiple_choice_q_and_a_en",validation head_qa,es,en,"multiple_choice_q_and_a_index_en",validation head_qa,es,en,"multiple_choice_a_and_q_with_context_en",validation head_qa,es,en,"multiple_choice_a_and_q_en",validation climate_fever,None,None,"first_evidence_and_claim_itemization",test climate_fever,None,None,"claim_and_all_supporting_evidences",test climate_fever,None,None,"fifth_evidence_and_claim_itemization",test climate_fever,None,None,"third_evidence_claim_pair",test climate_fever,None,None,"second_evidence_and_claim_itemization",test codah,codah,None,"interrogative_instruction_after_sentence_and_choices",train codah,codah,None,"affirmative_instruction_before_sentence_and_choices",train codah,codah,None,"affirmative_instruction_after_sentence_and_choices",train aqua_rat,raw,None,"select_the_best_option",validation aqua_rat,raw,None,"answer_quiz",validation aqua_rat,raw,None,"Answer questions from options",validation commonsense_qa,None,None,"answer_given_question_without_options",validation commonsense_qa,None,None,"question_answering",validation commonsense_qa,None,None,"most_suitable_answer",validation amazon_reviews_multi,en,en,"prompt_title_to_star",validation amazon_reviews_multi,en,en,"prompt_review_to_star",validation amazon_reviews_multi,en,en,"prompt_body_title_to_star",validation amazon_reviews_multi,zh,en,"prompt_title_to_star",validation amazon_reviews_multi,zh,en,"prompt_review_to_star",validation amazon_reviews_multi,zh,en,"prompt_body_title_to_star",validation amazon_reviews_multi,fr,en,"prompt_title_to_star",validation amazon_reviews_multi,fr,en,"prompt_review_to_star",validation amazon_reviews_multi,fr,en,"prompt_body_title_to_star",validation amazon_reviews_multi,es,en,"prompt_title_to_star",validation amazon_reviews_multi,es,en,"prompt_review_to_star",validation amazon_reviews_multi,es,en,"prompt_body_title_to_star",validation art,None,None,"choose_hypothesis_options",validation art,None,None,"choose_hypothesis_believable",validation art,None,None,"choose_hypothesis",validation art,None,None,"choose_hypothesis_desc",validation art,None,None,"choose_hypothesis_likely",validation banking77,None,None,"help_page_topic",test banking77,None,None,"direct_to_which_department",test banking77,None,None,"rephrase_as_banking_term",test blbooksgenre,title_genre_classifiction,None,"multi-choice",train blbooksgenre,title_genre_classifiction,None,"premise_context_first",train blbooksgenre,title_genre_classifiction,None,"classify",train blimp,adjunct_island,None,"grammatical_between_1_2",train blimp,adjunct_island,None,"grammatical_between_A_B",train blimp,adjunct_island,None,"grammatical_which_one_1_2",train blimp,adjunct_island,None,"single_sentence_bad_yes_no",train blimp,adjunct_island,None,"single_sentence_good_yes_no",train conv_ai_3,None,None,"clarification_needed",validation conv_ai_3,None,None,"score_give_number",validation conv_ai_3,None,None,"ambiguous",validation conv_ai_3,None,None,"directly_answer",validation conv_ai_3,None,None,"score_how_much",validation craigslist_bargains,None,None,"good deal for seller no list price implicit",validation craigslist_bargains,None,None,"good deal for seller no list price",validation craigslist_bargains,None,None,"good deal for seller",validation craigslist_bargains,None,None,"best deal",validation ecthr_cases,alleged-violation-prediction,None,"implicit_advice_number",validation ecthr_cases,alleged-violation-prediction,None,"ecthr_alleged_articles_declaration_at_end",validation ecthr_cases,alleged-violation-prediction,None,"ecthr_alleged_articles_question_at_start",validation ecthr_cases,alleged-violation-prediction,None,"implicit_judgment_paragraph",validation ecthr_cases,alleged-violation-prediction,None,"confirm number of violated articles",validation emo,None,None,"persons_describe",validation emo,None,None,"final_message",validation emo,None,None,"what_emotion_do_you_think",validation emo,None,None,"emotional_state",validation emo,None,None,"dialogue_between",validation emotion,None,None,"choose_the_best_emotion_label",test emotion,None,None,"reply_with_emoation_label",test emotion,None,None,"answer_with_class_label",test emotion,None,None,"answer_question_with_emotion_label",test financial_phrasebank,sentences_allagree,None,"share_price_option",train financial_phrasebank,sentences_allagree,None,"sentiment",train financial_phrasebank,sentences_allagree,None,"word_comes_to_mind",train financial_phrasebank,sentences_allagree,None,"complementary_industries",train financial_phrasebank,sentences_allagree,None,"bullish_neutral_bearish",train glue,cola,None,"Make sense yes no",validation glue,cola,None,"is_this_correct",validation glue,cola,None,"editing",validation glue,cola,None,"Following sentence acceptable",validation glue,cola,None,"Previous sentence acceptable",validation glue,sst2,None,"positive negative after",validation glue,sst2,None,"review",validation glue,sst2,None,"said",validation glue,sst2,None,"following positive negative",validation glue,sst2,None,"happy or mad",validation health_fact,None,None,"claim_veracity_classification_after_reading_I_believe",validation health_fact,None,None,"claim_explanation_classification",validation health_fact,None,None,"claim_veracity_classification_tell_me",validation hlgd,None,None,"is_same_event_with_time_interrogative_related",validation hlgd,None,None,"is_same_event_interrogative_talk",validation hlgd,None,None,"is_same_event_with_time_interrogative_talk",validation hlgd,None,None,"is_same_event_refer",validation hlgd,None,None,"is_same_event_editor_asks",validation hyperpartisan_news_detection,byarticle,None,"consider_does_it_follow_a_hyperpartisan_argumentation",train hyperpartisan_news_detection,byarticle,None,"follows_hyperpartisan_argumentation",train hyperpartisan_news_detection,byarticle,None,"consume_with_caution",train hyperpartisan_news_detection,byarticle,None,"extreme_left_wing_or_right_wing",train hyperpartisan_news_detection,byarticle,None,"consider_it_exhibits_extreme_one_sidedness",train liar,None,None,"Given statement guess category",validation lince,sa_spaeng,None,"original poster expressed sentiment",validation lince,sa_spaeng,None,"sentiment trying to express",validation lince,sa_spaeng,None,"express sentiment",validation lince,sa_spaeng,None,"negation template",validation lince,sa_spaeng,None,"the author seem",validation math_qa,None,None,"choose_correct_og",test math_qa,None,None,"pick_the_correct",test math_qa,None,None,"first_choice_then_problem",test math_qa,None,None,"problem_set_type",test math_qa,None,None,"gre_problem",test movie_rationales,None,None,"Standard binary sentiment analysis",validation movie_rationales,None,None,"Evidences sentiment classification",validation movie_rationales,None,None,"Evidences + review",validation movie_rationales,None,None,"Generate evidences and sentiment",validation mwsc,None,None,"in-the-sentence-question-first",validation mwsc,None,None,"what-think",validation mwsc,None,None,"in-the-sentence",validation mwsc,None,None,"options-or",validation mwsc,None,None,"is-correct",validation poem_sentiment,None,None,"positive_or_negative_sentiment_variation_2",validation poem_sentiment,None,None,"question_answer_format",validation poem_sentiment,None,None,"guess_sentiment_without_options_variation_1",validation poem_sentiment,None,None,"positive_or_negative_sentiment_variation_1",validation poem_sentiment,None,None,"most_appropriate_sentiment",validation onestop_english,None,None,"esl_context",train onestop_english,None,None,"ara_context",train onestop_english,None,None,"determine_reading_level_from_the_first_three_sentences",train onestop_english,None,None,"esl_variation",train onestop_english,None,None,"assess",train pubmed_qa,pqa_labeled,None,"Long Answer to Final Decision",train pubmed_qa,pqa_labeled,None,"Question Answering (Short)",train riddle_sense,None,None,"most_suitable_answer",validation riddle_sense,None,None,"answer_given_question_without_options",validation riddle_sense,None,None,"question_to_answer_index",validation riddle_sense,None,None,"question_answering",validation scicite,None,None,"Classify intent w/section (select choice)",validation scicite,None,None,"Classify intent (choices first)",validation scicite,None,None,"Classify intent (select choice)",validation scicite,None,None,"Classify intent",validation scicite,None,None,"can_describe",validation selqa,answer_selection_analysis,None,"is-he-talking-about",validation selqa,answer_selection_analysis,None,"would-make-sense-qu-rand",validation selqa,answer_selection_analysis,None,"make-sense-rand",validation selqa,answer_selection_analysis,None,"which-answer-1st-vs-random",validation snips_built_in_intents,None,None,"voice_intent",train snips_built_in_intents,None,None,"categorize_query",train snips_built_in_intents,None,None,"intent_query",train snips_built_in_intents,None,None,"categorize_query_brief",train snips_built_in_intents,None,None,"query_intent",train ) DATASETS_AND_CONFIGS_L1MISS=( story_cloze,2016,None,"Story Continuation and Options",validation story_cloze,2016,None,"Answer Given options",validation story_cloze,2016,None,"Novel Correct Ending",validation story_cloze,2016,None,"Generate Ending",validation story_cloze,2016,None,"Choose Story Ending",validation ) DATASETS_AND_CONFIGS_L1=( super_glue,copa,None,"best_option",validation super_glue,copa,None,"C1 or C2? premise, so/because…",validation super_glue,copa,None,"i_am_hesitating",validation super_glue,copa,None,"cause_effect",validation super_glue,copa,None,"plausible_alternatives",validation super_glue,rte,None,"MNLI crowdsource",validation super_glue,rte,None,"GPT-3 style",validation super_glue,rte,None,"does it follow that",validation super_glue,rte,None,"should assume",validation super_glue,rte,None,"guaranteed true",validation anli,dev_r1,None,"guaranteed/possible/impossible",dev_r1 anli,dev_r1,None,"MNLI crowdsource",dev_r1 anli,dev_r1,None,"GPT-3 style",dev_r1 anli,dev_r1,None,"justified in saying",dev_r1 anli,dev_r1,None,"can we infer",dev_r1 anli,dev_r2,None,"guaranteed/possible/impossible",dev_r2 anli,dev_r2,None,"MNLI crowdsource",dev_r2 anli,dev_r2,None,"GPT-3 style",dev_r2 anli,dev_r2,None,"justified in saying",dev_r2 anli,dev_r2,None,"can we infer",dev_r2 anli,dev_r3,None,"guaranteed/possible/impossible",dev_r3 anli,dev_r3,None,"MNLI crowdsource",dev_r3 anli,dev_r3,None,"GPT-3 style",dev_r3 anli,dev_r3,None,"justified in saying",dev_r3 anli,dev_r3,None,"can we infer",dev_r3 super_glue,cb,None,"guaranteed/possible/impossible",validation super_glue,cb,None,"MNLI crowdsource",validation super_glue,cb,None,"GPT-3 style",validation super_glue,cb,None,"justified in saying",validation super_glue,cb,None,"can we infer",validation winogrande,winogrande_xl,None,"underscore refer to",validation winogrande,winogrande_xl,None,"Replace",validation winogrande,winogrande_xl,None,"stand for",validation winogrande,winogrande_xl,None,"does underscore refer to",validation winogrande,winogrande_xl,None,"True or False",validation story_cloze,2016,None,"Story Continuation and Options",validation story_cloze,2016,None,"Answer Given options",validation story_cloze,2016,None,"Novel Correct Ending",validation story_cloze,2016,None,"Generate Ending",validation story_cloze,2016,None,"Choose Story Ending",validation Muennighoff/xstory_cloze,ar,en,"Story Continuation and Options",validation Muennighoff/xstory_cloze,ar,en,"Answer Given options",validation Muennighoff/xstory_cloze,ar,en,"Novel Correct Ending",validation Muennighoff/xstory_cloze,ar,en,"Generate Ending",validation Muennighoff/xstory_cloze,ar,en,"Choose Story Ending",validation Muennighoff/xstory_cloze,es,en,"Story Continuation and Options",validation Muennighoff/xstory_cloze,es,en,"Answer Given options",validation Muennighoff/xstory_cloze,es,en,"Novel Correct Ending",validation Muennighoff/xstory_cloze,es,en,"Generate Ending",validation Muennighoff/xstory_cloze,es,en,"Choose Story Ending",validation Muennighoff/xstory_cloze,eu,en,"Story Continuation and Options",validation Muennighoff/xstory_cloze,eu,en,"Answer Given options",validation Muennighoff/xstory_cloze,eu,en,"Novel Correct Ending",validation Muennighoff/xstory_cloze,eu,en,"Generate Ending",validation Muennighoff/xstory_cloze,eu,en,"Choose Story Ending",validation Muennighoff/xstory_cloze,id,en,"Story Continuation and Options",validation Muennighoff/xstory_cloze,id,en,"Answer Given options",validation Muennighoff/xstory_cloze,id,en,"Novel Correct Ending",validation Muennighoff/xstory_cloze,id,en,"Generate Ending",validation Muennighoff/xstory_cloze,id,en,"Choose Story Ending",validation Muennighoff/xstory_cloze,hi,en,"Story Continuation and Options",validation Muennighoff/xstory_cloze,hi,en,"Answer Given options",validation Muennighoff/xstory_cloze,hi,en,"Novel Correct Ending",validation Muennighoff/xstory_cloze,hi,en,"Generate Ending",validation Muennighoff/xstory_cloze,hi,en,"Choose Story Ending",validation Muennighoff/xstory_cloze,sw,en,"Story Continuation and Options",validation Muennighoff/xstory_cloze,sw,en,"Answer Given options",validation Muennighoff/xstory_cloze,sw,en,"Novel Correct Ending",validation Muennighoff/xstory_cloze,sw,en,"Generate Ending",validation Muennighoff/xstory_cloze,sw,en,"Choose Story Ending",validation Muennighoff/xstory_cloze,te,en,"Story Continuation and Options",validation Muennighoff/xstory_cloze,te,en,"Answer Given options",validation Muennighoff/xstory_cloze,te,en,"Novel Correct Ending",validation Muennighoff/xstory_cloze,te,en,"Generate Ending",validation Muennighoff/xstory_cloze,te,en,"Choose Story Ending",validation Muennighoff/xstory_cloze,zh,en,"Story Continuation and Options",validation Muennighoff/xstory_cloze,zh,en,"Answer Given options",validation Muennighoff/xstory_cloze,zh,en,"Novel Correct Ending",validation Muennighoff/xstory_cloze,zh,en,"Generate Ending",validation Muennighoff/xstory_cloze,zh,en,"Choose Story Ending",validation xnli,ar,en,"guaranteed/possible/impossible",validation xnli,ar,en,"MNLI crowdsource",validation xnli,ar,en,"GPT-3 style",validation xnli,ar,en,"justified in saying",validation xnli,ar,en,"can we infer",validation xnli,en,en,"guaranteed/possible/impossible",validation xnli,en,en,"MNLI crowdsource",validation xnli,en,en,"GPT-3 style",validation xnli,en,en,"justified in saying",validation xnli,en,en,"can we infer",validation xnli,es,en,"guaranteed/possible/impossible",validation xnli,es,en,"MNLI crowdsource",validation xnli,es,en,"GPT-3 style",validation xnli,es,en,"justified in saying",validation xnli,es,en,"can we infer",validation xnli,fr,en,"guaranteed/possible/impossible",validation xnli,fr,en,"MNLI crowdsource",validation xnli,fr,en,"GPT-3 style",validation xnli,fr,en,"justified in saying",validation xnli,fr,en,"can we infer",validation xnli,hi,en,"guaranteed/possible/impossible",validation xnli,hi,en,"MNLI crowdsource",validation xnli,hi,en,"GPT-3 style",validation xnli,hi,en,"justified in saying",validation xnli,hi,en,"can we infer",validation xnli,sw,en,"guaranteed/possible/impossible",validation xnli,sw,en,"MNLI crowdsource",validation xnli,sw,en,"GPT-3 style",validation xnli,sw,en,"justified in saying",validation xnli,sw,en,"can we infer",validation xnli,ur,en,"guaranteed/possible/impossible",validation xnli,ur,en,"MNLI crowdsource",validation xnli,ur,en,"GPT-3 style",validation xnli,ur,en,"justified in saying",validation xnli,ur,en,"can we infer",validation xnli,vi,en,"guaranteed/possible/impossible",validation xnli,vi,en,"MNLI crowdsource",validation xnli,vi,en,"GPT-3 style",validation xnli,vi,en,"justified in saying",validation xnli,vi,en,"can we infer",validation xnli,zh,en,"guaranteed/possible/impossible",validation xnli,zh,en,"MNLI crowdsource",validation xnli,zh,en,"GPT-3 style",validation xnli,zh,en,"justified in saying",validation xnli,zh,en,"can we infer",validation xcopa,id,en,"best_option",validation xcopa,id,en,"C1 or C2? premise, so/because…",validation xcopa,id,en,"i_am_hesitating",validation xcopa,id,en,"cause_effect",validation xcopa,id,en,"plausible_alternatives",validation xcopa,sw,en,"best_option",validation xcopa,sw,en,"C1 or C2? premise, so/because…",validation xcopa,sw,en,"i_am_hesitating",validation xcopa,sw,en,"cause_effect",validation xcopa,sw,en,"plausible_alternatives",validation xcopa,ta,en,"best_option",validation xcopa,ta,en,"C1 or C2? premise, so/because…",validation xcopa,ta,en,"i_am_hesitating",validation xcopa,ta,en,"cause_effect",validation xcopa,ta,en,"plausible_alternatives",validation xcopa,vi,en,"best_option",validation xcopa,vi,en,"C1 or C2? premise, so/because…",validation xcopa,vi,en,"i_am_hesitating",validation xcopa,vi,en,"cause_effect",validation xcopa,vi,en,"plausible_alternatives",validation xcopa,zh,en,"best_option",validation xcopa,zh,en,"C1 or C2? premise, so/because…",validation xcopa,zh,en,"i_am_hesitating",validation xcopa,zh,en,"cause_effect",validation xcopa,zh,en,"plausible_alternatives",validation Muennighoff/xwinograd,en,en,"underscore refer to",test Muennighoff/xwinograd,en,en,"Replace",test Muennighoff/xwinograd,en,en,"stand for",test Muennighoff/xwinograd,en,en,"does underscore refer to",test Muennighoff/xwinograd,en,en,"True or False",test Muennighoff/xwinograd,fr,en,"underscore refer to",test Muennighoff/xwinograd,fr,en,"Replace",test Muennighoff/xwinograd,fr,en,"stand for",test Muennighoff/xwinograd,fr,en,"does underscore refer to",test Muennighoff/xwinograd,fr,en,"True or False",test Muennighoff/xwinograd,pt,en,"underscore refer to",test Muennighoff/xwinograd,pt,en,"Replace",test Muennighoff/xwinograd,pt,en,"stand for",test Muennighoff/xwinograd,pt,en,"does underscore refer to",test Muennighoff/xwinograd,pt,en,"True or False",test Muennighoff/xwinograd,zh,en,"underscore refer to",test Muennighoff/xwinograd,zh,en,"Replace",test Muennighoff/xwinograd,zh,en,"stand for",test Muennighoff/xwinograd,zh,en,"does underscore refer to",test Muennighoff/xwinograd,zh,en,"True or False",test ) DATASETS_AND_CONFIGS_L2=( Muennighoff/xstory_cloze,ru,en,"Story Continuation and Options",validation Muennighoff/xstory_cloze,ru,en,"Answer Given options",validation Muennighoff/xstory_cloze,ru,en,"Novel Correct Ending",validation Muennighoff/xstory_cloze,ru,en,"Generate Ending",validation Muennighoff/xstory_cloze,ru,en,"Choose Story Ending",validation Muennighoff/xstory_cloze,my,en,"Story Continuation and Options",validation Muennighoff/xstory_cloze,my,en,"Answer Given options",validation Muennighoff/xstory_cloze,my,en,"Novel Correct Ending",validation Muennighoff/xstory_cloze,my,en,"Generate Ending",validation Muennighoff/xstory_cloze,my,en,"Choose Story Ending",validation xnli,bg,en,"guaranteed/possible/impossible",validation xnli,bg,en,"MNLI crowdsource",validation xnli,bg,en,"GPT-3 style",validation xnli,bg,en,"justified in saying",validation xnli,bg,en,"can we infer",validation xnli,de,en,"guaranteed/possible/impossible",validation xnli,de,en,"MNLI crowdsource",validation xnli,de,en,"GPT-3 style",validation xnli,de,en,"justified in saying",validation xnli,de,en,"can we infer",validation xnli,el,en,"guaranteed/possible/impossible",validation xnli,el,en,"MNLI crowdsource",validation xnli,el,en,"GPT-3 style",validation xnli,el,en,"justified in saying",validation xnli,el,en,"can we infer",validation xnli,ru,en,"guaranteed/possible/impossible",validation xnli,ru,en,"MNLI crowdsource",validation xnli,ru,en,"GPT-3 style",validation xnli,ru,en,"justified in saying",validation xnli,ru,en,"can we infer",validation xnli,th,en,"guaranteed/possible/impossible",validation xnli,th,en,"MNLI crowdsource",validation xnli,th,en,"GPT-3 style",validation xnli,th,en,"justified in saying",validation xnli,th,en,"can we infer",validation xnli,tr,en,"guaranteed/possible/impossible",validation xnli,tr,en,"MNLI crowdsource",validation xnli,tr,en,"GPT-3 style",validation xnli,tr,en,"justified in saying",validation xnli,tr,en,"can we infer",validation Muennighoff/xwinograd,ru,en,"underscore refer to",test Muennighoff/xwinograd,ru,en,"Replace",test Muennighoff/xwinograd,ru,en,"stand for",test Muennighoff/xwinograd,ru,en,"does underscore refer to",test Muennighoff/xwinograd,ru,en,"True or False",test Muennighoff/xwinograd,jp,en,"underscore refer to",test Muennighoff/xwinograd,jp,en,"Replace",test Muennighoff/xwinograd,jp,en,"stand for",test Muennighoff/xwinograd,jp,en,"does underscore refer to",test Muennighoff/xwinograd,jp,en,"True or False",test xcopa,et,en,"best_option",validation xcopa,et,en,"C1 or C2? premise, so/because…",validation xcopa,et,en,"i_am_hesitating",validation xcopa,et,en,"cause_effect",validation xcopa,et,en,"plausible_alternatives",validation xcopa,ht,en,"best_option",validation xcopa,ht,en,"C1 or C2? premise, so/because…",validation xcopa,ht,en,"i_am_hesitating",validation xcopa,ht,en,"cause_effect",validation xcopa,ht,en,"plausible_alternatives",validation xcopa,it,en,"best_option",validation xcopa,it,en,"C1 or C2? premise, so/because…",validation xcopa,it,en,"i_am_hesitating",validation xcopa,it,en,"cause_effect",validation xcopa,it,en,"plausible_alternatives",validation xcopa,qu,en,"best_option",validation xcopa,qu,en,"C1 or C2? premise, so/because…",validation xcopa,qu,en,"i_am_hesitating",validation xcopa,qu,en,"cause_effect",validation xcopa,qu,en,"plausible_alternatives",validation xcopa,th,en,"best_option",validation xcopa,th,en,"C1 or C2? premise, so/because…",validation xcopa,th,en,"i_am_hesitating",validation xcopa,th,en,"cause_effect",validation xcopa,th,en,"plausible_alternatives",validation xcopa,tr,en,"best_option",validation xcopa,tr,en,"C1 or C2? premise, so/because…",validation xcopa,tr,en,"i_am_hesitating",validation xcopa,tr,en,"cause_effect",validation xcopa,tr,en,"plausible_alternatives",validation ) DATASETS_AND_CONFIGS_MT_L1=( Muennighoff/xstory_cloze,ar,ar,"Story Continuation and Options_armt",validation Muennighoff/xstory_cloze,ar,ar,"Answer Given options_armt",validation Muennighoff/xstory_cloze,ar,ar,"Novel Correct Ending_armt",validation Muennighoff/xstory_cloze,ar,ar,"Generate Ending_armt",validation Muennighoff/xstory_cloze,ar,ar,"Choose Story Ending_armt",validation Muennighoff/xstory_cloze,es,es,"Story Continuation and Options_esmt",validation Muennighoff/xstory_cloze,es,es,"Answer Given options_esmt",validation Muennighoff/xstory_cloze,es,es,"Novel Correct Ending_esmt",validation Muennighoff/xstory_cloze,es,es,"Generate Ending_esmt",validation Muennighoff/xstory_cloze,es,es,"Choose Story Ending_esmt",validation Muennighoff/xstory_cloze,eu,eu,"Story Continuation and Options_eumt",validation Muennighoff/xstory_cloze,eu,eu,"Answer Given options_eumt",validation Muennighoff/xstory_cloze,eu,eu,"Novel Correct Ending_eumt",validation Muennighoff/xstory_cloze,eu,eu,"Generate Ending_eumt",validation Muennighoff/xstory_cloze,eu,eu,"Choose Story Ending_eumt",validation Muennighoff/xstory_cloze,id,id,"Story Continuation and Options_idmt",validation Muennighoff/xstory_cloze,id,id,"Answer Given options_idmt",validation Muennighoff/xstory_cloze,id,id,"Novel Correct Ending_idmt",validation Muennighoff/xstory_cloze,id,id,"Generate Ending_idmt",validation Muennighoff/xstory_cloze,id,id,"Choose Story Ending_idmt",validation Muennighoff/xstory_cloze,hi,hi,"Story Continuation and Options_himt",validation Muennighoff/xstory_cloze,hi,hi,"Answer Given options_himt",validation Muennighoff/xstory_cloze,hi,hi,"Novel Correct Ending_himt",validation Muennighoff/xstory_cloze,hi,hi,"Generate Ending_himt",validation Muennighoff/xstory_cloze,hi,hi,"Choose Story Ending_himt",validation Muennighoff/xstory_cloze,sw,sw,"Story Continuation and Options_swmt",validation Muennighoff/xstory_cloze,sw,sw,"Answer Given options_swmt",validation Muennighoff/xstory_cloze,sw,sw,"Novel Correct Ending_swmt",validation Muennighoff/xstory_cloze,sw,sw,"Generate Ending_swmt",validation Muennighoff/xstory_cloze,sw,sw,"Choose Story Ending_swmt",validation Muennighoff/xstory_cloze,te,te,"Story Continuation and Options_temt",validation Muennighoff/xstory_cloze,te,te,"Answer Given options_temt",validation Muennighoff/xstory_cloze,te,te,"Novel Correct Ending_temt",validation Muennighoff/xstory_cloze,te,te,"Generate Ending_temt",validation Muennighoff/xstory_cloze,te,te,"Choose Story Ending_temt",validation Muennighoff/xstory_cloze,zh,zh,"Story Continuation and Options_zhmt",validation Muennighoff/xstory_cloze,zh,zh,"Answer Given options_zhmt",validation Muennighoff/xstory_cloze,zh,zh,"Novel Correct Ending_zhmt",validation Muennighoff/xstory_cloze,zh,zh,"Generate Ending_zhmt",validation Muennighoff/xstory_cloze,zh,zh,"Choose Story Ending_zhmt",validation Muennighoff/xwinograd,fr,fr,"underscore refer to_frmt",test Muennighoff/xwinograd,fr,fr,"Replace_frmt",test Muennighoff/xwinograd,fr,fr,"stand for_frmt",test Muennighoff/xwinograd,fr,fr,"does underscore refer to_frmt",test Muennighoff/xwinograd,fr,fr,"True or False_frmt",test Muennighoff/xwinograd,pt,pt,"underscore refer to_ptmt",test Muennighoff/xwinograd,pt,pt,"Replace_ptmt",test Muennighoff/xwinograd,pt,pt,"stand for_ptmt",test Muennighoff/xwinograd,pt,pt,"does underscore refer to_ptmt",test Muennighoff/xwinograd,pt,pt,"True or False_ptmt",test Muennighoff/xwinograd,zh,zh,"underscore refer to_zhmt",test Muennighoff/xwinograd,zh,zh,"Replace_zhmt",test Muennighoff/xwinograd,zh,zh,"stand for_zhmt",test Muennighoff/xwinograd,zh,zh,"does underscore refer to_zhmt",test Muennighoff/xwinograd,zh,zh,"True or False_zhmt",test xcopa,id,id,"best_option_idmt",validation xcopa,id,id,"C1 or C2? premise_idmt",validation xcopa,id,id,"i_am_hesitating_idmt",validation xcopa,id,id,"cause_effect_idmt",validation xcopa,id,id,"plausible_alternatives_idmt",validation xcopa,sw,sw,"best_option_swmt",validation xcopa,sw,sw,"C1 or C2? premise_swmt",validation xcopa,sw,sw,"i_am_hesitating_swmt",validation xcopa,sw,sw,"cause_effect_swmt",validation xcopa,sw,sw,"plausible_alternatives_swmt",validation xcopa,ta,ta,"best_option_tamt",validation xcopa,ta,ta,"C1 or C2? premise_tamt",validation xcopa,ta,ta,"i_am_hesitating_tamt",validation xcopa,ta,ta,"cause_effect_tamt",validation xcopa,ta,ta,"plausible_alternatives_tamt",validation xcopa,vi,vi,"best_option_vimt",validation xcopa,vi,vi,"C1 or C2? premise_vimt",validation xcopa,vi,vi,"i_am_hesitating_vimt",validation xcopa,vi,vi,"cause_effect_vimt",validation xcopa,vi,vi,"plausible_alternatives_vimt",validation xcopa,zh,zh,"best_option_zhmt",validation xcopa,zh,zh,"C1 or C2? premise_zhmt",validation xcopa,zh,zh,"i_am_hesitating_zhmt",validation xcopa,zh,zh,"cause_effect_zhmt",validation xcopa,zh,zh,"plausible_alternatives_zhmt",validation ) DATASETS_AND_CONFIGS_ZHHT=( Muennighoff/xstory_cloze,zh,zh,"Story Continuation and Options_zhht",validation Muennighoff/xstory_cloze,zh,zh,"Answer Given options_zhht",validation Muennighoff/xstory_cloze,zh,zh,"Novel Correct Ending_zhht",validation Muennighoff/xstory_cloze,zh,zh,"Generate Ending_zhht",validation Muennighoff/xstory_cloze,zh,zh,"Choose Story Ending_zhht",validation Muennighoff/xwinograd,zh,zh,"underscore refer to_zhht",test Muennighoff/xwinograd,zh,zh,"Replace_zhht",test Muennighoff/xwinograd,zh,zh,"stand for_zhht",test Muennighoff/xwinograd,zh,zh,"does underscore refer to_zhht",test Muennighoff/xwinograd,zh,zh,"True or False_zhht",test xcopa,zh,zh,"best_option_zhht",validation xcopa,zh,zh,"C1 or C2? premise_zhht",validation xcopa,zh,zh,"i_am_hesitating_zhht",validation xcopa,zh,zh,"cause_effect_zhht",validation xcopa,zh,zh,"plausible_alternatives_zhht",validation ) DATASETS_AND_CONFIGS_XNLIHTMT=( xnli,ar,ar,"guaranteed/possible/impossible_arht",validation xnli,ar,ar,"MNLI crowdsource_arht",validation xnli,ar,ar,"GPT-3 style_arht",validation xnli,ar,ar,"justified in saying_arht",validation xnli,ar,ar,"can we infer_arht",validation xnli,ar,ar,"guaranteed/possible/impossible_armt",validation xnli,ar,ar,"MNLI crowdsource_armt",validation xnli,ar,ar,"GPT-3 style_armt",validation xnli,ar,ar,"justified in saying_armt",validation xnli,ar,ar,"can we infer_armt",validation xnli,es,es,"guaranteed/possible/impossible_esht",validation xnli,es,es,"MNLI crowdsource_esht",validation xnli,es,es,"GPT-3 style_esht",validation xnli,es,es,"justified in saying_esht",validation xnli,es,es,"can we infer_esht",validation xnli,es,es,"guaranteed/possible/impossible_esmt",validation xnli,es,es,"MNLI crowdsource_esmt",validation xnli,es,es,"GPT-3 style_esmt",validation xnli,es,es,"justified in saying_esmt",validation xnli,es,es,"can we infer_esmt",validation xnli,fr,fr,"guaranteed/possible/impossible_frht",validation xnli,fr,fr,"MNLI crowdsource_frht",validation xnli,fr,fr,"GPT-3 style_frht",validation xnli,fr,fr,"justified in saying_frht",validation xnli,fr,fr,"can we infer_frht",validation xnli,fr,fr,"guaranteed/possible/impossible_frmt",validation xnli,fr,fr,"MNLI crowdsource_frmt",validation xnli,fr,fr,"GPT-3 style_frmt",validation xnli,fr,fr,"justified in saying_frmt",validation xnli,fr,fr,"can we infer_frmt",validation xnli,hi,hi,"guaranteed/possible/impossible_hiht",validation xnli,hi,hi,"MNLI crowdsource_hiht",validation xnli,hi,hi,"GPT-3 style_hiht",validation xnli,hi,hi,"justified in saying_hiht",validation xnli,hi,hi,"can we infer_hiht",validation xnli,hi,hi,"guaranteed/possible/impossible_himt",validation xnli,hi,hi,"MNLI crowdsource_himt",validation xnli,hi,hi,"GPT-3 style_himt",validation xnli,hi,hi,"justified in saying_himt",validation xnli,hi,hi,"can we infer_himt",validation xnli,ur,ur,"guaranteed/possible/impossible_urht",validation xnli,ur,ur,"MNLI crowdsource_urht",validation xnli,ur,ur,"GPT-3 style_urht",validation xnli,ur,ur,"justified in saying_urht",validation xnli,ur,ur,"can we infer_urht",validation xnli,ur,ur,"guaranteed/possible/impossible_urmt",validation xnli,ur,ur,"MNLI crowdsource_urmt",validation xnli,ur,ur,"GPT-3 style_urmt",validation xnli,ur,ur,"justified in saying_urmt",validation xnli,ur,ur,"can we infer_urmt",validation xnli,sw,sw,"guaranteed/possible/impossible_swht",validation xnli,sw,sw,"MNLI crowdsource_swht",validation xnli,sw,sw,"GPT-3 style_swht",validation xnli,sw,sw,"justified in saying_swht",validation xnli,sw,sw,"can we infer_swht",validation xnli,sw,sw,"guaranteed/possible/impossible_swmt",validation xnli,sw,sw,"MNLI crowdsource_swmt",validation xnli,sw,sw,"GPT-3 style_swmt",validation xnli,sw,sw,"justified in saying_swmt",validation xnli,sw,sw,"can we infer_swmt",validation xnli,vi,vi,"guaranteed/possible/impossible_viht",validation xnli,vi,vi,"MNLI crowdsource_viht",validation xnli,vi,vi,"GPT-3 style_viht",validation xnli,vi,vi,"justified in saying_viht",validation xnli,vi,vi,"can we infer_viht",validation xnli,vi,vi,"guaranteed/possible/impossible_vimt",validation xnli,vi,vi,"MNLI crowdsource_vimt",validation xnli,vi,vi,"GPT-3 style_vimt",validation xnli,vi,vi,"justified in saying_vimt",validation xnli,vi,vi,"can we infer_vimt",validation xnli,zh,zh,"guaranteed/possible/impossible_zhht",validation xnli,zh,zh,"MNLI crowdsource_zhht",validation xnli,zh,zh,"GPT-3 style_zhht",validation xnli,zh,zh,"justified in saying_zhht",validation xnli,zh,zh,"can we infer_zhht",validation xnli,zh,zh,"guaranteed/possible/impossible_zhmt",validation xnli,zh,zh,"MNLI crowdsource_zhmt",validation xnli,zh,zh,"GPT-3 style_zhmt",validation xnli,zh,zh,"justified in saying_zhmt",validation xnli,zh,zh,"can we infer_zhmt",validation ) DATASETS_AND_CONFIGS_MT_L2=( Muennighoff/xstory_cloze,my,my,"Story Continuation and Options_mymt",validation Muennighoff/xstory_cloze,my,my,"Answer Given options_mymt",validation Muennighoff/xstory_cloze,my,my,"Novel Correct Ending_mymt",validation Muennighoff/xstory_cloze,my,my,"Generate Ending_mymt",validation Muennighoff/xstory_cloze,my,my,"Choose Story Ending_mymt",validation Muennighoff/xstory_cloze,ru,ru,"Story Continuation and Options_rumt",validation Muennighoff/xstory_cloze,ru,ru,"Answer Given options_rumt",validation Muennighoff/xstory_cloze,ru,ru,"Novel Correct Ending_rumt",validation Muennighoff/xstory_cloze,ru,ru,"Generate Ending_rumt",validation Muennighoff/xstory_cloze,ru,ru,"Choose Story Ending_rumt",validation Muennighoff/xstory_cloze,sw,sw,"Story Continuation and Options_swmt",validation Muennighoff/xstory_cloze,sw,sw,"Answer Given options_swmt",validation Muennighoff/xstory_cloze,sw,sw,"Novel Correct Ending_swmt",validation Muennighoff/xstory_cloze,sw,sw,"Generate Ending_swmt",validation Muennighoff/xstory_cloze,sw,sw,"Choose Story Ending_swmt",validation Muennighoff/xstory_cloze,te,te,"Story Continuation and Options_temt",validation Muennighoff/xstory_cloze,te,te,"Answer Given options_temt",validation Muennighoff/xstory_cloze,te,te,"Novel Correct Ending_temt",validation Muennighoff/xstory_cloze,te,te,"Generate Ending_temt",validation Muennighoff/xstory_cloze,te,te,"Choose Story Ending_temt",validation Muennighoff/xwinograd,jp,jp,"underscore refer to_jpmt",test Muennighoff/xwinograd,jp,jp,"Replace_jpmt",test Muennighoff/xwinograd,jp,jp,"stand for_jpmt",test Muennighoff/xwinograd,jp,jp,"does underscore refer to_jpmt",test Muennighoff/xwinograd,jp,jp,"True or False_jpmt",test Muennighoff/xwinograd,ru,ru,"underscore refer to_rumt",test Muennighoff/xwinograd,ru,ru,"Replace_rumt",test Muennighoff/xwinograd,ru,ru,"stand for_rumt",test Muennighoff/xwinograd,ru,ru,"does underscore refer to_rumt",test Muennighoff/xwinograd,ru,ru,"True or False_rumt",test xcopa,et,et,"best_option_etmt",validation xcopa,et,et,"C1 or C2? premise_etmt",validation xcopa,et,et,"i_am_hesitating_etmt",validation xcopa,et,et,"cause_effect_etmt",validation xcopa,et,et,"plausible_alternatives_etmt",validation xcopa,ht,ht,"best_option_htmt",validation xcopa,ht,ht,"C1 or C2? premise_htmt",validation xcopa,ht,ht,"i_am_hesitating_htmt",validation xcopa,ht,ht,"cause_effect_htmt",validation xcopa,ht,ht,"plausible_alternatives_htmt",validation xcopa,it,it,"best_option_itmt",validation xcopa,it,it,"C1 or C2? premise_itmt",validation xcopa,it,it,"i_am_hesitating_itmt",validation xcopa,it,it,"cause_effect_itmt",validation xcopa,it,it,"plausible_alternatives_itmt",validation xcopa,qu,qu,"best_option_qumt",validation xcopa,qu,qu,"C1 or C2? premise_qumt",validation xcopa,qu,qu,"i_am_hesitating_qumt",validation xcopa,qu,qu,"cause_effect_qumt",validation xcopa,qu,qu,"plausible_alternatives_qumt",validation xcopa,th,th,"best_option_thmt",validation xcopa,th,th,"C1 or C2? premise_thmt",validation xcopa,th,th,"i_am_hesitating_thmt",validation xcopa,th,th,"cause_effect_thmt",validation xcopa,th,th,"plausible_alternatives_thmt",validation xcopa,tr,tr,"best_option_trmt",validation xcopa,tr,tr,"C1 or C2? premise_trmt",validation xcopa,tr,tr,"i_am_hesitating_trmt",validation xcopa,tr,tr,"cause_effect_trmt",validation xcopa,tr,tr,"plausible_alternatives_trmt",validation xnli,bg,bg,"guaranteed/possible/impossible_bgmt",validation xnli,bg,bg,"MNLI crowdsource_bgmt",validation xnli,bg,bg,"GPT-3 style_bgmt",validation xnli,bg,bg,"justified in saying_bgmt",validation xnli,bg,bg,"can we infer_bgmt",validation xnli,de,de,"guaranteed/possible/impossible_demt",validation xnli,de,de,"MNLI crowdsource_demt",validation xnli,de,de,"GPT-3 style_demt",validation xnli,de,de,"justified in saying_demt",validation xnli,de,de,"can we infer_demt",validation xnli,el,el,"guaranteed/possible/impossible_elmt",validation xnli,el,el,"MNLI crowdsource_elmt",validation xnli,el,el,"GPT-3 style_elmt",validation xnli,el,el,"justified in saying_elmt",validation xnli,el,el,"can we infer_elmt",validation xnli,ru,ru,"guaranteed/possible/impossible_rumt",validation xnli,ru,ru,"MNLI crowdsource_rumt",validation xnli,ru,ru,"GPT-3 style_rumt",validation xnli,ru,ru,"justified in saying_rumt",validation xnli,ru,ru,"can we infer_rumt",validation xnli,th,th,"guaranteed/possible/impossible_thmt",validation xnli,th,th,"MNLI crowdsource_thmt",validation xnli,th,th,"GPT-3 style_thmt",validation xnli,th,th,"justified in saying_thmt",validation xnli,th,th,"can we infer_thmt",validation xnli,tr,tr,"guaranteed/possible/impossible_trmt",validation xnli,tr,tr,"MNLI crowdsource_trmt",validation xnli,tr,tr,"GPT-3 style_trmt",validation xnli,tr,tr,"justified in saying_trmt",validation xnli,tr,tr,"can we infer_trmt",validation ) DATASETS_AND_CONFIGS_RU=( Muennighoff/xstory_cloze,ru,en,"Story Continuation and Options",validation Muennighoff/xstory_cloze,ru,en,"Answer Given options",validation Muennighoff/xstory_cloze,ru,en,"Novel Correct Ending",validation Muennighoff/xstory_cloze,ru,en,"Generate Ending",validation Muennighoff/xstory_cloze,ru,en,"Choose Story Ending",validation Muennighoff/xwinograd,ru,en,"underscore refer to",test Muennighoff/xwinograd,ru,en,"Replace",test Muennighoff/xwinograd,ru,en,"stand for",test Muennighoff/xwinograd,ru,en,"does underscore refer to",test Muennighoff/xwinograd,ru,en,"True or False",test xnli,ru,en,"guaranteed/possible/impossible",validation xnli,ru,en,"MNLI crowdsource",validation xnli,ru,en,"GPT-3 style",validation xnli,ru,en,"justified in saying",validation xnli,ru,en,"can we infer",validation ) DATASET_AND_CONFIG=${DATASETS_AND_CONFIGS_RU[$SLURM_ARRAY_TASK_ID]} echo $ARGUMENT # Run T0 evaluation # For PrefixLM add --prefixlm IFS=',' read dataset_name dataset_config_name template_config_name template_name split <<< "${DATASET_AND_CONFIG}" python promptsource/t-zero/evaluation/run_eval.py \ --dataset_name $dataset_name \ --dataset_config_name $dataset_config_name \ --template_config_name $template_config_name \ --template_name "$template_name" \ --split $split \ --model_name_or_path $CHECKPOINT_PATH \ --output_dir $OUTPUT_DIR \ --per_device_eval_batch_size 4 \ --max_length 2048 \ --dtype float16