| {"amazon_reviews_multi_en": {"prompt_body_title_to_star": {"arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='amazon_reviews_multi', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "en", "dataset_name": "amazon_reviews_multi", "evaluation": {"accuracy": 0.5416}, "template_name": "prompt_body_title_to_star"}, "prompt_review_to_star": {"arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='amazon_reviews_multi', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "en", "dataset_name": "amazon_reviews_multi", "evaluation": {"accuracy": 0.509}, "template_name": "prompt_review_to_star"}, "prompt_title_to_star": {"arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='amazon_reviews_multi', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "en", "dataset_name": "amazon_reviews_multi", "evaluation": {"accuracy": 0.364}, "template_name": "prompt_title_to_star"}}, "amazon_reviews_multi_es": {"prompt_body_title_to_star": {"arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='amazon_reviews_multi', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "es", "dataset_name": "amazon_reviews_multi", "evaluation": {"accuracy": 0.4448}, "template_name": "prompt_body_title_to_star"}, "prompt_review_to_star": {"arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='amazon_reviews_multi', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "es", "dataset_name": "amazon_reviews_multi", "evaluation": {"accuracy": 0.4326}, "template_name": "prompt_review_to_star"}, "prompt_title_to_star": {"arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='amazon_reviews_multi', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "es", "dataset_name": "amazon_reviews_multi", "evaluation": {"accuracy": 0.2802}, "template_name": "prompt_title_to_star"}}, "amazon_reviews_multi_fr": {"prompt_body_title_to_star": {"arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='amazon_reviews_multi', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "fr", "dataset_name": "amazon_reviews_multi", "evaluation": {"accuracy": 0.449}, "template_name": "prompt_body_title_to_star"}, "prompt_review_to_star": {"arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='amazon_reviews_multi', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "fr", "dataset_name": "amazon_reviews_multi", "evaluation": {"accuracy": 0.4392}, "template_name": "prompt_review_to_star"}, "prompt_title_to_star": {"arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='amazon_reviews_multi', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "fr", "dataset_name": "amazon_reviews_multi", "evaluation": {"accuracy": 0.3128}, "template_name": "prompt_title_to_star"}}, "amazon_reviews_multi_zh": {"prompt_body_title_to_star": {"arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='amazon_reviews_multi', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "zh", "dataset_name": "amazon_reviews_multi", "evaluation": {"accuracy": 0.421}, "template_name": "prompt_body_title_to_star"}, "prompt_review_to_star": {"arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='amazon_reviews_multi', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "zh", "dataset_name": "amazon_reviews_multi", "evaluation": {"accuracy": 0.4048}, "template_name": "prompt_review_to_star"}, "prompt_title_to_star": {"arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='amazon_reviews_multi', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "zh", "dataset_name": "amazon_reviews_multi", "evaluation": {"accuracy": 0.302}, "template_name": "prompt_title_to_star"}}, "aqua_rat_raw": {"Answer questions from options": {"arguments": "Namespace(config_name=None, dataset_config_name='raw', dataset_name='aqua_rat', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Answer questions from options', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "raw", "dataset_name": "aqua_rat", "evaluation": {"accuracy": 0.2755905511811024}, "template_name": "Answer questions from options"}, "answer_quiz": {"arguments": "Namespace(config_name=None, dataset_config_name='raw', dataset_name='aqua_rat', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='answer_quiz', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "raw", "dataset_name": "aqua_rat", "evaluation": {"accuracy": 0.2677165354330709}, "template_name": "answer_quiz"}, "select_the_best_option": {"arguments": "Namespace(config_name=None, dataset_config_name='raw', dataset_name='aqua_rat', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='select_the_best_option', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "raw", "dataset_name": "aqua_rat", "evaluation": {"accuracy": 0.28346456692913385}, "template_name": "select_the_best_option"}}, "art_None": {"choose_hypothesis": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "art", "evaluation": {"accuracy": 0.6742819843342036}, "template_name": "choose_hypothesis"}, "choose_hypothesis_believable": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_believable', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "art", "evaluation": {"accuracy": 0.6677545691906005}, "template_name": "choose_hypothesis_believable"}, "choose_hypothesis_desc": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_desc', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "art", "evaluation": {"accuracy": 0.5515665796344648}, "template_name": "choose_hypothesis_desc"}, "choose_hypothesis_likely": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_likely', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "art", "evaluation": {"accuracy": 0.5737597911227154}, "template_name": "choose_hypothesis_likely"}, "choose_hypothesis_options": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_options', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "art", "evaluation": {"accuracy": 0.6657963446475196}, "template_name": "choose_hypothesis_options"}}, "banking77_None": {"direct_to_which_department": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='banking77', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='direct_to_which_department', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "banking77", "evaluation": {"accuracy": 0.16688311688311688}, "template_name": "direct_to_which_department"}, "help_page_topic": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='banking77', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='help_page_topic', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "banking77", "evaluation": {"accuracy": 0.2857142857142857}, "template_name": "help_page_topic"}, "rephrase_as_banking_term": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='banking77', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='rephrase_as_banking_term', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "banking77", "evaluation": {"accuracy": 0.2905844155844156}, "template_name": "rephrase_as_banking_term"}}, "blbooksgenre_title_genre_classifiction": {"classify": {"arguments": "Namespace(config_name=None, dataset_config_name='title_genre_classifiction', dataset_name='blbooksgenre', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='classify', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "title_genre_classifiction", "dataset_name": "blbooksgenre", "evaluation": {"accuracy": 0.3127880184331797}, "template_name": "classify"}, "multi-choice": {"arguments": "Namespace(config_name=None, dataset_config_name='title_genre_classifiction', dataset_name='blbooksgenre', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='multi-choice', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "title_genre_classifiction", "dataset_name": "blbooksgenre", "evaluation": {"accuracy": 0.8640552995391705}, "template_name": "multi-choice"}, "premise_context_first": {"arguments": "Namespace(config_name=None, dataset_config_name='title_genre_classifiction', dataset_name='blbooksgenre', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='premise_context_first', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "title_genre_classifiction", "dataset_name": "blbooksgenre", "evaluation": {"accuracy": 0.7707373271889401}, "template_name": "premise_context_first"}}, "blimp_adjunct_island": {"grammatical_between_1_2": {"arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='grammatical_between_1_2', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "adjunct_island", "dataset_name": "blimp", "evaluation": {"accuracy": 0.466}, "template_name": "grammatical_between_1_2"}, "grammatical_between_A_B": {"arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='grammatical_between_A_B', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "adjunct_island", "dataset_name": "blimp", "evaluation": {"accuracy": 0.327}, "template_name": "grammatical_between_A_B"}, "grammatical_which_one_1_2": {"arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='grammatical_which_one_1_2', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "adjunct_island", "dataset_name": "blimp", "evaluation": {"accuracy": 0.498}, "template_name": "grammatical_which_one_1_2"}, "single_sentence_bad_yes_no": {"arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='single_sentence_bad_yes_no', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "adjunct_island", "dataset_name": "blimp", "evaluation": {"accuracy": 0.507}, "template_name": "single_sentence_bad_yes_no"}, "single_sentence_good_yes_no": {"arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='single_sentence_good_yes_no', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "adjunct_island", "dataset_name": "blimp", "evaluation": {"accuracy": 0.516}, "template_name": "single_sentence_good_yes_no"}}, "climate_fever_None": {"claim_and_all_supporting_evidences": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='claim_and_all_supporting_evidences', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "climate_fever", "evaluation": {"accuracy": 0.24299674267100976}, "template_name": "claim_and_all_supporting_evidences"}, "fifth_evidence_and_claim_itemization": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='fifth_evidence_and_claim_itemization', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "climate_fever", "evaluation": {"accuracy": 0.36612377850162864}, "template_name": "fifth_evidence_and_claim_itemization"}, "first_evidence_and_claim_itemization": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='first_evidence_and_claim_itemization', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "climate_fever", "evaluation": {"accuracy": 0.15895765472312703}, "template_name": "first_evidence_and_claim_itemization"}, "second_evidence_and_claim_itemization": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='second_evidence_and_claim_itemization', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "climate_fever", "evaluation": {"accuracy": 0.14788273615635178}, "template_name": "second_evidence_and_claim_itemization"}, "third_evidence_claim_pair": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='third_evidence_claim_pair', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "climate_fever", "evaluation": {"accuracy": 0.18631921824104233}, "template_name": "third_evidence_claim_pair"}}, "codah_codah": {"affirmative_instruction_after_sentence_and_choices": {"arguments": "Namespace(config_name=None, dataset_config_name='codah', dataset_name='codah', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='affirmative_instruction_after_sentence_and_choices', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "codah", "dataset_name": "codah", "evaluation": {"accuracy": 0.8065561959654178}, "template_name": "affirmative_instruction_after_sentence_and_choices"}, "affirmative_instruction_before_sentence_and_choices": {"arguments": "Namespace(config_name=None, dataset_config_name='codah', dataset_name='codah', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='affirmative_instruction_before_sentence_and_choices', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "codah", "dataset_name": "codah", "evaluation": {"accuracy": 0.7885446685878963}, "template_name": "affirmative_instruction_before_sentence_and_choices"}, "interrogative_instruction_after_sentence_and_choices": {"arguments": "Namespace(config_name=None, dataset_config_name='codah', dataset_name='codah', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='interrogative_instruction_after_sentence_and_choices', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "codah", "dataset_name": "codah", "evaluation": {"accuracy": 0.8090778097982709}, "template_name": "interrogative_instruction_after_sentence_and_choices"}}, "commonsense_qa_None": {"answer_given_question_without_options": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='commonsense_qa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='answer_given_question_without_options', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "commonsense_qa", "evaluation": {"accuracy": 0.7018837018837019}, "template_name": "answer_given_question_without_options"}, "most_suitable_answer": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='commonsense_qa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='most_suitable_answer', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "commonsense_qa", "evaluation": {"accuracy": 0.8304668304668305}, "template_name": "most_suitable_answer"}, "question_answering": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='commonsense_qa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='question_answering', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "commonsense_qa", "evaluation": {"accuracy": 0.8026208026208026}, "template_name": "question_answering"}}, "conv_ai_3_None": {"ambiguous": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='ambiguous', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "conv_ai_3", "evaluation": {"accuracy": 0.39040207522697795}, "template_name": "ambiguous"}, "clarification_needed": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='clarification_needed', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "conv_ai_3", "evaluation": {"accuracy": 0.39040207522697795}, "template_name": "clarification_needed"}, "directly_answer": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='directly_answer', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "conv_ai_3", "evaluation": {"accuracy": 0.6095979247730221}, "template_name": "directly_answer"}, "score_give_number": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='score_give_number', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "conv_ai_3", "evaluation": {"accuracy": 0.21444012105490703}, "template_name": "score_give_number"}, "score_how_much": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='score_how_much', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "conv_ai_3", "evaluation": {"accuracy": 0.21444012105490703}, "template_name": "score_how_much"}}, "craigslist_bargains_None": {"best deal": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='craigslist_bargains', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='best deal', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "craigslist_bargains", "evaluation": {"accuracy": 0.5175879396984925}, "template_name": "best deal"}, "good deal for seller": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='craigslist_bargains', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='good deal for seller', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "craigslist_bargains", "evaluation": {"accuracy": 0.2864321608040201}, "template_name": "good deal for seller"}, "good deal for seller no list price": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='craigslist_bargains', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='good deal for seller no list price', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "craigslist_bargains", "evaluation": {"accuracy": 0.16917922948073702}, "template_name": "good deal for seller no list price"}, "good deal for seller no list price implicit": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='craigslist_bargains', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='good deal for seller no list price implicit', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "craigslist_bargains", "evaluation": {"accuracy": 0.24288107202680068}, "template_name": "good deal for seller no list price implicit"}}, "emotion_None": {"answer_question_with_emotion_label": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='emotion', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='answer_question_with_emotion_label', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "emotion", "evaluation": {"accuracy": 0.3675}, "template_name": "answer_question_with_emotion_label"}, "answer_with_class_label": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='emotion', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='answer_with_class_label', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "emotion", "evaluation": {"accuracy": 0.1445}, "template_name": "answer_with_class_label"}, "choose_the_best_emotion_label": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='emotion', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='choose_the_best_emotion_label', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "emotion", "evaluation": {"accuracy": 0.3665}, "template_name": "choose_the_best_emotion_label"}, "reply_with_emoation_label": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='emotion', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='reply_with_emoation_label', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "emotion", "evaluation": {"accuracy": 0.452}, "template_name": "reply_with_emoation_label"}}, "financial_phrasebank_sentences_allagree": {"bullish_neutral_bearish": {"arguments": "Namespace(config_name=None, dataset_config_name='sentences_allagree', dataset_name='financial_phrasebank', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='bullish_neutral_bearish', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sentences_allagree", "dataset_name": "financial_phrasebank", "evaluation": {"accuracy": 0.24823321554770317}, "template_name": "bullish_neutral_bearish"}, "complementary_industries": {"arguments": "Namespace(config_name=None, dataset_config_name='sentences_allagree', dataset_name='financial_phrasebank', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='complementary_industries', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sentences_allagree", "dataset_name": "financial_phrasebank", "evaluation": {"accuracy": 0.0627208480565371}, "template_name": "complementary_industries"}, "sentiment": {"arguments": "Namespace(config_name=None, dataset_config_name='sentences_allagree', dataset_name='financial_phrasebank', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='sentiment', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sentences_allagree", "dataset_name": "financial_phrasebank", "evaluation": {"accuracy": 0.3630742049469965}, "template_name": "sentiment"}, "share_price_option": {"arguments": "Namespace(config_name=None, dataset_config_name='sentences_allagree', dataset_name='financial_phrasebank', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='share_price_option', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sentences_allagree", "dataset_name": "financial_phrasebank", "evaluation": {"accuracy": 0.37234982332155475}, "template_name": "share_price_option"}, "word_comes_to_mind": {"arguments": "Namespace(config_name=None, dataset_config_name='sentences_allagree', dataset_name='financial_phrasebank', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='word_comes_to_mind', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sentences_allagree", "dataset_name": "financial_phrasebank", "evaluation": {"accuracy": 0.05830388692579505}, "template_name": "word_comes_to_mind"}}, "glue_cola": {"Following sentence acceptable": {"arguments": "Namespace(config_name=None, dataset_config_name='cola', dataset_name='glue', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Following sentence acceptable', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "cola", "dataset_name": "glue", "evaluation": {"accuracy": 0.50143815915628}, "template_name": "Following sentence acceptable"}, "Make sense yes no": {"arguments": "Namespace(config_name=None, dataset_config_name='cola', dataset_name='glue', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Make sense yes no', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "cola", "dataset_name": "glue", "evaluation": {"accuracy": 0.6337488015340365}, "template_name": "Make sense yes no"}, "Previous sentence acceptable": {"arguments": "Namespace(config_name=None, dataset_config_name='cola', dataset_name='glue', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Previous sentence acceptable', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "cola", "dataset_name": "glue", "evaluation": {"accuracy": 0.3461169702780441}, "template_name": "Previous sentence acceptable"}, "editing": {"arguments": "Namespace(config_name=None, dataset_config_name='cola', dataset_name='glue', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='editing', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "cola", "dataset_name": "glue", "evaluation": {"accuracy": 0.4458293384467881}, "template_name": "editing"}, "is_this_correct": {"arguments": "Namespace(config_name=None, dataset_config_name='cola', dataset_name='glue', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='is_this_correct', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "cola", "dataset_name": "glue", "evaluation": {"accuracy": 0.4228187919463087}, "template_name": "is_this_correct"}}, "glue_sst2": {"following positive negative": {"arguments": "Namespace(config_name=None, dataset_config_name='sst2', dataset_name='glue', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='following positive negative', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sst2", "dataset_name": "glue", "evaluation": {"accuracy": 0.944954128440367}, "template_name": "following positive negative"}, "happy or mad": {"arguments": "Namespace(config_name=None, dataset_config_name='sst2', dataset_name='glue', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='happy or mad', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sst2", "dataset_name": "glue", "evaluation": {"accuracy": 0.9334862385321101}, "template_name": "happy or mad"}, "positive negative after": {"arguments": "Namespace(config_name=None, dataset_config_name='sst2', dataset_name='glue', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='positive negative after', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sst2", "dataset_name": "glue", "evaluation": {"accuracy": 0.9392201834862385}, "template_name": "positive negative after"}, "review": {"arguments": "Namespace(config_name=None, dataset_config_name='sst2', dataset_name='glue', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='review', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sst2", "dataset_name": "glue", "evaluation": {"accuracy": 0.9506880733944955}, "template_name": "review"}, "said": {"arguments": "Namespace(config_name=None, dataset_config_name='sst2', dataset_name='glue', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='said', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sst2", "dataset_name": "glue", "evaluation": {"accuracy": 0.819954128440367}, "template_name": "said"}}, "head_qa_en": {"multiple_choice_a_and_q_en": {"arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='head_qa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='multiple_choice_a_and_q_en', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "en", "dataset_name": "head_qa", "evaluation": {"accuracy": 0.32430453879941434}, "template_name": "multiple_choice_a_and_q_en"}, "multiple_choice_a_and_q_with_context_en": {"arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='head_qa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='multiple_choice_a_and_q_with_context_en', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "en", "dataset_name": "head_qa", "evaluation": {"accuracy": 0.3330893118594436}, "template_name": "multiple_choice_a_and_q_with_context_en"}, "multiple_choice_q_and_a_en": {"arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='head_qa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='multiple_choice_q_and_a_en', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "en", "dataset_name": "head_qa", "evaluation": {"accuracy": 0.5395314787701317}, "template_name": "multiple_choice_q_and_a_en"}, "multiple_choice_q_and_a_index_en": {"arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='head_qa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='multiple_choice_q_and_a_index_en', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "en", "dataset_name": "head_qa", "evaluation": {"accuracy": 0.5314787701317716}, "template_name": "multiple_choice_q_and_a_index_en"}, "multiple_choice_q_and_a_index_with_context_en": {"arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='head_qa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='multiple_choice_q_and_a_index_with_context_en', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "en", "dataset_name": "head_qa", "evaluation": {"accuracy": 0.5380673499267935}, "template_name": "multiple_choice_q_and_a_index_with_context_en"}}, "head_qa_es": {"multiple_choice_a_and_q_en": {"arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='head_qa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='multiple_choice_a_and_q_en', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "es", "dataset_name": "head_qa", "evaluation": {"accuracy": 0.3213762811127379}, "template_name": "multiple_choice_a_and_q_en"}, "multiple_choice_a_and_q_with_context_en": {"arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='head_qa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='multiple_choice_a_and_q_with_context_en', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "es", "dataset_name": "head_qa", "evaluation": {"accuracy": 0.32723279648609077}, "template_name": "multiple_choice_a_and_q_with_context_en"}, "multiple_choice_q_and_a_en": {"arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='head_qa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='multiple_choice_q_and_a_en', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "es", "dataset_name": "head_qa", "evaluation": {"accuracy": 0.5080527086383602}, "template_name": "multiple_choice_q_and_a_en"}, "multiple_choice_q_and_a_index_en": {"arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='head_qa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='multiple_choice_q_and_a_index_en', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "es", "dataset_name": "head_qa", "evaluation": {"accuracy": 0.5175695461200586}, "template_name": "multiple_choice_q_and_a_index_en"}, "multiple_choice_q_and_a_index_with_context_en": {"arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='head_qa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='multiple_choice_q_and_a_index_with_context_en', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "es", "dataset_name": "head_qa", "evaluation": {"accuracy": 0.5153733528550513}, "template_name": "multiple_choice_q_and_a_index_with_context_en"}}, "health_fact_None": {"claim_explanation_classification": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='health_fact', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='claim_explanation_classification', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "health_fact", "evaluation": {"accuracy": 0.6130612244897959}, "template_name": "claim_explanation_classification"}, "claim_veracity_classification_after_reading_I_believe": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='health_fact', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='claim_veracity_classification_after_reading_I_believe', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "health_fact", "evaluation": {"accuracy": 0.4791836734693877}, "template_name": "claim_veracity_classification_after_reading_I_believe"}, "claim_veracity_classification_tell_me": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='health_fact', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='claim_veracity_classification_tell_me', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "health_fact", "evaluation": {"accuracy": 0.052244897959183675}, "template_name": "claim_veracity_classification_tell_me"}}, "hlgd_None": {"is_same_event_editor_asks": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='hlgd', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='is_same_event_editor_asks', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "hlgd", "evaluation": {"accuracy": 0.5360077332044466}, "template_name": "is_same_event_editor_asks"}, "is_same_event_interrogative_talk": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='hlgd', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='is_same_event_interrogative_talk', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "hlgd", "evaluation": {"accuracy": 0.6549057515708071}, "template_name": "is_same_event_interrogative_talk"}, "is_same_event_refer": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='hlgd', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='is_same_event_refer', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "hlgd", "evaluation": {"accuracy": 0.7114548090865153}, "template_name": "is_same_event_refer"}, "is_same_event_with_time_interrogative_related": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='hlgd', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='is_same_event_with_time_interrogative_related', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "hlgd", "evaluation": {"accuracy": 0.6756887385210246}, "template_name": "is_same_event_with_time_interrogative_related"}, "is_same_event_with_time_interrogative_talk": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='hlgd', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='is_same_event_with_time_interrogative_talk', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "hlgd", "evaluation": {"accuracy": 0.7844369260512325}, "template_name": "is_same_event_with_time_interrogative_talk"}}, "hyperpartisan_news_detection_byarticle": {"consider_does_it_follow_a_hyperpartisan_argumentation": {"arguments": "Namespace(config_name=None, dataset_config_name='byarticle', dataset_name='hyperpartisan_news_detection', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='consider_does_it_follow_a_hyperpartisan_argumentation', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "byarticle", "dataset_name": "hyperpartisan_news_detection", "evaluation": {"accuracy": 0.6372093023255814}, "template_name": "consider_does_it_follow_a_hyperpartisan_argumentation"}, "consider_it_exhibits_extreme_one_sidedness": {"arguments": "Namespace(config_name=None, dataset_config_name='byarticle', dataset_name='hyperpartisan_news_detection', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='consider_it_exhibits_extreme_one_sidedness', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "byarticle", "dataset_name": "hyperpartisan_news_detection", "evaluation": {"accuracy": 0.6310077519379845}, "template_name": "consider_it_exhibits_extreme_one_sidedness"}, "consume_with_caution": {"arguments": "Namespace(config_name=None, dataset_config_name='byarticle', dataset_name='hyperpartisan_news_detection', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='consume_with_caution', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "byarticle", "dataset_name": "hyperpartisan_news_detection", "evaluation": {"accuracy": 0.6310077519379845}, "template_name": "consume_with_caution"}, "extreme_left_wing_or_right_wing": {"arguments": "Namespace(config_name=None, dataset_config_name='byarticle', dataset_name='hyperpartisan_news_detection', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='extreme_left_wing_or_right_wing', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "byarticle", "dataset_name": "hyperpartisan_news_detection", "evaluation": {"accuracy": 0.6310077519379845}, "template_name": "extreme_left_wing_or_right_wing"}, "follows_hyperpartisan_argumentation": {"arguments": "Namespace(config_name=None, dataset_config_name='byarticle', dataset_name='hyperpartisan_news_detection', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='follows_hyperpartisan_argumentation', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "byarticle", "dataset_name": "hyperpartisan_news_detection", "evaluation": {"accuracy": 0.6310077519379845}, "template_name": "follows_hyperpartisan_argumentation"}}, "liar_None": {"Given statement guess category": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='liar', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Given statement guess category', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "liar", "evaluation": {"accuracy": 0.2087227414330218}, "template_name": "Given statement guess category"}}, "lince_sa_spaeng": {"express sentiment": {"arguments": "Namespace(config_name=None, dataset_config_name='sa_spaeng', dataset_name='lince', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='express sentiment', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sa_spaeng", "dataset_name": "lince", "evaluation": {"accuracy": 0.5960193652501344}, "template_name": "express sentiment"}, "negation template": {"arguments": "Namespace(config_name=None, dataset_config_name='sa_spaeng', dataset_name='lince', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='negation template', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sa_spaeng", "dataset_name": "lince", "evaluation": {"accuracy": 0.36847767616998384}, "template_name": "negation template"}, "original poster expressed sentiment": {"arguments": "Namespace(config_name=None, dataset_config_name='sa_spaeng', dataset_name='lince', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='original poster expressed sentiment', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sa_spaeng", "dataset_name": "lince", "evaluation": {"accuracy": 0.6008606777837547}, "template_name": "original poster expressed sentiment"}, "sentiment trying to express": {"arguments": "Namespace(config_name=None, dataset_config_name='sa_spaeng', dataset_name='lince', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='sentiment trying to express', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sa_spaeng", "dataset_name": "lince", "evaluation": {"accuracy": 0.5954814416352878}, "template_name": "sentiment trying to express"}, "the author seem": {"arguments": "Namespace(config_name=None, dataset_config_name='sa_spaeng', dataset_name='lince', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='the author seem', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "sa_spaeng", "dataset_name": "lince", "evaluation": {"accuracy": 0.5965572888649812}, "template_name": "the author seem"}}, "math_qa_None": {"choose_correct_og": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='math_qa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='choose_correct_og', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "math_qa", "evaluation": {"accuracy": 0.22981574539363483}, "template_name": "choose_correct_og"}, "first_choice_then_problem": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='math_qa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='first_choice_then_problem', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "math_qa", "evaluation": {"accuracy": 0.192964824120603}, "template_name": "first_choice_then_problem"}, "gre_problem": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='math_qa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='gre_problem', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "math_qa", "evaluation": {"accuracy": 0.2184254606365159}, "template_name": "gre_problem"}, "pick_the_correct": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='math_qa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='pick_the_correct', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "math_qa", "evaluation": {"accuracy": 0.2150753768844221}, "template_name": "pick_the_correct"}, "problem_set_type": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='math_qa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='problem_set_type', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "math_qa", "evaluation": {"accuracy": 0.4737018425460637}, "template_name": "problem_set_type"}}, "mlsum_es": {"layman_summ_es": {"bleu": 3.606126125049115, "bleu_stderr": 0.1777428611437274, "rouge1_fmeasure": 0.22369539928629967, "rouge1_fmeasure_stderr": 0.002355115682657273, "rouge1_precision": 0.24897524860913609, "rouge1_precision_stderr": 0.002769352726963559, "rouge1_recall": 0.24862486574088613, "rouge1_recall_stderr": 0.0030999097459434703, "rouge2_fmeasure": 0.06070627110770783, "rouge2_fmeasure_stderr": 0.001726497816324924, "rouge2_precision": 0.06634301490578277, "rouge2_precision_stderr": 0.002048301218559081, "rouge2_recall": 0.06986753000720762, "rouge2_recall_stderr": 0.002064436828187958, "rougeL_fmeasure": 0.1716590640884715, "rougeL_fmeasure_stderr": 0.001979894431520568, "rougeL_precision": 0.1940954836427296, "rougeL_precision_stderr": 0.002431855538295837, "rougeL_recall": 0.18959437921291328, "rougeL_recall_stderr": 0.0025078934952384247, "rougeLsum_fmeasure": 0.17657777023351298, "rougeLsum_fmeasure_stderr": 0.002018383139682419, "rougeLsum_precision": 0.19939821083949505, "rougeLsum_precision_stderr": 0.0024800156414648273, "rougeLsum_recall": 0.19543176650107444, "rougeLsum_recall_stderr": 0.0025892786548937794}, "palm_prompt": {"bleu": 4.155428402841844, "bleu_stderr": 0.13385139476742874, "rouge1_fmeasure": 0.25337122760428693, "rouge1_fmeasure_stderr": 0.002292834308471866, "rouge1_precision": 0.24619254732411308, "rouge1_precision_stderr": 0.0024831470256754013, "rouge1_recall": 0.3117827742604337, "rouge1_recall_stderr": 0.003195536871718303, "rouge2_fmeasure": 0.0764102814843215, "rouge2_fmeasure_stderr": 0.0018719981450326982, "rouge2_precision": 0.07273490542705281, "rouge2_precision_stderr": 0.0018966264658225132, "rouge2_recall": 0.09639944399072355, "rouge2_recall_stderr": 0.0023803811434546587, "rougeL_fmeasure": 0.19022987411946263, "rougeL_fmeasure_stderr": 0.0019583024425978233, "rougeL_precision": 0.18574476795133907, "rougeL_precision_stderr": 0.0021245793603333414, "rougeL_recall": 0.2339554472733289, "rougeL_recall_stderr": 0.0026473792404859554, "rougeLsum_fmeasure": 0.1982216464293535, "rougeLsum_fmeasure_stderr": 0.002000958689349937, "rougeLsum_precision": 0.19337241472874026, "rougeLsum_precision_stderr": 0.002172473379459937, "rougeLsum_recall": 0.24449955120220104, "rougeLsum_recall_stderr": 0.0027610430609216626}, "summarise_this_in_es_few_sentences": {"bleu": 2.7821053236675306, "bleu_stderr": 0.11547426906777203, "rouge1_fmeasure": 0.20977060183296548, "rouge1_fmeasure_stderr": 0.002215423821196786, "rouge1_precision": 0.21888011387446746, "rouge1_precision_stderr": 0.002467769363765386, "rouge1_recall": 0.2599926409001897, "rouge1_recall_stderr": 0.003199531151457232, "rouge2_fmeasure": 0.05234733312406368, "rouge2_fmeasure_stderr": 0.0015580839680454735, "rouge2_precision": 0.05195446332105216, "rouge2_precision_stderr": 0.0016363375624019096, "rouge2_recall": 0.0686141481900639, "rouge2_recall_stderr": 0.002095509341583397, "rougeL_fmeasure": 0.15879516629311632, "rougeL_fmeasure_stderr": 0.0018075014060560566, "rougeL_precision": 0.16853408734216013, "rougeL_precision_stderr": 0.0021008078358421135, "rougeL_recall": 0.19597150935158797, "rougeL_recall_stderr": 0.0025551844547328268, "rougeLsum_fmeasure": 0.16512892741582888, "rougeLsum_fmeasure_stderr": 0.00185597817383672, "rougeLsum_precision": 0.17450463895712406, "rougeLsum_precision_stderr": 0.0021431721861983684, "rougeLsum_recall": 0.20509730700063683, "rougeLsum_recall_stderr": 0.002701824866588575}}, "movie_rationales_None": {"Evidences + review": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='movie_rationales', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Evidences + review', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "movie_rationales", "evaluation": {"accuracy": 0.985}, "template_name": "Evidences + review"}, "Evidences sentiment classification": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='movie_rationales', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Evidences sentiment classification', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "movie_rationales", "evaluation": {"accuracy": 0.995}, "template_name": "Evidences sentiment classification"}, "Standard binary sentiment analysis": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='movie_rationales', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Standard binary sentiment analysis', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "movie_rationales", "evaluation": {"accuracy": 0.955}, "template_name": "Standard binary sentiment analysis"}}, "mwsc_None": {"in-the-sentence": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='mwsc', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='in-the-sentence', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "mwsc", "evaluation": {"accuracy": 0.6829268292682927}, "template_name": "in-the-sentence"}, "in-the-sentence-question-first": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='mwsc', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='in-the-sentence-question-first', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "mwsc", "evaluation": {"accuracy": 0.6585365853658537}, "template_name": "in-the-sentence-question-first"}, "is-correct": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='mwsc', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='is-correct', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "mwsc", "evaluation": {"accuracy": 0.7195121951219512}, "template_name": "is-correct"}, "options-or": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='mwsc', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='options-or', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "mwsc", "evaluation": {"accuracy": 0.8048780487804879}, "template_name": "options-or"}, "what-think": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='mwsc', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='what-think', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "mwsc", "evaluation": {"accuracy": 0.7682926829268293}, "template_name": "what-think"}}, "onestop_english_None": {"ara_context": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='onestop_english', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='ara_context', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "onestop_english", "evaluation": {"accuracy": 0.4673721340388007}, "template_name": "ara_context"}, "assess": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='onestop_english', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='assess', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "onestop_english", "evaluation": {"accuracy": 0.3350970017636684}, "template_name": "assess"}, "determine_reading_level_from_the_first_three_sentences": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='onestop_english', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='determine_reading_level_from_the_first_three_sentences', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "onestop_english", "evaluation": {"accuracy": 0.5308641975308642}, "template_name": "determine_reading_level_from_the_first_three_sentences"}, "esl_context": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='onestop_english', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='esl_context', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "onestop_english", "evaluation": {"accuracy": 0.41798941798941797}, "template_name": "esl_context"}, "esl_variation": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='onestop_english', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='esl_variation', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "onestop_english", "evaluation": {"accuracy": 0.3386243386243386}, "template_name": "esl_variation"}}, "poem_sentiment_None": {"guess_sentiment_without_options_variation_1": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='poem_sentiment', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='guess_sentiment_without_options_variation_1', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "poem_sentiment", "evaluation": {"accuracy": 0.20952380952380953}, "template_name": "guess_sentiment_without_options_variation_1"}, "most_appropriate_sentiment": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='poem_sentiment', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='most_appropriate_sentiment', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "poem_sentiment", "evaluation": {"accuracy": 0.23809523809523808}, "template_name": "most_appropriate_sentiment"}, "positive_or_negative_sentiment_variation_1": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='poem_sentiment', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='positive_or_negative_sentiment_variation_1', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "poem_sentiment", "evaluation": {"accuracy": 0.23809523809523808}, "template_name": "positive_or_negative_sentiment_variation_1"}, "positive_or_negative_sentiment_variation_2": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='poem_sentiment', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='positive_or_negative_sentiment_variation_2', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "poem_sentiment", "evaluation": {"accuracy": 0.23809523809523808}, "template_name": "positive_or_negative_sentiment_variation_2"}, "question_answer_format": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='poem_sentiment', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='question_answer_format', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "poem_sentiment", "evaluation": {"accuracy": 0.24761904761904763}, "template_name": "question_answer_format"}}, "pubmed_qa_pqa_labeled": {"Long Answer to Final Decision": {"arguments": "Namespace(config_name=None, dataset_config_name='pqa_labeled', dataset_name='pubmed_qa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='Long Answer to Final Decision', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "pqa_labeled", "dataset_name": "pubmed_qa", "evaluation": {"accuracy": 0.704}, "template_name": "Long Answer to Final Decision"}, "Question Answering (Short)": {"arguments": "Namespace(config_name=None, dataset_config_name='pqa_labeled', dataset_name='pubmed_qa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='Question Answering (Short)', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "pqa_labeled", "dataset_name": "pubmed_qa", "evaluation": {"accuracy": 0.744}, "template_name": "Question Answering (Short)"}}, "riddle_sense_None": {"answer_given_question_without_options": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='riddle_sense', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='answer_given_question_without_options', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "riddle_sense", "evaluation": {"accuracy": 0.5925563173359452}, "template_name": "answer_given_question_without_options"}, "most_suitable_answer": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='riddle_sense', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='most_suitable_answer', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "riddle_sense", "evaluation": {"accuracy": 0.5161606268364348}, "template_name": "most_suitable_answer"}, "question_answering": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='riddle_sense', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='question_answering', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "riddle_sense", "evaluation": {"accuracy": 0.47502448579823703}, "template_name": "question_answering"}, "question_to_answer_index": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='riddle_sense', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='question_to_answer_index', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "riddle_sense", "evaluation": {"accuracy": 0.49657198824681686}, "template_name": "question_to_answer_index"}}, "scicite_None": {"Classify intent": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='scicite', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Classify intent', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "scicite", "evaluation": {"accuracy": 0.6266375545851528}, "template_name": "Classify intent"}, "Classify intent (choices first)": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='scicite', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Classify intent (choices first)', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "scicite", "evaluation": {"accuracy": 0.4705240174672489}, "template_name": "Classify intent (choices first)"}, "Classify intent (select choice)": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='scicite', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Classify intent (select choice)', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "scicite", "evaluation": {"accuracy": 0.4388646288209607}, "template_name": "Classify intent (select choice)"}, "Classify intent w/section (select choice)": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='scicite', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Classify intent w/section (select choice)', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "scicite", "evaluation": {"accuracy": 0.5491266375545851}, "template_name": "Classify intent w/section (select choice)"}, "can_describe": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='scicite', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='can_describe', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "scicite", "evaluation": {"accuracy": 0.6342794759825328}, "template_name": "can_describe"}}, "selqa_answer_selection_analysis": {"is-he-talking-about": {"arguments": "Namespace(config_name=None, dataset_config_name='answer_selection_analysis', dataset_name='selqa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='is-he-talking-about', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "answer_selection_analysis", "dataset_name": "selqa", "evaluation": {"accuracy": 0.9184713375796179}, "template_name": "is-he-talking-about"}, "make-sense-rand": {"arguments": "Namespace(config_name=None, dataset_config_name='answer_selection_analysis', dataset_name='selqa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='make-sense-rand', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "answer_selection_analysis", "dataset_name": "selqa", "evaluation": {"accuracy": 0.9426751592356688}, "template_name": "make-sense-rand"}, "which-answer-1st-vs-random": {"arguments": "Namespace(config_name=None, dataset_config_name='answer_selection_analysis', dataset_name='selqa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='which-answer-1st-vs-random', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "answer_selection_analysis", "dataset_name": "selqa", "evaluation": {"accuracy": 0.9006369426751593}, "template_name": "which-answer-1st-vs-random"}, "would-make-sense-qu-rand": {"arguments": "Namespace(config_name=None, dataset_config_name='answer_selection_analysis', dataset_name='selqa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='would-make-sense-qu-rand', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": "answer_selection_analysis", "dataset_name": "selqa", "evaluation": {"accuracy": 0.910828025477707}, "template_name": "would-make-sense-qu-rand"}}, "snips_built_in_intents_None": {"categorize_query": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='snips_built_in_intents', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='categorize_query', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "snips_built_in_intents", "evaluation": {"accuracy": 0.7865853658536586}, "template_name": "categorize_query"}, "categorize_query_brief": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='snips_built_in_intents', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='categorize_query_brief', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "snips_built_in_intents", "evaluation": {"accuracy": 0.7012195121951219}, "template_name": "categorize_query_brief"}, "intent_query": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='snips_built_in_intents', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='intent_query', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "snips_built_in_intents", "evaluation": {"accuracy": 0.4176829268292683}, "template_name": "intent_query"}, "query_intent": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='snips_built_in_intents', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='query_intent', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "snips_built_in_intents", "evaluation": {"accuracy": 0.7835365853658537}, "template_name": "query_intent"}, "voice_intent": {"arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='snips_built_in_intents', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step498/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='voice_intent', tokenizer_name=None, use_slow_tokenizer=False)", "dataset_config_name": null, "dataset_name": "snips_built_in_intents", "evaluation": {"accuracy": 0.7012195121951219}, "template_name": "voice_intent"}}, "wmt14_fr_en_en-fr": {"a_good_translation-en-fr-source+target": {"bleu": 26.028441633496957, "bleu_stderr": 0.2862988886727081}, "a_good_translation-en-fr-target": {"bleu": 26.105356968174956, "bleu_stderr": 0.5439462553854422}, "gpt3-en-fr": {"bleu": 17.923414272364486, "bleu_stderr": 0.6888556307061847}, "version-en-fr-target": {"bleu": 23.518794525011923, "bleu_stderr": 0.37640905744448383}, "xglm-en-fr-target": {"bleu": 27.490320032481684, "bleu_stderr": 0.23073871057079665}}, "wmt14_fr_en_fr-en": {"a_good_translation-fr-en-source+target": {"bleu": 22.344520948134363, "bleu_stderr": 0.5313668861761752}, "a_good_translation-fr-en-target": {"bleu": 29.883879388882114, "bleu_stderr": 0.37879956515947893}, "gpt3-fr-en": {"bleu": 28.97671081332691, "bleu_stderr": 0.3781852673062757}, "version-fr-en-target": {"bleu": 33.70883690137962, "bleu_stderr": 0.35450837635429083}, "xglm-fr-en-target": {"bleu": 26.028992585410116, "bleu_stderr": 0.34940072782524134}}, "wmt14_hi_en_en-hi": {"a_good_translation-en-hi-source+target": {"bleu": 9.550778502148496, "bleu_stderr": 0.29692742965341107}, "a_good_translation-en-hi-target": {"bleu": 10.547062820945454, "bleu_stderr": 0.2804478168203892}, "gpt-3-en-hi-target": {"bleu": 3.4030829410154912, "bleu_stderr": 0.2921082537198064}, "version-en-hi-target": {"bleu": 11.49224530123302, "bleu_stderr": 0.31153966689284257}, "xglm-en-hi-target": {"bleu": 6.980407323250922, "bleu_stderr": 0.16891945366229555}}, "wmt14_hi_en_hi-en": {"a_good_translation-hi-en-source+target": {"bleu": 4.963973034828739, "bleu_stderr": 0.1969483704844997}, "a_good_translation-hi-en-target": {"bleu": 11.802320249982353, "bleu_stderr": 0.3390801992684086}, "gpt-3-hi-en-target": {"bleu": 1.9401417583412613e-13, "bleu_stderr": 3.676244357840319e-12}, "version-hi-en-target": {"bleu": 21.17559943306028, "bleu_stderr": 0.3691754541018668}, "xglm-hi-en-target": {"bleu": 18.34661289471336, "bleu_stderr": 0.3049159109479028}}} |