diff --git a/.gitattributes b/.gitattributes index c7d9f3332a950355d5a77d85000f05e6f45435ea..3d2dd626471fa69e7703e256ab511ff13a0ab48f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -32,3 +32,27 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_1.jsonl filter=lfs diff=lfs merge=lfs -text +evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_2.jsonl filter=lfs diff=lfs merge=lfs -text +evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_4.jsonl filter=lfs diff=lfs merge=lfs -text +evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_2.jsonl filter=lfs diff=lfs merge=lfs -text +evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_5.jsonl filter=lfs diff=lfs merge=lfs -text +evaluation/generation/examples.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_0.jsonl filter=lfs diff=lfs merge=lfs -text +evaluation/generation/examples.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_1.jsonl filter=lfs diff=lfs merge=lfs -text +evaluation/generation/examples.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl filter=lfs diff=lfs merge=lfs -text +evaluation/generation/examples.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl filter=lfs diff=lfs merge=lfs -text +evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_3.jsonl filter=lfs diff=lfs merge=lfs -text +evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_1.jsonl filter=lfs diff=lfs merge=lfs -text +evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_3.jsonl filter=lfs diff=lfs merge=lfs -text +evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_4.jsonl filter=lfs diff=lfs merge=lfs -text +evaluation/generation/examples.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_2.jsonl filter=lfs diff=lfs merge=lfs -text +evaluation/generation/examples.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_4.jsonl filter=lfs diff=lfs merge=lfs -text +evaluation/generation/examples.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl filter=lfs diff=lfs merge=lfs -text +evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_0.jsonl filter=lfs diff=lfs merge=lfs -text +evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_5.jsonl filter=lfs diff=lfs merge=lfs -text +evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_0.jsonl filter=lfs diff=lfs merge=lfs -text +evaluation/generation/examples.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_5.jsonl filter=lfs diff=lfs merge=lfs -text +evaluation/generation/examples.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl filter=lfs diff=lfs merge=lfs -text +evaluation/generation/examples.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl filter=lfs diff=lfs merge=lfs -text +evaluation/generation/examples.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_3.jsonl filter=lfs diff=lfs merge=lfs -text +evaluation/generation/examples.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl filter=lfs diff=lfs merge=lfs -text diff --git a/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_0.json b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_0.json new file mode 100644 index 0000000000000000000000000000000000000000..1641cf8a3aafddf22c302c57102b5b3647f430cc --- /dev/null +++ b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.3552258551749086, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.02004694008107349}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.07050133636806707, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002333415676412846}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.28446739915287816, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.00508507867804917}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.09756985013732229, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0019859335153510982}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.030757559038758858, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0013508528253275045}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.13243423248660713, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.003309647236077234}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.043506125605077994, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0012188241069113202}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.0666262101718174, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002246557443572289}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.2713467656341814, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004908738323537121}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.09211119558985698, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0018415131593686243}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.06617315282201941, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0022690978668772816}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.2645417006842324, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004684946802992825}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.0909840574356387, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018721336661780886}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_1.json b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_1.json new file mode 100644 index 0000000000000000000000000000000000000000..ab90a03ad36519f656c9911a83b70a54acff327d --- /dev/null +++ b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.455214213531198, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.027035455129998113}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.10632219422720153, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0035658627377499956}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.2768550528667316, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004832345662545228}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.12895359157667396, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0030401563578052807}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.04940516839667132, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0021584007358498585}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.13404371723129116, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.003324815594078369}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.06090727845863285, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0019209569415793166}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.09546267515845965, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.003141923276845849}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.25843481586517253, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004474300115661777}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.11713897308464187, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0026334790087808417}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.09758720980677024, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0032165997299972056}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.2609559973772103, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.00448462250486365}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.11922109901669682, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002697373964352153}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_2.json b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_2.json new file mode 100644 index 0000000000000000000000000000000000000000..c4efceb5ae4ee4ae10cb47525b1d82374966f03f --- /dev/null +++ b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.43672089515970186, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.021621818007392418}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.12427774623745924, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.00396632764513016}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.31153733817117696, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004862182707219614}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.15107097447841042, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0035335849672525825}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.061122334555414014, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0024752531890197427}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.1553537719896923, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.003521786839397489}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.073579744703458, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0022399508108041687}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.10862059194070799, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0032794570812240747}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.2880040612685418, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004452369074617207}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.1343719484645983, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0029138401067494405}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.11186894190492254, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0034133834955339783}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.2921099418130985, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0044981987732144475}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.13751466322243355, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0030168231727520462}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_3.json b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_3.json new file mode 100644 index 0000000000000000000000000000000000000000..9c87effbb8fbcb44b054792167371ccaeb5a52f6 --- /dev/null +++ b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.6005511019489902, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.03451530801039688}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.13595589597527172, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.004222986885593746}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.33095418350217154, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0051126113252587}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.16481776831623232, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0038682235238341116}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.06960048861029224, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0027985379538191905}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.16775824178008983, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0037608840409452783}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.08281371490457289, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.002614718922183248}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.11879183737182318, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.003583977041097007}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.30406006315984274, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004643180781154661}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.1462708825192011, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0032850506015328065}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.12214153222625562, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0037097941549426336}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.3076377720124529, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0046729883356302375}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.14946371142957862, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0033711111276765737}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_4.json b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_4.json new file mode 100644 index 0000000000000000000000000000000000000000..56f627cc95c8926e25ed7cb032d6999b95a5138e --- /dev/null +++ b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.6245188789842254, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.043490522758658445}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.14786580825593648, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.004627492119408001}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.3383750446597362, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004749410285992037}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.17222603573029266, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.003921731867593103}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.07698153502052321, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0031209348406096736}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.1718812686921174, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.003529827492808453}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.08713860881109199, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.002642546281127278}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.1294373882259293, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.003923291361672725}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.31167522711756507, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.00437544273005526}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.15325650518011516, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0033232600369196875}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.13331179138633883, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.004083238268648474}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.3154382483492274, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0043923097345808845}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.15679817099191065, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.003436174687018384}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_5.json b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_5.json new file mode 100644 index 0000000000000000000000000000000000000000..a57138453f77788be718e0d1dbca5f01bb50420c --- /dev/null +++ b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.690349543623733, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.037462509241486704}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.15622015160603553, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.004716488431001402}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.34792374373732987, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004822507263881878}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.18047036475829212, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.004008854616724969}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.08275177731282365, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0031808930078121365}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.18065193154470569, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.003617404779594222}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.0930691012178671, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0027170170506852424}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.13603046372345778, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.004020369923896149}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.3181992629421937, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.00438340700122448}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.15947807280240164, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.00339577303947939}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.14015533154496237, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.004159324286093359}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.3231216711299358, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004412774889842664}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.16344169338350165, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0034931588485009957}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_0.json b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_0.json new file mode 100644 index 0000000000000000000000000000000000000000..868d16502542ec352eedcbf4c8548c1702399a9f --- /dev/null +++ b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.10101653488556242, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0032530374257675938}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.12382359005658215, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003311350263072308}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.09262226734827564, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002438006555562961}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.021119629322483437, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.000867268036727509}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.03184028832690896, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001315196256520591}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.023458200677554667, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009269048009616802}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.08164131674574245, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002890373817028126}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.09744037722102081, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002611244990111791}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.0717645545672235, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0018475862298041712}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.09506262524089595, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.003153876364088973}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.11499957069726778, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0030864345858981576}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.08593512322791458, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0022685460052984064}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 1.7401292150366783, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.05870950000759347}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_1.json b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_1.json new file mode 100644 index 0000000000000000000000000000000000000000..953d815738626cba15758d847c2185be42054bf2 --- /dev/null +++ b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.2169406307055522, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.003725498619269963}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.1602777922005014, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0023194868352840897}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.15283090429353868, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0019824413317849196}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.05250080669816919, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.002263016873913571}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.029595318390725434, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0011280692140449152}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.030516647625530857, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0011026203736681755}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.17465569803991965, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.003198873475886084}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.1267991344776356, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.001844558639815861}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.12076719641202475, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015785014789520583}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.20508454330750828, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0035353326008678463}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.1517146954188497, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0021834314823700506}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.1443601231383759, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018505558858251069}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 1.8195228421640262, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.08352209916798846}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_2.json b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_2.json new file mode 100644 index 0000000000000000000000000000000000000000..2c42b606e1256de607680236f53beeb15ef10c85 --- /dev/null +++ b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.3400358202564136, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.004368456331414668}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.19549655317783554, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0026239543642537643}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.2060405071459818, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002251177818368773}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.10414705512855632, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0029764211051013624}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.052252858151520365, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0013882181124994304}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.05659098002273207, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0013825154700548739}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.2760810172165432, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.003817392548869396}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.15431925374992486, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0020853728076912085}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.1632339625235188, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017925107334137358}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.32272701619444116, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.004237054067018847}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.18387985811231844, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002467066081787791}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.19413584042277324, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0021219960169286357}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 2.8890982233924274, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.07489960916470248}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_3.json b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_3.json new file mode 100644 index 0000000000000000000000000000000000000000..8ddfc22bcfce23bc2d22c09065a6844a4cd62ff2 --- /dev/null +++ b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.3143827035331544, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.004814681871436694}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.16312653314207656, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0028028425609925864}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.17768534554173257, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002497324854394021}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.10012288852389835, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0030408513717817094}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.04640858531424939, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0014571215580260206}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.0510208899127158, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0014011608421103333}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.25883690093372125, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.004162927984690604}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.13073894232805758, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0022683935578694356}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.14320515112174437, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0020306931695434237}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.2982192683665052, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.004629601809394366}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.1526514018679153, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0026042434071650534}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.16704762132090992, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002342688337895595}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 1.890195824238381, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.07805309283654865}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_4.json b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_4.json new file mode 100644 index 0000000000000000000000000000000000000000..94932d0f0f3a6c04bfda3182744a4baf88f979d6 --- /dev/null +++ b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.10233714832400315, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.00387170332308452}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.052719560358311074, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0021360645688620044}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.05712785595257542, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002109365575892117}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.03236639334422414, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0020360160794573286}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.014676186959067014, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.000991238095490706}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.015994832602106063, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009271960700065752}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.08539656258425662, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.003320722236518427}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.04253430518648541, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0017210052280975444}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.0463830391676825, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001707583877770572}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.09625340675143208, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0036657803196257096}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.048881870192824625, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0019848013068469554}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.053156186981754575, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0019608311676159664}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.016910620160853877, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.00364577886299514}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_5.json b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_5.json new file mode 100644 index 0000000000000000000000000000000000000000..4990c98adf69f60f4d5cd366dcec08ced7ccdb4a --- /dev/null +++ b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.018095685011338746, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.00185447805960437}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.008607244652028168, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0009194028038185624}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.0091861211614601, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0008969173014851819}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.006014959811595776, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0009805624350940475}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.0023894753339408787, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0003776589361725097}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.002725147355524338, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0004048462437499985}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.015576109016400224, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001653922803893889}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.007132007829695769, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0007575676626861256}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.007656339590582189, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0007546272499233359}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.01720829814617215, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001777168147572916}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.00818345077420238, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.000884640095931326}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.008689332392498271, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.000852751466212803}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 1.4571231589477147e-17, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 2.554040800257738e-15}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_0.json b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_0.json new file mode 100644 index 0000000000000000000000000000000000000000..961bedae793dc53b19dd63b5931ecae7bc7aaa14 --- /dev/null +++ b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 1.8176691711249537, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.05995542560680462}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.11995524671016493, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002674303424141786}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.15915193434349853, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0032972190406054414}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.12962478833198351, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0026733541462707695}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.03280986461326413, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0011206239984645086}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.04550105277967982, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0016902484384178876}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.03603975902395983, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0012403288414746835}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.09452651957178927, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0018963480115027633}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.12988017210774444, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0025557466013964663}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.10376435983061806, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0019651761643805117}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.10794550532453419, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002439267929763401}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.14233639462065437, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.00302666138111876}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.11626650837633533, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002446339204031}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_1.json b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_1.json new file mode 100644 index 0000000000000000000000000000000000000000..a41cb2bc6fad7723cf01b65a082c5f8b727038d6 --- /dev/null +++ b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 11.849671487305415, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.1360222064360011}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.54188605893543, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.003246758820446708}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.4314024243954027, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0029616850251012895}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.45449662451133166, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002333690577738812}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.2562791831846635, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.002639527749513042}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.20134452678189727, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0021665892377752327}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.2121926766511087, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0020146533728953066}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.3968055602906763, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002884957533719286}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.3135108993655708, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002421217906260913}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.3309578358109194, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.002047514034389017}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.44543454886093686, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.003160000527797213}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.3537585371537755, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0027469285035271475}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.3729160698976979, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002328237682996928}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_2.json b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_2.json new file mode 100644 index 0000000000000000000000000000000000000000..3def69f626886d9479ca6d1dfab3f830957ee8b6 --- /dev/null +++ b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 14.500866321633556, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.21652939369858226}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.5577858917634193, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.003247785025897439}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.4675413035402858, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002820078928761888}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.4837551797964578, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0022147223749886054}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.27932023326938354, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0026973630619244273}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.23167147979272804, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0022172627136572156}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.23945439459321566, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.002051628654300672}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.41572829682147167, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0029006552995420294}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.3478030944709029, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002413752959029049}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.3598482761759142, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0020447863689909054}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.4676801531783358, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0031627544428160766}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.39184282368555173, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002694948881819574}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.4056412361905643, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0022938418811786596}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_3.json b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_3.json new file mode 100644 index 0000000000000000000000000000000000000000..4164cb9d1a5ff28d93481aed74e0586b762b4728 --- /dev/null +++ b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 14.99356088381018, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.1939852688470928}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.560904943171157, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.003222339717903183}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.46948710399711374, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0027577686714870798}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.4885274020508225, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002197446931041306}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.2844738744368861, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.002710055422490507}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.23612605197891567, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0022922087007235313}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.24528072211886423, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0021164271571705827}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.42065354031616564, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0029044985788341323}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.351254841158613, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0024156544408651705}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.36547104984192025, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0020631564845025475}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.473294687864865, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0031673965755000533}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.39610686350311647, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0026942679498965384}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.4121617578480254, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002317480403210135}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_4.json b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_4.json new file mode 100644 index 0000000000000000000000000000000000000000..03ec8ebefbf7ae8b6afc25f3556418a5690e9d05 --- /dev/null +++ b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 15.136408152402511, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.15749919205112253}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.5635851478232797, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.003229028521004473}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.46904362683842155, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002666498752628597}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.49008120449442055, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0021552779856922706}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.2877028860236848, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0027336630161171856}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.23648543820909804, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0022192023733188957}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.2472610815193413, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.002097203486030851}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.4232377532809423, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002905210399187484}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.3512864132211812, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002353821229944638}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.3672468672821708, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0020504167271609996}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.4758982382137059, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.003149472337396323}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.3964206507125601, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0026288828261706028}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.4139945135555939, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0022731887846098205}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_5.json b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_5.json new file mode 100644 index 0000000000000000000000000000000000000000..3aaf91ebb8a0d49f45558a27a3edc57cf9835ee0 --- /dev/null +++ b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 14.94032761568535, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.16102163935491987}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.5647971657629298, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0032050177663230373}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.46466204351732154, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002636551188261336}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.4891066764513762, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002148432709098188}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.28959208448012996, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.002723369175901688}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.2349545016224875, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0021871784713310173}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.2475971300723347, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0020740967240209687}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.4250552095964905, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002889497509660382}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.3494271521246103, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0023269838649924057}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.3676953770310896, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.002043411505256857}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.47747050984587586, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0031407117559106392}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.3928622107079271, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0025636957992988966}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.41342063708708626, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002240351112078022}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_0.json b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_0.json new file mode 100644 index 0000000000000000000000000000000000000000..a9e03f575642808c75bc5e3eb2200bfdfb557d70 --- /dev/null +++ b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.14726644892868132, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0024412028465323945}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.33236412764323403, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004292159791327218}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.19805083031043677, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0025145733920038504}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.03338148146365739, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0014454654718671273}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.0775963205240964, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.002642772816149882}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.045029580196435964, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0015159796252548507}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.11274328349904972, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0021403391288850487}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.253884590300403, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0033701265161261294}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.150708876966239, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0019106881335396962}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.11711469488381401, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0022541549353202995}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.26459055857012664, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0037902287339387276}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.15690226758738812, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0021606792108516933}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 1.8631748315575967, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.06480985196854291}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_1.json b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_1.json new file mode 100644 index 0000000000000000000000000000000000000000..728af6b85dcb26fd19694cfc330a673d3f7a34f3 --- /dev/null +++ b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.201520198934412, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0036780464289758986}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.28041072507766257, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004470154542708083}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.21492593421689296, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0031040934015378273}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.04262064763605291, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0020875394875545616}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.059979799287456115, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.002459545390487186}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.045086359683801695, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0019034075416325229}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.15057288573196015, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002929976392840731}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.20592545696933923, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0033110515398117922}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.15921676436636747, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.002420430590555424}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.15642146751721803, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0029104557088062938}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.2199677896065397, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0037276875782133504}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.16736923260398065, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0025093332042761462}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 2.002473681001451, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.10305039390700386}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_2.json b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_2.json new file mode 100644 index 0000000000000000000000000000000000000000..4b089edf63d04c16bad8877d970ee91f405b5c3e --- /dev/null +++ b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.2088093664550383, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.003591375397921926}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.28037833021310865, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.00418073064186992}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.21928064355323418, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0029906506411637493}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.04282030935093115, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0018963252751686556}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.05836728040338388, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0022732577336655444}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.04498559646410484, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001797572147267301}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.15422100446453424, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0027848170802648774}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.20777843317638175, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0032512385248102136}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.16195880232863943, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.002341838899049842}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.15873674200935498, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0027827884387405604}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.21825435569219825, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0035822959483663586}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.16811888843078998, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0024215692583980006}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 1.9767413966669858, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.12553384851261437}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_3.json b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_3.json new file mode 100644 index 0000000000000000000000000000000000000000..7d0d0729a858a2137c586bc29aaca0b22cf259bd --- /dev/null +++ b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.2040643001997412, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.003978248036078804}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.27307347161473916, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004455443229628111}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.21363414740028736, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0033547861779570426}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.0427132375189158, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0020323931003614964}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.056734403722373934, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0023427635867331836}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.044315357975490785, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0018794354037841253}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.15146281632387826, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0031018365348921524}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.20314923616617458, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.003479004056873755}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.15848715345045697, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0026312535881803656}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.15567331452586086, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0030921047823091897}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.21288484516878498, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0037476662681603583}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.1642507173243999, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0026893440158164876}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 2.0020914675362764, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.14545306393065818}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_4.json b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_4.json new file mode 100644 index 0000000000000000000000000000000000000000..0a851a4d015837c05e070d4b5189ee9200f4a0f7 --- /dev/null +++ b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.05845678560810508, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.003653372410391946}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.06546523451488562, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.00397320947448918}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.054580713604579945, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0032077855926012644}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.011526059837135257, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0014149014965583362}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.013070566192947936, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0013385142420944767}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.010993328089942682, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0011929721928833114}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.044187543013943426, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0028507649598664116}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.04837933471428071, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0029701161810768185}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.04045511548154733, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.002398056707020803}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.045470269077929414, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002904632966614191}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.050635363980909044, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0031412698278665725}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.041976934818540805, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0024886979052922598}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.3313737532545398, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.057620671081541584}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_5.json b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_5.json new file mode 100644 index 0000000000000000000000000000000000000000..f67fea01fab9ef3639a44d7bfd30708ea2cda55a --- /dev/null +++ b/evaluation/generation/agg.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.0022940371923293613, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0007143995060460097}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.0017348914893632757, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0005253123299517621}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.001906885101987677, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0005780948322075463}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.00011486155354079882, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 8.136503714231734e-05}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.00010005717552887364, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 7.143995839898157e-05}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.00010679337289617054, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 7.591909328912052e-05}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.0016335291518965692, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0004934641653958932}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.001252059737745911, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0003636768915155132}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.001362800409731286, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0003965668916012721}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.0016839781479615478, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0005010040710225287}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.0013025087338108892, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0003738878391567918}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.001413249405796264, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0004059395809283557}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 1.8341021693095957e-44, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 2.1637012594857334e-38}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_0.jsonl b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..aa9609b3931f9c66253472673257f5e83fa36cf0 --- /dev/null +++ b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f65e18c599fabcd78e237cfd1a661b54e81533bf233f387bf8dbc3e5c6274b19 +size 4092974 diff --git a/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_1.jsonl b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c116217548e1299ab7813d073b5f989763bed60f --- /dev/null +++ b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f8b6de0b8d969705a83201746dd64306505612f2fc1b590c2551c56d91082e1 +size 4865061 diff --git a/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_2.jsonl b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ff3028115de26f1b27b494a52892d679a055293e --- /dev/null +++ b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3863fec2a3901fe20a31aa63b6c55c8d4b65815d28cd570153fa1b04c19a9c9f +size 5749269 diff --git a/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_3.jsonl b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d9be2a8f5622e7354d725b8c4d939ca12dd1eb6f --- /dev/null +++ b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed1958cd8546f757d6885843c7a090feb8a23e065a3b6db61f2a5e7146e1760f +size 6635061 diff --git a/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_4.jsonl b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0c3c3b4b02904f11cd7b8fb9bd7981aa3ba5152c --- /dev/null +++ b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be8f57c35cbacaebb4bb8e860afa95df1c04589ba64fe301702fee80e66e4e92 +size 7516438 diff --git a/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_5.jsonl b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0b78421ee7c3642ec7445b44c9b0ce50639a65c2 --- /dev/null +++ b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6ddc62b4d9f7ab7e482748e898929a797db971685b53bb2cde79d5841c8d495 +size 8385484 diff --git a/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_0.jsonl b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1771c4400ee863fca4b49adb2220b0d8ee22ccae --- /dev/null +++ b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53c030e6cb4afcc9eee20866810a3a3f6bd196726772c9ef70b82da60ddecacb +size 7213592 diff --git a/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_1.jsonl b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b0f7b7482b6cfad1d9625389a6a3852653387c06 --- /dev/null +++ b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5588b892f5100c745b28c0ce21011c40a281cc7715052737d787d6f9fcc8362 +size 12990746 diff --git a/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_2.jsonl b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a2fe69e270c748fecb576665d10e239736ccf16c --- /dev/null +++ b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e91ee1a7871e720d003de30bff208a50b27594d186628819528347e85db90e72 +size 18474586 diff --git a/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_3.jsonl b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..275e6502e4479eb627bd555ecceb6f6ea2606d3f --- /dev/null +++ b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bec3e7fb4056e7e9bbc10b31a271495337a4f40da103b071e455c0ac9d00b91 +size 23939904 diff --git a/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_4.jsonl b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..9d4c355dd82d1d68e388a911fbde64c006a48934 --- /dev/null +++ b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e378cdb1041d1cfca22327cb4ea851575a369561603fca976d809bbc1ca32a6b +size 29344701 diff --git a/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_5.jsonl b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a678355ee1818abd97cf941ca595c60ebcea1be4 --- /dev/null +++ b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2f38154716aca59e747539d0ba9218077d54a956c1745855deb2dca1be67fc3 +size 34780805 diff --git a/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..170ece3c1ea50a9e7e25e60401bc9305cbd7dd2d --- /dev/null +++ b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ea6a268ec9c8897066cbc64ac077e3d9dd815a96ed02326fcaa8d3ba873c8aa +size 4184231 diff --git a/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8fb057c258db8da3faaf6f12f0111c66799cf9c4 --- /dev/null +++ b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0968b157b7a585011b804a3da5eb6f2f0a85fda4865d537b4d7f3e1b151759c +size 5013641 diff --git a/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..35269fb09c36241a16501166d400ba16803d43c2 --- /dev/null +++ b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d20937d45d88c29dc9b8f4de8369150e207f1c9b70b71e5311e55159558301ab +size 6109445 diff --git a/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3ec24ec5d72b0c365b1d1843f9d365e174d7ef3e --- /dev/null +++ b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cc4c5d2c00ac3c1330cb3475284fbf89be7f59c3a9e85fdd1344e130ae15388 +size 7186934 diff --git a/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b0a60c9e179e73e3a63cd0c5a684b8e80e47156e --- /dev/null +++ b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:437ce017a361972f7a015cb809a189991f51f2e30dc6ef710e0de1e74c5275a6 +size 8261309 diff --git a/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..fa0d2b09f5b3109e447d87bbf1528609271d53c1 --- /dev/null +++ b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:effdf46057b702351e3b679faaefd44c1544bb178b12ea5ae1051c2097e429ee +size 9342939 diff --git a/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_0.jsonl b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a45e7855706dcb4e848ca8dc9baa29063f931f73 --- /dev/null +++ b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28099ed84399ed261e13b68ccae0a11863c7a9343a8cae565b0dbbf0913c1e4f +size 2816952 diff --git a/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_1.jsonl b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..7f4a4a4f9be6109e262d923e78d61cff1d12c0e0 --- /dev/null +++ b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e870ecec6c0d3b95334321c1fd996baec8bf7dae838b612601ef409664a694b7 +size 4983742 diff --git a/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_2.jsonl b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..4bc3ba99b3311c824062584969786694bb5675bb --- /dev/null +++ b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77762ad05ad21ccb84acbbaba5e9da04f3cdab8420ad0b7e05b64c93404439f2 +size 7252078 diff --git a/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_3.jsonl b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b2a74f7a4783d2486e79a0e03b71029bf080a043 --- /dev/null +++ b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42821d6aaa0de3a81f1ea3730f9445e21ba5ac9d3cc4241b1e0d1d4712754b91 +size 9535583 diff --git a/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_4.jsonl b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..92f3a0cf9e56279f576da2d96509a8f1c2cd46aa --- /dev/null +++ b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abb1ea0500b907f37cea2d79443588f9b8eba1ce60bc6d5f1acf85a57cf13aa2 +size 11644219 diff --git a/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_5.jsonl b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..825a13e800d66d3b1509d0192a9d0e79ad6214d2 --- /dev/null +++ b/evaluation/generation/examples.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:172f9b45897e532361793c6bc4801c16ddf7e7ceab38b2dfa68da348185f3ea4 +size 13897329 diff --git a/evaluation/generation/merged.csv b/evaluation/generation/merged.csv new file mode 100644 index 0000000000000000000000000000000000000000..66adc3c1437f33f69cb2e3b94601fc269bc9ad0c --- /dev/null +++ b/evaluation/generation/merged.csv @@ -0,0 +1,53 @@ +dataset,fewshots,prompt,metric,value +e2e_nlg_cleaned,0,generate_text_restaurant,rouge2_fmeasure,0.03603975902395983 +e2e_nlg_cleaned,0,median,rouge2_fmeasure,0.03603975902395983 +e2e_nlg_cleaned,1,generate_text_restaurant,rouge2_fmeasure,0.2121926766511087 +e2e_nlg_cleaned,1,median,rouge2_fmeasure,0.2121926766511087 +e2e_nlg_cleaned,2,generate_text_restaurant,rouge2_fmeasure,0.23945439459321566 +e2e_nlg_cleaned,2,median,rouge2_fmeasure,0.23945439459321566 +e2e_nlg_cleaned,3,generate_text_restaurant,rouge2_fmeasure,0.24528072211886423 +e2e_nlg_cleaned,3,median,rouge2_fmeasure,0.24528072211886423 +e2e_nlg_cleaned,4,generate_text_restaurant,rouge2_fmeasure,0.2472610815193413 +e2e_nlg_cleaned,4,median,rouge2_fmeasure,0.2472610815193413 +e2e_nlg_cleaned,5,generate_text_restaurant,rouge2_fmeasure,0.2475971300723347 +e2e_nlg_cleaned,5,median,rouge2_fmeasure,0.2475971300723347 +e2e_nlg_cleaned,5,average,multiple,0.20463762732980406 +gem_xsum,0,article_DOC_summary,rouge2_fmeasure,0.045029580196435964 +gem_xsum,0,median,rouge2_fmeasure,0.045029580196435964 +gem_xsum,1,article_DOC_summary,rouge2_fmeasure,0.045086359683801695 +gem_xsum,1,median,rouge2_fmeasure,0.045086359683801695 +gem_xsum,2,article_DOC_summary,rouge2_fmeasure,0.04498559646410484 +gem_xsum,2,median,rouge2_fmeasure,0.04498559646410484 +gem_xsum,3,article_DOC_summary,rouge2_fmeasure,0.044315357975490785 +gem_xsum,3,median,rouge2_fmeasure,0.044315357975490785 +gem_xsum,4,article_DOC_summary,rouge2_fmeasure,0.010993328089942682 +gem_xsum,4,median,rouge2_fmeasure,0.010993328089942682 +gem_xsum,5,article_DOC_summary,rouge2_fmeasure,0.00010679337289617054 +gem_xsum,5,median,rouge2_fmeasure,0.00010679337289617054 +gem_xsum,5,average,multiple,0.03175283596377869 +web_nlg_en,0,PALM_prompt,rouge2_fmeasure,0.043506125605077994 +web_nlg_en,0,median,rouge2_fmeasure,0.043506125605077994 +web_nlg_en,1,PALM_prompt,rouge2_fmeasure,0.06090727845863285 +web_nlg_en,1,median,rouge2_fmeasure,0.06090727845863285 +web_nlg_en,2,PALM_prompt,rouge2_fmeasure,0.073579744703458 +web_nlg_en,2,median,rouge2_fmeasure,0.073579744703458 +web_nlg_en,3,PALM_prompt,rouge2_fmeasure,0.08281371490457289 +web_nlg_en,3,median,rouge2_fmeasure,0.08281371490457289 +web_nlg_en,4,PALM_prompt,rouge2_fmeasure,0.08713860881109199 +web_nlg_en,4,median,rouge2_fmeasure,0.08713860881109199 +web_nlg_en,5,PALM_prompt,rouge2_fmeasure,0.0930691012178671 +web_nlg_en,5,median,rouge2_fmeasure,0.0930691012178671 +web_nlg_en,5,average,multiple,0.0735024289501168 +wiki_lingua_en,0,tldr_en,rouge2_fmeasure,0.023458200677554667 +wiki_lingua_en,0,median,rouge2_fmeasure,0.023458200677554667 +wiki_lingua_en,1,tldr_en,rouge2_fmeasure,0.030516647625530857 +wiki_lingua_en,1,median,rouge2_fmeasure,0.030516647625530857 +wiki_lingua_en,2,tldr_en,rouge2_fmeasure,0.05659098002273207 +wiki_lingua_en,2,median,rouge2_fmeasure,0.05659098002273207 +wiki_lingua_en,3,tldr_en,rouge2_fmeasure,0.0510208899127158 +wiki_lingua_en,3,median,rouge2_fmeasure,0.0510208899127158 +wiki_lingua_en,4,tldr_en,rouge2_fmeasure,0.015994832602106063 +wiki_lingua_en,4,median,rouge2_fmeasure,0.015994832602106063 +wiki_lingua_en,5,tldr_en,rouge2_fmeasure,0.002725147355524338 +wiki_lingua_en,5,median,rouge2_fmeasure,0.002725147355524338 +wiki_lingua_en,5,average,multiple,0.030051116366027298 diff --git a/evaluation/generation/merged.json b/evaluation/generation/merged.json new file mode 100644 index 0000000000000000000000000000000000000000..bb6c1c93e6d38b3a3a53ac1ea794a4e3eea0729e --- /dev/null +++ b/evaluation/generation/merged.json @@ -0,0 +1 @@ +{"GEM/web_nlg_en": {"0": {"PALM_prompt": {"bleu": 0.3552258551749086, "bleu_stderr": 0.02004694008107349, "rouge1_fmeasure": 0.09756985013732229, "rouge1_fmeasure_stderr": 0.0019859335153510982, "rouge1_precision": 0.07050133636806707, "rouge1_precision_stderr": 0.002333415676412846, "rouge1_recall": 0.28446739915287816, "rouge1_recall_stderr": 0.00508507867804917, "rouge2_fmeasure": 0.043506125605077994, "rouge2_fmeasure_stderr": 0.0012188241069113202, "rouge2_precision": 0.030757559038758858, "rouge2_precision_stderr": 0.0013508528253275045, "rouge2_recall": 0.13243423248660713, "rouge2_recall_stderr": 0.003309647236077234, "rougeL_fmeasure": 0.09211119558985698, "rougeL_fmeasure_stderr": 0.0018415131593686243, "rougeL_precision": 0.0666262101718174, "rougeL_precision_stderr": 0.002246557443572289, "rougeL_recall": 0.2713467656341814, "rougeL_recall_stderr": 0.004908738323537121, "rougeLsum_fmeasure": 0.0909840574356387, "rougeLsum_fmeasure_stderr": 0.0018721336661780886, "rougeLsum_precision": 0.06617315282201941, "rougeLsum_precision_stderr": 0.0022690978668772816, "rougeLsum_recall": 0.2645417006842324, "rougeLsum_recall_stderr": 0.004684946802992825}}, "1": {"PALM_prompt": {"bleu": 0.455214213531198, "bleu_stderr": 0.027035455129998113, "rouge1_fmeasure": 0.12895359157667396, "rouge1_fmeasure_stderr": 0.0030401563578052807, "rouge1_precision": 0.10632219422720153, "rouge1_precision_stderr": 0.0035658627377499956, "rouge1_recall": 0.2768550528667316, "rouge1_recall_stderr": 0.004832345662545228, "rouge2_fmeasure": 0.06090727845863285, "rouge2_fmeasure_stderr": 0.0019209569415793166, "rouge2_precision": 0.04940516839667132, "rouge2_precision_stderr": 0.0021584007358498585, "rouge2_recall": 0.13404371723129116, "rouge2_recall_stderr": 0.003324815594078369, "rougeL_fmeasure": 0.11713897308464187, "rougeL_fmeasure_stderr": 0.0026334790087808417, "rougeL_precision": 0.09546267515845965, "rougeL_precision_stderr": 0.003141923276845849, "rougeL_recall": 0.25843481586517253, "rougeL_recall_stderr": 0.004474300115661777, "rougeLsum_fmeasure": 0.11922109901669682, "rougeLsum_fmeasure_stderr": 0.002697373964352153, "rougeLsum_precision": 0.09758720980677024, "rougeLsum_precision_stderr": 0.0032165997299972056, "rougeLsum_recall": 0.2609559973772103, "rougeLsum_recall_stderr": 0.00448462250486365}}, "2": {"PALM_prompt": {"bleu": 0.43672089515970186, "bleu_stderr": 0.021621818007392418, "rouge1_fmeasure": 0.15107097447841042, "rouge1_fmeasure_stderr": 0.0035335849672525825, "rouge1_precision": 0.12427774623745924, "rouge1_precision_stderr": 0.00396632764513016, "rouge1_recall": 0.31153733817117696, "rouge1_recall_stderr": 0.004862182707219614, "rouge2_fmeasure": 0.073579744703458, "rouge2_fmeasure_stderr": 0.0022399508108041687, "rouge2_precision": 0.061122334555414014, "rouge2_precision_stderr": 0.0024752531890197427, "rouge2_recall": 0.1553537719896923, "rouge2_recall_stderr": 0.003521786839397489, "rougeL_fmeasure": 0.1343719484645983, "rougeL_fmeasure_stderr": 0.0029138401067494405, "rougeL_precision": 0.10862059194070799, "rougeL_precision_stderr": 0.0032794570812240747, "rougeL_recall": 0.2880040612685418, "rougeL_recall_stderr": 0.004452369074617207, "rougeLsum_fmeasure": 0.13751466322243355, "rougeLsum_fmeasure_stderr": 0.0030168231727520462, "rougeLsum_precision": 0.11186894190492254, "rougeLsum_precision_stderr": 0.0034133834955339783, "rougeLsum_recall": 0.2921099418130985, "rougeLsum_recall_stderr": 0.0044981987732144475}}, "3": {"PALM_prompt": {"bleu": 0.6005511019489902, "bleu_stderr": 0.03451530801039688, "rouge1_fmeasure": 0.16481776831623232, "rouge1_fmeasure_stderr": 0.0038682235238341116, "rouge1_precision": 0.13595589597527172, "rouge1_precision_stderr": 0.004222986885593746, "rouge1_recall": 0.33095418350217154, "rouge1_recall_stderr": 0.0051126113252587, "rouge2_fmeasure": 0.08281371490457289, "rouge2_fmeasure_stderr": 0.002614718922183248, "rouge2_precision": 0.06960048861029224, "rouge2_precision_stderr": 0.0027985379538191905, "rouge2_recall": 0.16775824178008983, "rouge2_recall_stderr": 0.0037608840409452783, "rougeL_fmeasure": 0.1462708825192011, "rougeL_fmeasure_stderr": 0.0032850506015328065, "rougeL_precision": 0.11879183737182318, "rougeL_precision_stderr": 0.003583977041097007, "rougeL_recall": 0.30406006315984274, "rougeL_recall_stderr": 0.004643180781154661, "rougeLsum_fmeasure": 0.14946371142957862, "rougeLsum_fmeasure_stderr": 0.0033711111276765737, "rougeLsum_precision": 0.12214153222625562, "rougeLsum_precision_stderr": 0.0037097941549426336, "rougeLsum_recall": 0.3076377720124529, "rougeLsum_recall_stderr": 0.0046729883356302375}}, "4": {"PALM_prompt": {"bleu": 0.6245188789842254, "bleu_stderr": 0.043490522758658445, "rouge1_fmeasure": 0.17222603573029266, "rouge1_fmeasure_stderr": 0.003921731867593103, "rouge1_precision": 0.14786580825593648, "rouge1_precision_stderr": 0.004627492119408001, "rouge1_recall": 0.3383750446597362, "rouge1_recall_stderr": 0.004749410285992037, "rouge2_fmeasure": 0.08713860881109199, "rouge2_fmeasure_stderr": 0.002642546281127278, "rouge2_precision": 0.07698153502052321, "rouge2_precision_stderr": 0.0031209348406096736, "rouge2_recall": 0.1718812686921174, "rouge2_recall_stderr": 0.003529827492808453, "rougeL_fmeasure": 0.15325650518011516, "rougeL_fmeasure_stderr": 0.0033232600369196875, "rougeL_precision": 0.1294373882259293, "rougeL_precision_stderr": 0.003923291361672725, "rougeL_recall": 0.31167522711756507, "rougeL_recall_stderr": 0.00437544273005526, "rougeLsum_fmeasure": 0.15679817099191065, "rougeLsum_fmeasure_stderr": 0.003436174687018384, "rougeLsum_precision": 0.13331179138633883, "rougeLsum_precision_stderr": 0.004083238268648474, "rougeLsum_recall": 0.3154382483492274, "rougeLsum_recall_stderr": 0.0043923097345808845}}, "5": {"PALM_prompt": {"bleu": 0.690349543623733, "bleu_stderr": 0.037462509241486704, "rouge1_fmeasure": 0.18047036475829212, "rouge1_fmeasure_stderr": 0.004008854616724969, "rouge1_precision": 0.15622015160603553, "rouge1_precision_stderr": 0.004716488431001402, "rouge1_recall": 0.34792374373732987, "rouge1_recall_stderr": 0.004822507263881878, "rouge2_fmeasure": 0.0930691012178671, "rouge2_fmeasure_stderr": 0.0027170170506852424, "rouge2_precision": 0.08275177731282365, "rouge2_precision_stderr": 0.0031808930078121365, "rouge2_recall": 0.18065193154470569, "rouge2_recall_stderr": 0.003617404779594222, "rougeL_fmeasure": 0.15947807280240164, "rougeL_fmeasure_stderr": 0.00339577303947939, "rougeL_precision": 0.13603046372345778, "rougeL_precision_stderr": 0.004020369923896149, "rougeL_recall": 0.3181992629421937, "rougeL_recall_stderr": 0.00438340700122448, "rougeLsum_fmeasure": 0.16344169338350165, "rougeLsum_fmeasure_stderr": 0.0034931588485009957, "rougeLsum_precision": 0.14015533154496237, "rougeLsum_precision_stderr": 0.004159324286093359, "rougeLsum_recall": 0.3231216711299358, "rougeLsum_recall_stderr": 0.004412774889842664}}}, "GEM/wiki_lingua_en": {"0": {"tldr_en": {"bleu": 1.7401292150366783, "bleu_stderr": 0.05870950000759347, "rouge1_fmeasure": 0.09262226734827564, "rouge1_fmeasure_stderr": 0.002438006555562961, "rouge1_precision": 0.10101653488556242, "rouge1_precision_stderr": 0.0032530374257675938, "rouge1_recall": 0.12382359005658215, "rouge1_recall_stderr": 0.003311350263072308, "rouge2_fmeasure": 0.023458200677554667, "rouge2_fmeasure_stderr": 0.0009269048009616802, "rouge2_precision": 0.021119629322483437, "rouge2_precision_stderr": 0.000867268036727509, "rouge2_recall": 0.03184028832690896, "rouge2_recall_stderr": 0.001315196256520591, "rougeL_fmeasure": 0.0717645545672235, "rougeL_fmeasure_stderr": 0.0018475862298041712, "rougeL_precision": 0.08164131674574245, "rougeL_precision_stderr": 0.002890373817028126, "rougeL_recall": 0.09744037722102081, "rougeL_recall_stderr": 0.002611244990111791, "rougeLsum_fmeasure": 0.08593512322791458, "rougeLsum_fmeasure_stderr": 0.0022685460052984064, "rougeLsum_precision": 0.09506262524089595, "rougeLsum_precision_stderr": 0.003153876364088973, "rougeLsum_recall": 0.11499957069726778, "rougeLsum_recall_stderr": 0.0030864345858981576}}, "1": {"tldr_en": {"bleu": 1.8195228421640262, "bleu_stderr": 0.08352209916798846, "rouge1_fmeasure": 0.15283090429353868, "rouge1_fmeasure_stderr": 0.0019824413317849196, "rouge1_precision": 0.2169406307055522, "rouge1_precision_stderr": 0.003725498619269963, "rouge1_recall": 0.1602777922005014, "rouge1_recall_stderr": 0.0023194868352840897, "rouge2_fmeasure": 0.030516647625530857, "rouge2_fmeasure_stderr": 0.0011026203736681755, "rouge2_precision": 0.05250080669816919, "rouge2_precision_stderr": 0.002263016873913571, "rouge2_recall": 0.029595318390725434, "rouge2_recall_stderr": 0.0011280692140449152, "rougeL_fmeasure": 0.12076719641202475, "rougeL_fmeasure_stderr": 0.0015785014789520583, "rougeL_precision": 0.17465569803991965, "rougeL_precision_stderr": 0.003198873475886084, "rougeL_recall": 0.1267991344776356, "rougeL_recall_stderr": 0.001844558639815861, "rougeLsum_fmeasure": 0.1443601231383759, "rougeLsum_fmeasure_stderr": 0.0018505558858251069, "rougeLsum_precision": 0.20508454330750828, "rougeLsum_precision_stderr": 0.0035353326008678463, "rougeLsum_recall": 0.1517146954188497, "rougeLsum_recall_stderr": 0.0021834314823700506}}, "2": {"tldr_en": {"bleu": 2.8890982233924274, "bleu_stderr": 0.07489960916470248, "rouge1_fmeasure": 0.2060405071459818, "rouge1_fmeasure_stderr": 0.002251177818368773, "rouge1_precision": 0.3400358202564136, "rouge1_precision_stderr": 0.004368456331414668, "rouge1_recall": 0.19549655317783554, "rouge1_recall_stderr": 0.0026239543642537643, "rouge2_fmeasure": 0.05659098002273207, "rouge2_fmeasure_stderr": 0.0013825154700548739, "rouge2_precision": 0.10414705512855632, "rouge2_precision_stderr": 0.0029764211051013624, "rouge2_recall": 0.052252858151520365, "rouge2_recall_stderr": 0.0013882181124994304, "rougeL_fmeasure": 0.1632339625235188, "rougeL_fmeasure_stderr": 0.0017925107334137358, "rougeL_precision": 0.2760810172165432, "rougeL_precision_stderr": 0.003817392548869396, "rougeL_recall": 0.15431925374992486, "rougeL_recall_stderr": 0.0020853728076912085, "rougeLsum_fmeasure": 0.19413584042277324, "rougeLsum_fmeasure_stderr": 0.0021219960169286357, "rougeLsum_precision": 0.32272701619444116, "rougeLsum_precision_stderr": 0.004237054067018847, "rougeLsum_recall": 0.18387985811231844, "rougeLsum_recall_stderr": 0.002467066081787791}}, "3": {"tldr_en": {"bleu": 1.890195824238381, "bleu_stderr": 0.07805309283654865, "rouge1_fmeasure": 0.17768534554173257, "rouge1_fmeasure_stderr": 0.002497324854394021, "rouge1_precision": 0.3143827035331544, "rouge1_precision_stderr": 0.004814681871436694, "rouge1_recall": 0.16312653314207656, "rouge1_recall_stderr": 0.0028028425609925864, "rouge2_fmeasure": 0.0510208899127158, "rouge2_fmeasure_stderr": 0.0014011608421103333, "rouge2_precision": 0.10012288852389835, "rouge2_precision_stderr": 0.0030408513717817094, "rouge2_recall": 0.04640858531424939, "rouge2_recall_stderr": 0.0014571215580260206, "rougeL_fmeasure": 0.14320515112174437, "rougeL_fmeasure_stderr": 0.0020306931695434237, "rougeL_precision": 0.25883690093372125, "rougeL_precision_stderr": 0.004162927984690604, "rougeL_recall": 0.13073894232805758, "rougeL_recall_stderr": 0.0022683935578694356, "rougeLsum_fmeasure": 0.16704762132090992, "rougeLsum_fmeasure_stderr": 0.002342688337895595, "rougeLsum_precision": 0.2982192683665052, "rougeLsum_precision_stderr": 0.004629601809394366, "rougeLsum_recall": 0.1526514018679153, "rougeLsum_recall_stderr": 0.0026042434071650534}}, "4": {"tldr_en": {"bleu": 0.016910620160853877, "bleu_stderr": 0.00364577886299514, "rouge1_fmeasure": 0.05712785595257542, "rouge1_fmeasure_stderr": 0.002109365575892117, "rouge1_precision": 0.10233714832400315, "rouge1_precision_stderr": 0.00387170332308452, "rouge1_recall": 0.052719560358311074, "rouge1_recall_stderr": 0.0021360645688620044, "rouge2_fmeasure": 0.015994832602106063, "rouge2_fmeasure_stderr": 0.0009271960700065752, "rouge2_precision": 0.03236639334422414, "rouge2_precision_stderr": 0.0020360160794573286, "rouge2_recall": 0.014676186959067014, "rouge2_recall_stderr": 0.000991238095490706, "rougeL_fmeasure": 0.0463830391676825, "rougeL_fmeasure_stderr": 0.001707583877770572, "rougeL_precision": 0.08539656258425662, "rougeL_precision_stderr": 0.003320722236518427, "rougeL_recall": 0.04253430518648541, "rougeL_recall_stderr": 0.0017210052280975444, "rougeLsum_fmeasure": 0.053156186981754575, "rougeLsum_fmeasure_stderr": 0.0019608311676159664, "rougeLsum_precision": 0.09625340675143208, "rougeLsum_precision_stderr": 0.0036657803196257096, "rougeLsum_recall": 0.048881870192824625, "rougeLsum_recall_stderr": 0.0019848013068469554}}, "5": {"tldr_en": {"bleu": 1.4571231589477147e-17, "bleu_stderr": 2.554040800257738e-15, "rouge1_fmeasure": 0.0091861211614601, "rouge1_fmeasure_stderr": 0.0008969173014851819, "rouge1_precision": 0.018095685011338746, "rouge1_precision_stderr": 0.00185447805960437, "rouge1_recall": 0.008607244652028168, "rouge1_recall_stderr": 0.0009194028038185624, "rouge2_fmeasure": 0.002725147355524338, "rouge2_fmeasure_stderr": 0.0004048462437499985, "rouge2_precision": 0.006014959811595776, "rouge2_precision_stderr": 0.0009805624350940475, "rouge2_recall": 0.0023894753339408787, "rouge2_recall_stderr": 0.0003776589361725097, "rougeL_fmeasure": 0.007656339590582189, "rougeL_fmeasure_stderr": 0.0007546272499233359, "rougeL_precision": 0.015576109016400224, "rougeL_precision_stderr": 0.001653922803893889, "rougeL_recall": 0.007132007829695769, "rougeL_recall_stderr": 0.0007575676626861256, "rougeLsum_fmeasure": 0.008689332392498271, "rougeLsum_fmeasure_stderr": 0.000852751466212803, "rougeLsum_precision": 0.01720829814617215, "rougeLsum_precision_stderr": 0.001777168147572916, "rougeLsum_recall": 0.00818345077420238, "rougeLsum_recall_stderr": 0.000884640095931326}}}, "e2e_nlg_cleaned": {"0": {"generate_text_restaurant": {"bleu": 1.8176691711249537, "bleu_stderr": 0.05995542560680462, "rouge1_fmeasure": 0.12962478833198351, "rouge1_fmeasure_stderr": 0.0026733541462707695, "rouge1_precision": 0.11995524671016493, "rouge1_precision_stderr": 0.002674303424141786, "rouge1_recall": 0.15915193434349853, "rouge1_recall_stderr": 0.0032972190406054414, "rouge2_fmeasure": 0.03603975902395983, "rouge2_fmeasure_stderr": 0.0012403288414746835, "rouge2_precision": 0.03280986461326413, "rouge2_precision_stderr": 0.0011206239984645086, "rouge2_recall": 0.04550105277967982, "rouge2_recall_stderr": 0.0016902484384178876, "rougeL_fmeasure": 0.10376435983061806, "rougeL_fmeasure_stderr": 0.0019651761643805117, "rougeL_precision": 0.09452651957178927, "rougeL_precision_stderr": 0.0018963480115027633, "rougeL_recall": 0.12988017210774444, "rougeL_recall_stderr": 0.0025557466013964663, "rougeLsum_fmeasure": 0.11626650837633533, "rougeLsum_fmeasure_stderr": 0.002446339204031, "rougeLsum_precision": 0.10794550532453419, "rougeLsum_precision_stderr": 0.002439267929763401, "rougeLsum_recall": 0.14233639462065437, "rougeLsum_recall_stderr": 0.00302666138111876}}, "1": {"generate_text_restaurant": {"bleu": 11.849671487305415, "bleu_stderr": 0.1360222064360011, "rouge1_fmeasure": 0.45449662451133166, "rouge1_fmeasure_stderr": 0.002333690577738812, "rouge1_precision": 0.54188605893543, "rouge1_precision_stderr": 0.003246758820446708, "rouge1_recall": 0.4314024243954027, "rouge1_recall_stderr": 0.0029616850251012895, "rouge2_fmeasure": 0.2121926766511087, "rouge2_fmeasure_stderr": 0.0020146533728953066, "rouge2_precision": 0.2562791831846635, "rouge2_precision_stderr": 0.002639527749513042, "rouge2_recall": 0.20134452678189727, "rouge2_recall_stderr": 0.0021665892377752327, "rougeL_fmeasure": 0.3309578358109194, "rougeL_fmeasure_stderr": 0.002047514034389017, "rougeL_precision": 0.3968055602906763, "rougeL_precision_stderr": 0.002884957533719286, "rougeL_recall": 0.3135108993655708, "rougeL_recall_stderr": 0.002421217906260913, "rougeLsum_fmeasure": 0.3729160698976979, "rougeLsum_fmeasure_stderr": 0.002328237682996928, "rougeLsum_precision": 0.44543454886093686, "rougeLsum_precision_stderr": 0.003160000527797213, "rougeLsum_recall": 0.3537585371537755, "rougeLsum_recall_stderr": 0.0027469285035271475}}, "2": {"generate_text_restaurant": {"bleu": 14.500866321633556, "bleu_stderr": 0.21652939369858226, "rouge1_fmeasure": 0.4837551797964578, "rouge1_fmeasure_stderr": 0.0022147223749886054, "rouge1_precision": 0.5577858917634193, "rouge1_precision_stderr": 0.003247785025897439, "rouge1_recall": 0.4675413035402858, "rouge1_recall_stderr": 0.002820078928761888, "rouge2_fmeasure": 0.23945439459321566, "rouge2_fmeasure_stderr": 0.002051628654300672, "rouge2_precision": 0.27932023326938354, "rouge2_precision_stderr": 0.0026973630619244273, "rouge2_recall": 0.23167147979272804, "rouge2_recall_stderr": 0.0022172627136572156, "rougeL_fmeasure": 0.3598482761759142, "rougeL_fmeasure_stderr": 0.0020447863689909054, "rougeL_precision": 0.41572829682147167, "rougeL_precision_stderr": 0.0029006552995420294, "rougeL_recall": 0.3478030944709029, "rougeL_recall_stderr": 0.002413752959029049, "rougeLsum_fmeasure": 0.4056412361905643, "rougeLsum_fmeasure_stderr": 0.0022938418811786596, "rougeLsum_precision": 0.4676801531783358, "rougeLsum_precision_stderr": 0.0031627544428160766, "rougeLsum_recall": 0.39184282368555173, "rougeLsum_recall_stderr": 0.002694948881819574}}, "3": {"generate_text_restaurant": {"bleu": 14.99356088381018, "bleu_stderr": 0.1939852688470928, "rouge1_fmeasure": 0.4885274020508225, "rouge1_fmeasure_stderr": 0.002197446931041306, "rouge1_precision": 0.560904943171157, "rouge1_precision_stderr": 0.003222339717903183, "rouge1_recall": 0.46948710399711374, "rouge1_recall_stderr": 0.0027577686714870798, "rouge2_fmeasure": 0.24528072211886423, "rouge2_fmeasure_stderr": 0.0021164271571705827, "rouge2_precision": 0.2844738744368861, "rouge2_precision_stderr": 0.002710055422490507, "rouge2_recall": 0.23612605197891567, "rouge2_recall_stderr": 0.0022922087007235313, "rougeL_fmeasure": 0.36547104984192025, "rougeL_fmeasure_stderr": 0.0020631564845025475, "rougeL_precision": 0.42065354031616564, "rougeL_precision_stderr": 0.0029044985788341323, "rougeL_recall": 0.351254841158613, "rougeL_recall_stderr": 0.0024156544408651705, "rougeLsum_fmeasure": 0.4121617578480254, "rougeLsum_fmeasure_stderr": 0.002317480403210135, "rougeLsum_precision": 0.473294687864865, "rougeLsum_precision_stderr": 0.0031673965755000533, "rougeLsum_recall": 0.39610686350311647, "rougeLsum_recall_stderr": 0.0026942679498965384}}, "4": {"generate_text_restaurant": {"bleu": 15.136408152402511, "bleu_stderr": 0.15749919205112253, "rouge1_fmeasure": 0.49008120449442055, "rouge1_fmeasure_stderr": 0.0021552779856922706, "rouge1_precision": 0.5635851478232797, "rouge1_precision_stderr": 0.003229028521004473, "rouge1_recall": 0.46904362683842155, "rouge1_recall_stderr": 0.002666498752628597, "rouge2_fmeasure": 0.2472610815193413, "rouge2_fmeasure_stderr": 0.002097203486030851, "rouge2_precision": 0.2877028860236848, "rouge2_precision_stderr": 0.0027336630161171856, "rouge2_recall": 0.23648543820909804, "rouge2_recall_stderr": 0.0022192023733188957, "rougeL_fmeasure": 0.3672468672821708, "rougeL_fmeasure_stderr": 0.0020504167271609996, "rougeL_precision": 0.4232377532809423, "rougeL_precision_stderr": 0.002905210399187484, "rougeL_recall": 0.3512864132211812, "rougeL_recall_stderr": 0.002353821229944638, "rougeLsum_fmeasure": 0.4139945135555939, "rougeLsum_fmeasure_stderr": 0.0022731887846098205, "rougeLsum_precision": 0.4758982382137059, "rougeLsum_precision_stderr": 0.003149472337396323, "rougeLsum_recall": 0.3964206507125601, "rougeLsum_recall_stderr": 0.0026288828261706028}}, "5": {"generate_text_restaurant": {"bleu": 14.94032761568535, "bleu_stderr": 0.16102163935491987, "rouge1_fmeasure": 0.4891066764513762, "rouge1_fmeasure_stderr": 0.002148432709098188, "rouge1_precision": 0.5647971657629298, "rouge1_precision_stderr": 0.0032050177663230373, "rouge1_recall": 0.46466204351732154, "rouge1_recall_stderr": 0.002636551188261336, "rouge2_fmeasure": 0.2475971300723347, "rouge2_fmeasure_stderr": 0.0020740967240209687, "rouge2_precision": 0.28959208448012996, "rouge2_precision_stderr": 0.002723369175901688, "rouge2_recall": 0.2349545016224875, "rouge2_recall_stderr": 0.0021871784713310173, "rougeL_fmeasure": 0.3676953770310896, "rougeL_fmeasure_stderr": 0.002043411505256857, "rougeL_precision": 0.4250552095964905, "rougeL_precision_stderr": 0.002889497509660382, "rougeL_recall": 0.3494271521246103, "rougeL_recall_stderr": 0.0023269838649924057, "rougeLsum_fmeasure": 0.41342063708708626, "rougeLsum_fmeasure_stderr": 0.002240351112078022, "rougeLsum_precision": 0.47747050984587586, "rougeLsum_precision_stderr": 0.0031407117559106392, "rougeLsum_recall": 0.3928622107079271, "rougeLsum_recall_stderr": 0.0025636957992988966}}}, "gem_xsum": {"0": {"article_DOC_summary": {"bleu": 1.8631748315575967, "bleu_stderr": 0.06480985196854291, "rouge1_fmeasure": 0.19805083031043677, "rouge1_fmeasure_stderr": 0.0025145733920038504, "rouge1_precision": 0.14726644892868132, "rouge1_precision_stderr": 0.0024412028465323945, "rouge1_recall": 0.33236412764323403, "rouge1_recall_stderr": 0.004292159791327218, "rouge2_fmeasure": 0.045029580196435964, "rouge2_fmeasure_stderr": 0.0015159796252548507, "rouge2_precision": 0.03338148146365739, "rouge2_precision_stderr": 0.0014454654718671273, "rouge2_recall": 0.0775963205240964, "rouge2_recall_stderr": 0.002642772816149882, "rougeL_fmeasure": 0.150708876966239, "rougeL_fmeasure_stderr": 0.0019106881335396962, "rougeL_precision": 0.11274328349904972, "rougeL_precision_stderr": 0.0021403391288850487, "rougeL_recall": 0.253884590300403, "rougeL_recall_stderr": 0.0033701265161261294, "rougeLsum_fmeasure": 0.15690226758738812, "rougeLsum_fmeasure_stderr": 0.0021606792108516933, "rougeLsum_precision": 0.11711469488381401, "rougeLsum_precision_stderr": 0.0022541549353202995, "rougeLsum_recall": 0.26459055857012664, "rougeLsum_recall_stderr": 0.0037902287339387276}}, "1": {"article_DOC_summary": {"bleu": 2.002473681001451, "bleu_stderr": 0.10305039390700386, "rouge1_fmeasure": 0.21492593421689296, "rouge1_fmeasure_stderr": 0.0031040934015378273, "rouge1_precision": 0.201520198934412, "rouge1_precision_stderr": 0.0036780464289758986, "rouge1_recall": 0.28041072507766257, "rouge1_recall_stderr": 0.004470154542708083, "rouge2_fmeasure": 0.045086359683801695, "rouge2_fmeasure_stderr": 0.0019034075416325229, "rouge2_precision": 0.04262064763605291, "rouge2_precision_stderr": 0.0020875394875545616, "rouge2_recall": 0.059979799287456115, "rouge2_recall_stderr": 0.002459545390487186, "rougeL_fmeasure": 0.15921676436636747, "rougeL_fmeasure_stderr": 0.002420430590555424, "rougeL_precision": 0.15057288573196015, "rougeL_precision_stderr": 0.002929976392840731, "rougeL_recall": 0.20592545696933923, "rougeL_recall_stderr": 0.0033110515398117922, "rougeLsum_fmeasure": 0.16736923260398065, "rougeLsum_fmeasure_stderr": 0.0025093332042761462, "rougeLsum_precision": 0.15642146751721803, "rougeLsum_precision_stderr": 0.0029104557088062938, "rougeLsum_recall": 0.2199677896065397, "rougeLsum_recall_stderr": 0.0037276875782133504}}, "2": {"article_DOC_summary": {"bleu": 1.9767413966669858, "bleu_stderr": 0.12553384851261437, "rouge1_fmeasure": 0.21928064355323418, "rouge1_fmeasure_stderr": 0.0029906506411637493, "rouge1_precision": 0.2088093664550383, "rouge1_precision_stderr": 0.003591375397921926, "rouge1_recall": 0.28037833021310865, "rouge1_recall_stderr": 0.00418073064186992, "rouge2_fmeasure": 0.04498559646410484, "rouge2_fmeasure_stderr": 0.001797572147267301, "rouge2_precision": 0.04282030935093115, "rouge2_precision_stderr": 0.0018963252751686556, "rouge2_recall": 0.05836728040338388, "rouge2_recall_stderr": 0.0022732577336655444, "rougeL_fmeasure": 0.16195880232863943, "rougeL_fmeasure_stderr": 0.002341838899049842, "rougeL_precision": 0.15422100446453424, "rougeL_precision_stderr": 0.0027848170802648774, "rougeL_recall": 0.20777843317638175, "rougeL_recall_stderr": 0.0032512385248102136, "rougeLsum_fmeasure": 0.16811888843078998, "rougeLsum_fmeasure_stderr": 0.0024215692583980006, "rougeLsum_precision": 0.15873674200935498, "rougeLsum_precision_stderr": 0.0027827884387405604, "rougeLsum_recall": 0.21825435569219825, "rougeLsum_recall_stderr": 0.0035822959483663586}}, "3": {"article_DOC_summary": {"bleu": 2.0020914675362764, "bleu_stderr": 0.14545306393065818, "rouge1_fmeasure": 0.21363414740028736, "rouge1_fmeasure_stderr": 0.0033547861779570426, "rouge1_precision": 0.2040643001997412, "rouge1_precision_stderr": 0.003978248036078804, "rouge1_recall": 0.27307347161473916, "rouge1_recall_stderr": 0.004455443229628111, "rouge2_fmeasure": 0.044315357975490785, "rouge2_fmeasure_stderr": 0.0018794354037841253, "rouge2_precision": 0.0427132375189158, "rouge2_precision_stderr": 0.0020323931003614964, "rouge2_recall": 0.056734403722373934, "rouge2_recall_stderr": 0.0023427635867331836, "rougeL_fmeasure": 0.15848715345045697, "rougeL_fmeasure_stderr": 0.0026312535881803656, "rougeL_precision": 0.15146281632387826, "rougeL_precision_stderr": 0.0031018365348921524, "rougeL_recall": 0.20314923616617458, "rougeL_recall_stderr": 0.003479004056873755, "rougeLsum_fmeasure": 0.1642507173243999, "rougeLsum_fmeasure_stderr": 0.0026893440158164876, "rougeLsum_precision": 0.15567331452586086, "rougeLsum_precision_stderr": 0.0030921047823091897, "rougeLsum_recall": 0.21288484516878498, "rougeLsum_recall_stderr": 0.0037476662681603583}}, "4": {"article_DOC_summary": {"bleu": 0.3313737532545398, "bleu_stderr": 0.057620671081541584, "rouge1_fmeasure": 0.054580713604579945, "rouge1_fmeasure_stderr": 0.0032077855926012644, "rouge1_precision": 0.05845678560810508, "rouge1_precision_stderr": 0.003653372410391946, "rouge1_recall": 0.06546523451488562, "rouge1_recall_stderr": 0.00397320947448918, "rouge2_fmeasure": 0.010993328089942682, "rouge2_fmeasure_stderr": 0.0011929721928833114, "rouge2_precision": 0.011526059837135257, "rouge2_precision_stderr": 0.0014149014965583362, "rouge2_recall": 0.013070566192947936, "rouge2_recall_stderr": 0.0013385142420944767, "rougeL_fmeasure": 0.04045511548154733, "rougeL_fmeasure_stderr": 0.002398056707020803, "rougeL_precision": 0.044187543013943426, "rougeL_precision_stderr": 0.0028507649598664116, "rougeL_recall": 0.04837933471428071, "rougeL_recall_stderr": 0.0029701161810768185, "rougeLsum_fmeasure": 0.041976934818540805, "rougeLsum_fmeasure_stderr": 0.0024886979052922598, "rougeLsum_precision": 0.045470269077929414, "rougeLsum_precision_stderr": 0.002904632966614191, "rougeLsum_recall": 0.050635363980909044, "rougeLsum_recall_stderr": 0.0031412698278665725}}, "5": {"article_DOC_summary": {"bleu": 1.8341021693095957e-44, "bleu_stderr": 2.1637012594857334e-38, "rouge1_fmeasure": 0.001906885101987677, "rouge1_fmeasure_stderr": 0.0005780948322075463, "rouge1_precision": 0.0022940371923293613, "rouge1_precision_stderr": 0.0007143995060460097, "rouge1_recall": 0.0017348914893632757, "rouge1_recall_stderr": 0.0005253123299517621, "rouge2_fmeasure": 0.00010679337289617054, "rouge2_fmeasure_stderr": 7.591909328912052e-05, "rouge2_precision": 0.00011486155354079882, "rouge2_precision_stderr": 8.136503714231734e-05, "rouge2_recall": 0.00010005717552887364, "rouge2_recall_stderr": 7.143995839898157e-05, "rougeL_fmeasure": 0.001362800409731286, "rougeL_fmeasure_stderr": 0.0003965668916012721, "rougeL_precision": 0.0016335291518965692, "rougeL_precision_stderr": 0.0004934641653958932, "rougeL_recall": 0.001252059737745911, "rougeL_recall_stderr": 0.0003636768915155132, "rougeLsum_fmeasure": 0.001413249405796264, "rougeLsum_fmeasure_stderr": 0.0004059395809283557, "rougeLsum_precision": 0.0016839781479615478, "rougeLsum_precision_stderr": 0.0005010040710225287, "rougeLsum_recall": 0.0013025087338108892, "rougeLsum_recall_stderr": 0.0003738878391567918}}}} \ No newline at end of file diff --git a/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_0.json b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_0.json new file mode 100644 index 0000000000000000000000000000000000000000..c8a5d5461af15d65f788a56f4e9b5d7b94fca796 --- /dev/null +++ b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.3552258551749086, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.02004694008107349 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.07050133636806707, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.002333415676412846 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.28446739915287816, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.00508507867804917 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.09756985013732229, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0019859335153510982 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.030757559038758858, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0013508528253275045 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.13243423248660713, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.003309647236077234 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.043506125605077994, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0012188241069113202 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.0666262101718174, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.002246557443572289 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.2713467656341814, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004908738323537121 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.09211119558985698, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0018415131593686243 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.06617315282201941, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0022690978668772816 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.2645417006842324, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004684946802992825 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.0909840574356387, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0018721336661780886 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_1.json b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_1.json new file mode 100644 index 0000000000000000000000000000000000000000..5cfc811391875cf4d39fb043a5a9e6c19dc34615 --- /dev/null +++ b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.455214213531198, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.027035455129998113 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.10632219422720153, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0035658627377499956 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.2768550528667316, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.004832345662545228 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.12895359157667396, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0030401563578052807 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.04940516839667132, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0021584007358498585 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.13404371723129116, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.003324815594078369 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.06090727845863285, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0019209569415793166 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.09546267515845965, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.003141923276845849 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.25843481586517253, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004474300115661777 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.11713897308464187, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0026334790087808417 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.09758720980677024, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0032165997299972056 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.2609559973772103, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.00448462250486365 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.11922109901669682, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.002697373964352153 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_2.json b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_2.json new file mode 100644 index 0000000000000000000000000000000000000000..83b4a414ef2bd44326c82c8bcac5e9fb22bc6169 --- /dev/null +++ b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.43672089515970186, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.021621818007392418 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.12427774623745924, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.00396632764513016 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.31153733817117696, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.004862182707219614 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.15107097447841042, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0035335849672525825 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.061122334555414014, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0024752531890197427 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.1553537719896923, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.003521786839397489 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.073579744703458, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0022399508108041687 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.10862059194070799, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0032794570812240747 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.2880040612685418, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004452369074617207 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.1343719484645983, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0029138401067494405 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.11186894190492254, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0034133834955339783 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.2921099418130985, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0044981987732144475 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.13751466322243355, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0030168231727520462 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_3.json b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_3.json new file mode 100644 index 0000000000000000000000000000000000000000..4facb6d536f3724fd803950fefd36f5fc4e78902 --- /dev/null +++ b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.6005511019489902, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.03451530801039688 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.13595589597527172, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.004222986885593746 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.33095418350217154, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0051126113252587 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.16481776831623232, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0038682235238341116 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.06960048861029224, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0027985379538191905 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.16775824178008983, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0037608840409452783 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.08281371490457289, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.002614718922183248 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.11879183737182318, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.003583977041097007 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.30406006315984274, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004643180781154661 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.1462708825192011, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0032850506015328065 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.12214153222625562, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0037097941549426336 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.3076377720124529, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0046729883356302375 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.14946371142957862, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0033711111276765737 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_4.json b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_4.json new file mode 100644 index 0000000000000000000000000000000000000000..5044cf5a554ed909955d1f2ece5396fff687b305 --- /dev/null +++ b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.6245188789842254, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.043490522758658445 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.14786580825593648, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.004627492119408001 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.3383750446597362, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.004749410285992037 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.17222603573029266, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.003921731867593103 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.07698153502052321, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0031209348406096736 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.1718812686921174, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.003529827492808453 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.08713860881109199, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.002642546281127278 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.1294373882259293, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.003923291361672725 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.31167522711756507, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.00437544273005526 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.15325650518011516, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0033232600369196875 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.13331179138633883, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.004083238268648474 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.3154382483492274, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0043923097345808845 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.15679817099191065, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.003436174687018384 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_5.json b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_5.json new file mode 100644 index 0000000000000000000000000000000000000000..f24ff3bbc07765bddb1eda7019c31ebe93e86b6d --- /dev/null +++ b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-web_nlg_en_PALM_prompt_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.690349543623733, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.037462509241486704 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.15622015160603553, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.004716488431001402 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.34792374373732987, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.004822507263881878 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.18047036475829212, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.004008854616724969 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.08275177731282365, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0031808930078121365 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.18065193154470569, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.003617404779594222 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.0930691012178671, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0027170170506852424 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.13603046372345778, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.004020369923896149 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.3181992629421937, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.00438340700122448 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.15947807280240164, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.00339577303947939 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.14015533154496237, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.004159324286093359 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.3231216711299358, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004412774889842664 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.16344169338350165, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0034931588485009957 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_0.json b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_0.json new file mode 100644 index 0000000000000000000000000000000000000000..397e87b0abd2d93986fb1cc9ca45bfec1760d076 --- /dev/null +++ b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.10101653488556242, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0032530374257675938 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.12382359005658215, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.003311350263072308 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.09262226734827564, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.002438006555562961 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.021119629322483437, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.000867268036727509 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.03184028832690896, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.001315196256520591 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.023458200677554667, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0009269048009616802 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.08164131674574245, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.002890373817028126 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.09744037722102081, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.002611244990111791 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.0717645545672235, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0018475862298041712 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.09506262524089595, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.003153876364088973 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.11499957069726778, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0030864345858981576 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.08593512322791458, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0022685460052984064 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 1.7401292150366783, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.05870950000759347 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_1.json b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_1.json new file mode 100644 index 0000000000000000000000000000000000000000..bea551d614bc0014615fb13832ef26142d79fed9 --- /dev/null +++ b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.2169406307055522, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.003725498619269963 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.1602777922005014, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0023194868352840897 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.15283090429353868, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0019824413317849196 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.05250080669816919, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.002263016873913571 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.029595318390725434, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0011280692140449152 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.030516647625530857, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0011026203736681755 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.17465569803991965, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.003198873475886084 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.1267991344776356, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.001844558639815861 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.12076719641202475, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0015785014789520583 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.20508454330750828, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0035353326008678463 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.1517146954188497, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0021834314823700506 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.1443601231383759, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0018505558858251069 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 1.8195228421640262, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.08352209916798846 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_2.json b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_2.json new file mode 100644 index 0000000000000000000000000000000000000000..a43c31ab0285c35ddb1df42f60a3349ff71e2386 --- /dev/null +++ b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.3400358202564136, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.004368456331414668 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.19549655317783554, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0026239543642537643 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.2060405071459818, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.002251177818368773 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.10414705512855632, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0029764211051013624 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.052252858151520365, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0013882181124994304 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.05659098002273207, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0013825154700548739 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.2760810172165432, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.003817392548869396 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.15431925374992486, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0020853728076912085 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.1632339625235188, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0017925107334137358 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.32272701619444116, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.004237054067018847 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.18387985811231844, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.002467066081787791 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.19413584042277324, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0021219960169286357 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 2.8890982233924274, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.07489960916470248 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_3.json b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_3.json new file mode 100644 index 0000000000000000000000000000000000000000..ed51d585f03837183a02b2fcb16043906fa41038 --- /dev/null +++ b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.3143827035331544, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.004814681871436694 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.16312653314207656, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0028028425609925864 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.17768534554173257, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.002497324854394021 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.10012288852389835, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0030408513717817094 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.04640858531424939, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0014571215580260206 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.0510208899127158, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0014011608421103333 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.25883690093372125, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.004162927984690604 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.13073894232805758, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0022683935578694356 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.14320515112174437, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0020306931695434237 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.2982192683665052, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.004629601809394366 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.1526514018679153, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0026042434071650534 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.16704762132090992, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.002342688337895595 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 1.890195824238381, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.07805309283654865 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_4.json b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_4.json new file mode 100644 index 0000000000000000000000000000000000000000..c6a83c75ca150e99676839135f7efaa8d7ab361f --- /dev/null +++ b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.10233714832400315, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.00387170332308452 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.052719560358311074, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0021360645688620044 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.05712785595257542, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.002109365575892117 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.03236639334422414, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0020360160794573286 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.014676186959067014, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.000991238095490706 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.015994832602106063, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0009271960700065752 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.08539656258425662, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.003320722236518427 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.04253430518648541, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0017210052280975444 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.0463830391676825, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.001707583877770572 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.09625340675143208, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0036657803196257096 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.048881870192824625, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0019848013068469554 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.053156186981754575, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0019608311676159664 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 0.016910620160853877, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.00364577886299514 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_5.json b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_5.json new file mode 100644 index 0000000000000000000000000000000000000000..5d84090053dff72ccf338c29cec3e6ae935fcf3c --- /dev/null +++ b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_GEM-wiki_lingua_en_tldr_en_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.018095685011338746, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.00185447805960437 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.008607244652028168, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0009194028038185624 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.0091861211614601, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0008969173014851819 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.006014959811595776, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0009805624350940475 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.0023894753339408787, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0003776589361725097 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.002725147355524338, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0004048462437499985 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.015576109016400224, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.001653922803893889 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.007132007829695769, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0007575676626861256 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.007656339590582189, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0007546272499233359 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.01720829814617215, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.001777168147572916 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.00818345077420238, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.000884640095931326 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.008689332392498271, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.000852751466212803 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 1.4571231589477147e-17, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 2.554040800257738e-15 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_0.json b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_0.json new file mode 100644 index 0000000000000000000000000000000000000000..0f1e3c7ff046213c3e68c6571e3994a314f843fa --- /dev/null +++ b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 1.8176691711249537, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.05995542560680462 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.11995524671016493, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.002674303424141786 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.15915193434349853, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0032972190406054414 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.12962478833198351, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0026733541462707695 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.03280986461326413, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0011206239984645086 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.04550105277967982, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0016902484384178876 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.03603975902395983, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0012403288414746835 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.09452651957178927, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0018963480115027633 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.12988017210774444, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0025557466013964663 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.10376435983061806, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0019651761643805117 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.10794550532453419, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.002439267929763401 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.14233639462065437, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.00302666138111876 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.11626650837633533, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.002446339204031 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_1.json b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_1.json new file mode 100644 index 0000000000000000000000000000000000000000..2461c4bd5fbcba124a5fd89cb61a925f887a1ebf --- /dev/null +++ b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 11.849671487305415, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.1360222064360011 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.54188605893543, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.003246758820446708 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.4314024243954027, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0029616850251012895 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.45449662451133166, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.002333690577738812 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.2562791831846635, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.002639527749513042 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.20134452678189727, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0021665892377752327 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.2121926766511087, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0020146533728953066 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.3968055602906763, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.002884957533719286 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.3135108993655708, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.002421217906260913 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.3309578358109194, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.002047514034389017 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.44543454886093686, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.003160000527797213 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.3537585371537755, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0027469285035271475 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.3729160698976979, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.002328237682996928 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_2.json b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_2.json new file mode 100644 index 0000000000000000000000000000000000000000..24e359d799233e68cf7bf18914a7ecb2ee8764bb --- /dev/null +++ b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 14.500866321633556, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.21652939369858226 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.5577858917634193, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.003247785025897439 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.4675413035402858, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.002820078928761888 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.4837551797964578, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0022147223749886054 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.27932023326938354, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0026973630619244273 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.23167147979272804, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0022172627136572156 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.23945439459321566, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.002051628654300672 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.41572829682147167, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0029006552995420294 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.3478030944709029, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.002413752959029049 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.3598482761759142, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0020447863689909054 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.4676801531783358, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0031627544428160766 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.39184282368555173, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.002694948881819574 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.4056412361905643, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0022938418811786596 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_3.json b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_3.json new file mode 100644 index 0000000000000000000000000000000000000000..37dc8ad6c75c7777fe402d147a9996d8f50db9ff --- /dev/null +++ b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 14.99356088381018, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.1939852688470928 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.560904943171157, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.003222339717903183 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.46948710399711374, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0027577686714870798 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.4885274020508225, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.002197446931041306 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.2844738744368861, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.002710055422490507 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.23612605197891567, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0022922087007235313 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.24528072211886423, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0021164271571705827 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.42065354031616564, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0029044985788341323 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.351254841158613, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0024156544408651705 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.36547104984192025, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0020631564845025475 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.473294687864865, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0031673965755000533 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.39610686350311647, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0026942679498965384 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.4121617578480254, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.002317480403210135 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_4.json b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_4.json new file mode 100644 index 0000000000000000000000000000000000000000..5da794154ebbba36c37e23498cb33b4d2ba78d38 --- /dev/null +++ b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 15.136408152402511, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.15749919205112253 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.5635851478232797, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.003229028521004473 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.46904362683842155, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.002666498752628597 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.49008120449442055, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0021552779856922706 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.2877028860236848, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0027336630161171856 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.23648543820909804, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0022192023733188957 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.2472610815193413, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.002097203486030851 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.4232377532809423, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.002905210399187484 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.3512864132211812, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.002353821229944638 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.3672468672821708, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0020504167271609996 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.4758982382137059, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.003149472337396323 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.3964206507125601, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0026288828261706028 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.4139945135555939, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0022731887846098205 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_5.json b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_5.json new file mode 100644 index 0000000000000000000000000000000000000000..b111b61cac4b9e44ba94be4b970981e4556d42f8 --- /dev/null +++ b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_e2e_nlg_cleaned_generate_text_restaurant_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 14.94032761568535, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.16102163935491987 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.5647971657629298, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.0032050177663230373 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.46466204351732154, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.002636551188261336 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.4891066764513762, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.002148432709098188 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.28959208448012996, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.002723369175901688 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.2349545016224875, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0021871784713310173 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.2475971300723347, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0020740967240209687 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.4250552095964905, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.002889497509660382 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.3494271521246103, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0023269838649924057 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.3676953770310896, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.002043411505256857 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.47747050984587586, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0031407117559106392 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.3928622107079271, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0025636957992988966 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.41342063708708626, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.002240351112078022 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_0.json b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_0.json new file mode 100644 index 0000000000000000000000000000000000000000..2b1696e81a57d1de1a6c5d31c6adcb514843a29e --- /dev/null +++ b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.14726644892868132, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0024412028465323945 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.33236412764323403, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.004292159791327218 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.19805083031043677, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0025145733920038504 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.03338148146365739, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0014454654718671273 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.0775963205240964, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.002642772816149882 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.045029580196435964, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0015159796252548507 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.11274328349904972, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0021403391288850487 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.253884590300403, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0033701265161261294 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.150708876966239, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0019106881335396962 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.11711469488381401, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0022541549353202995 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.26459055857012664, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.0037902287339387276 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.15690226758738812, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0021606792108516933 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 1.8631748315575967, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.06480985196854291 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_1.json b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_1.json new file mode 100644 index 0000000000000000000000000000000000000000..1570bc2323d9aed38601b94f6e3670ed0ce1430d --- /dev/null +++ b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.201520198934412, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0036780464289758986 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.28041072507766257, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.004470154542708083 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.21492593421689296, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0031040934015378273 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.04262064763605291, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0020875394875545616 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.059979799287456115, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.002459545390487186 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.045086359683801695, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0019034075416325229 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.15057288573196015, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.002929976392840731 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.20592545696933923, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0033110515398117922 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.15921676436636747, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.002420430590555424 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.15642146751721803, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0029104557088062938 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.2199677896065397, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.0037276875782133504 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.16736923260398065, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0025093332042761462 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 2.002473681001451, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.10305039390700386 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_2.json b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_2.json new file mode 100644 index 0000000000000000000000000000000000000000..efcc7c93d7ec21b42155f5c3f0b6176a24113223 --- /dev/null +++ b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.2088093664550383, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.003591375397921926 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.28037833021310865, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.00418073064186992 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.21928064355323418, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0029906506411637493 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.04282030935093115, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0018963252751686556 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.05836728040338388, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0022732577336655444 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.04498559646410484, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.001797572147267301 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.15422100446453424, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0027848170802648774 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.20777843317638175, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0032512385248102136 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.16195880232863943, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.002341838899049842 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.15873674200935498, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0027827884387405604 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.21825435569219825, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.0035822959483663586 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.16811888843078998, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0024215692583980006 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 1.9767413966669858, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.12553384851261437 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_3.json b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_3.json new file mode 100644 index 0000000000000000000000000000000000000000..37f7e1c45273dd367e92a8e88b6343dc8b1f1c5d --- /dev/null +++ b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.2040643001997412, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.003978248036078804 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.27307347161473916, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.004455443229628111 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.21363414740028736, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0033547861779570426 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.0427132375189158, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0020323931003614964 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.056734403722373934, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0023427635867331836 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.044315357975490785, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0018794354037841253 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.15146281632387826, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0031018365348921524 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.20314923616617458, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.003479004056873755 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.15848715345045697, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0026312535881803656 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.15567331452586086, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0030921047823091897 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.21288484516878498, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.0037476662681603583 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.1642507173243999, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0026893440158164876 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 2.0020914675362764, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.14545306393065818 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_4.json b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_4.json new file mode 100644 index 0000000000000000000000000000000000000000..628610e3bdda169bb0732e77905d99947ead4c73 --- /dev/null +++ b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.05845678560810508, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.003653372410391946 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.06546523451488562, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.00397320947448918 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.054580713604579945, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0032077855926012644 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.011526059837135257, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0014149014965583362 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.013070566192947936, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0013385142420944767 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.010993328089942682, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0011929721928833114 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.044187543013943426, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0028507649598664116 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.04837933471428071, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0029701161810768185 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.04045511548154733, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.002398056707020803 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.045470269077929414, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.002904632966614191 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.050635363980909044, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.0031412698278665725 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.041976934818540805, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0024886979052922598 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.3313737532545398, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.057620671081541584 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_5.json b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_5.json new file mode 100644 index 0000000000000000000000000000000000000000..0a1cafaa3ac107505f1abbbcc6d17fca4ce960bd --- /dev/null +++ b/evaluation/generation/slim.lm1-2b8-55b-oscardedup25_gem_xsum_article_DOC_summary_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.0022940371923293613, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0007143995060460097 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.0017348914893632757, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.0005253123299517621 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.001906885101987677, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0005780948322075463 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.00011486155354079882, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 8.136503714231734e-05 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.00010005717552887364, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 7.143995839898157e-05 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.00010679337289617054, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 7.591909328912052e-05 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.0016335291518965692, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0004934641653958932 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.001252059737745911, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0003636768915155132 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.001362800409731286, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0003965668916012721 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.0016839781479615478, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0005010040710225287 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.0013025087338108892, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.0003738878391567918 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.001413249405796264, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0004059395809283557 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 1.8341021693095957e-44, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 2.1637012594857334e-38 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscardedup25/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/evaluation/rankeval/lm1-2b8-55b-oscardedup25_0.csv b/evaluation/rankeval/lm1-2b8-55b-oscardedup25_0.csv new file mode 100644 index 0000000000000000000000000000000000000000..d1be14e1b7e39a7374129ee3a62742ba665bd03d --- /dev/null +++ b/evaluation/rankeval/lm1-2b8-55b-oscardedup25_0.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.307,0.014593284892852627,0 +anli_r2,acc,0.32,0.014758652303574886,0 +anli_r3,acc,0.3383333333333333,0.01366414400661827,0 +arc_challenge,acc,0.25170648464163825,0.012682496334042963,0 +arc_challenge,acc_norm,0.2687713310580205,0.012955065963710686,0 +arc_easy,acc,0.5711279461279462,0.010155440652900154,0 +arc_easy,acc_norm,0.5138888888888888,0.01025582450719034,0 +boolq,acc,0.5226299694189602,0.008736093428015823,1 +cb,acc,0.48214285714285715,0.0673769750864465,1 +cb,f1,0.3,,1 +copa,acc,0.71,0.04560480215720683,0 +hellaswag,acc,0.4257120095598486,0.004934399802594904,0 +hellaswag,acc_norm,0.538338976299542,0.004975091055697192,0 +piqa,acc,0.7279651795429815,0.01038276378624738,0 +piqa,acc_norm,0.7334058759521219,0.010316749863541365,0 +rte,acc,0.5776173285198556,0.029731622646495887,0 +sciq,acc,0.812,0.012361586015103754,0 +sciq,acc_norm,0.735,0.01396316475480995,0 +storycloze_2016,acc,0.6889363976483164,0.010705164869803168,0 +winogrande,acc,0.5516969218626677,0.013977171307126338,0 diff --git a/evaluation/rankeval/lm1-2b8-55b-oscardedup25_0.json b/evaluation/rankeval/lm1-2b8-55b-oscardedup25_0.json new file mode 100644 index 0000000000000000000000000000000000000000..3b9b04a61f9275eb75c19656cdd958bba3566fbd --- /dev/null +++ b/evaluation/rankeval/lm1-2b8-55b-oscardedup25_0.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.307, + "acc_stderr": 0.014593284892852627 + }, + "anli_r2": { + "acc": 0.32, + "acc_stderr": 0.014758652303574886 + }, + "anli_r3": { + "acc": 0.3383333333333333, + "acc_stderr": 0.01366414400661827 + }, + "cb": { + "acc": 0.48214285714285715, + "acc_stderr": 0.0673769750864465, + "f1": 0.3 + }, + "copa": { + "acc": 0.71, + "acc_stderr": 0.04560480215720683 + }, + "hellaswag": { + "acc": 0.4257120095598486, + "acc_stderr": 0.004934399802594904, + "acc_norm": 0.538338976299542, + "acc_norm_stderr": 0.004975091055697192 + }, + "rte": { + "acc": 0.5776173285198556, + "acc_stderr": 0.029731622646495887 + }, + "winogrande": { + "acc": 0.5516969218626677, + "acc_stderr": 0.013977171307126338 + }, + "storycloze_2016": { + "acc": 0.6889363976483164, + "acc_stderr": 0.010705164869803168 + }, + "boolq": { + "acc": 0.5226299694189602, + "acc_stderr": 0.008736093428015823 + }, + "arc_easy": { + "acc": 0.5711279461279462, + "acc_stderr": 0.010155440652900154, + "acc_norm": 0.5138888888888888, + "acc_norm_stderr": 0.01025582450719034 + }, + "arc_challenge": { + "acc": 0.25170648464163825, + "acc_stderr": 0.012682496334042963, + "acc_norm": 0.2687713310580205, + "acc_norm_stderr": 0.012955065963710686 + }, + "sciq": { + "acc": 0.812, + "acc_stderr": 0.012361586015103754, + "acc_norm": 0.735, + "acc_norm_stderr": 0.01396316475480995 + }, + "piqa": { + "acc": 0.7279651795429815, + "acc_stderr": 0.01038276378624738, + "acc_norm": 0.7334058759521219, + "acc_norm_stderr": 0.010316749863541365 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/evaluation/rankeval/lm1-2b8-55b-oscardedup25_1.csv b/evaluation/rankeval/lm1-2b8-55b-oscardedup25_1.csv new file mode 100644 index 0000000000000000000000000000000000000000..110ddb3768a424b0ae0a13784731fd8640ee76b5 --- /dev/null +++ b/evaluation/rankeval/lm1-2b8-55b-oscardedup25_1.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.327,0.01484221315341124,0 +anli_r2,acc,0.356,0.015149042659306625,0 +anli_r3,acc,0.3333333333333333,0.013613950010225601,0 +arc_challenge,acc,0.26706484641638223,0.01292893319649635,0 +arc_challenge,acc_norm,0.2883959044368601,0.01323839442242817,0 +arc_easy,acc,0.5812289562289562,0.010123487160167808,0 +arc_easy,acc_norm,0.5664983164983165,0.010168640625454107,0 +boolq,acc,0.537308868501529,0.008720675606388452,1 +cb,acc,0.5178571428571429,0.06737697508644648,1 +cb,f1,0.33915343915343915,,1 +copa,acc,0.73,0.044619604333847394,0 +hellaswag,acc,0.4208325034853615,0.004926837572202164,0 +hellaswag,acc_norm,0.5406293567018522,0.004973280417705514,0 +piqa,acc,0.720892274211099,0.01046565794849823,0 +piqa,acc_norm,0.7377584330794341,0.01026250256517244,0 +rte,acc,0.51985559566787,0.03007272316731717,0 +sciq,acc,0.866,0.010777762298369686,0 +sciq,acc_norm,0.855,0.011139977517890145,0 +storycloze_2016,acc,0.6862640299305185,0.010730179119317623,0 +winogrande,acc,0.5682715074980268,0.01392087211001071,0 diff --git a/evaluation/rankeval/lm1-2b8-55b-oscardedup25_1.json b/evaluation/rankeval/lm1-2b8-55b-oscardedup25_1.json new file mode 100644 index 0000000000000000000000000000000000000000..e12ff24348451b11a158fe1a27445dc2e28d7b4e --- /dev/null +++ b/evaluation/rankeval/lm1-2b8-55b-oscardedup25_1.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.327, + "acc_stderr": 0.01484221315341124 + }, + "anli_r2": { + "acc": 0.356, + "acc_stderr": 0.015149042659306625 + }, + "anli_r3": { + "acc": 0.3333333333333333, + "acc_stderr": 0.013613950010225601 + }, + "cb": { + "acc": 0.5178571428571429, + "acc_stderr": 0.06737697508644648, + "f1": 0.33915343915343915 + }, + "copa": { + "acc": 0.73, + "acc_stderr": 0.044619604333847394 + }, + "hellaswag": { + "acc": 0.4208325034853615, + "acc_stderr": 0.004926837572202164, + "acc_norm": 0.5406293567018522, + "acc_norm_stderr": 0.004973280417705514 + }, + "rte": { + "acc": 0.51985559566787, + "acc_stderr": 0.03007272316731717 + }, + "winogrande": { + "acc": 0.5682715074980268, + "acc_stderr": 0.01392087211001071 + }, + "storycloze_2016": { + "acc": 0.6862640299305185, + "acc_stderr": 0.010730179119317623 + }, + "boolq": { + "acc": 0.537308868501529, + "acc_stderr": 0.008720675606388452 + }, + "arc_easy": { + "acc": 0.5812289562289562, + "acc_stderr": 0.010123487160167808, + "acc_norm": 0.5664983164983165, + "acc_norm_stderr": 0.010168640625454107 + }, + "arc_challenge": { + "acc": 0.26706484641638223, + "acc_stderr": 0.01292893319649635, + "acc_norm": 0.2883959044368601, + "acc_norm_stderr": 0.01323839442242817 + }, + "sciq": { + "acc": 0.866, + "acc_stderr": 0.010777762298369686, + "acc_norm": 0.855, + "acc_norm_stderr": 0.011139977517890145 + }, + "piqa": { + "acc": 0.720892274211099, + "acc_stderr": 0.01046565794849823, + "acc_norm": 0.7377584330794341, + "acc_norm_stderr": 0.01026250256517244 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/evaluation/rankeval/lm1-2b8-55b-oscardedup25_2.csv b/evaluation/rankeval/lm1-2b8-55b-oscardedup25_2.csv new file mode 100644 index 0000000000000000000000000000000000000000..c07d30f50007d24641170751508eb9642ff3a1ab --- /dev/null +++ b/evaluation/rankeval/lm1-2b8-55b-oscardedup25_2.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.312,0.014658474370508998,0 +anli_r2,acc,0.347,0.01506047203170662,0 +anli_r3,acc,0.31333333333333335,0.013395739415639082,0 +arc_challenge,acc,0.26535836177474403,0.012902554762313967,0 +arc_challenge,acc_norm,0.2858361774744027,0.013203196088537369,0 +arc_easy,acc,0.5862794612794613,0.010105878530238137,0 +arc_easy,acc_norm,0.5732323232323232,0.010149141043955638,0 +boolq,acc,0.5318042813455658,0.00872734558341918,1 +cb,acc,0.44642857142857145,0.06703189227942398,1 +cb,f1,0.28909506521446815,,1 +copa,acc,0.74,0.04408440022768079,0 +hellaswag,acc,0.42013543118900615,0.004925717008099714,0 +hellaswag,acc_norm,0.5406293567018522,0.004973280417705514,0 +piqa,acc,0.7268770402611534,0.010395730264453267,0 +piqa,acc_norm,0.7317736670293797,0.010336761992404485,0 +rte,acc,0.5090252707581228,0.030091559826331334,0 +sciq,acc,0.882,0.01020686926438179,0 +sciq,acc_norm,0.875,0.010463483381956722,0 +storycloze_2016,acc,0.6825227151256013,0.010764505409830932,0 +winogrande,acc,0.5659037095501184,0.013929882555694053,0 diff --git a/evaluation/rankeval/lm1-2b8-55b-oscardedup25_2.json b/evaluation/rankeval/lm1-2b8-55b-oscardedup25_2.json new file mode 100644 index 0000000000000000000000000000000000000000..dc3647cae44f003a5510af121d8e6cb14200d9fd --- /dev/null +++ b/evaluation/rankeval/lm1-2b8-55b-oscardedup25_2.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.312, + "acc_stderr": 0.014658474370508998 + }, + "anli_r2": { + "acc": 0.347, + "acc_stderr": 0.01506047203170662 + }, + "anli_r3": { + "acc": 0.31333333333333335, + "acc_stderr": 0.013395739415639082 + }, + "cb": { + "acc": 0.44642857142857145, + "acc_stderr": 0.06703189227942398, + "f1": 0.28909506521446815 + }, + "copa": { + "acc": 0.74, + "acc_stderr": 0.04408440022768079 + }, + "hellaswag": { + "acc": 0.42013543118900615, + "acc_stderr": 0.004925717008099714, + "acc_norm": 0.5406293567018522, + "acc_norm_stderr": 0.004973280417705514 + }, + "rte": { + "acc": 0.5090252707581228, + "acc_stderr": 0.030091559826331334 + }, + "winogrande": { + "acc": 0.5659037095501184, + "acc_stderr": 0.013929882555694053 + }, + "storycloze_2016": { + "acc": 0.6825227151256013, + "acc_stderr": 0.010764505409830932 + }, + "boolq": { + "acc": 0.5318042813455658, + "acc_stderr": 0.00872734558341918 + }, + "arc_easy": { + "acc": 0.5862794612794613, + "acc_stderr": 0.010105878530238137, + "acc_norm": 0.5732323232323232, + "acc_norm_stderr": 0.010149141043955638 + }, + "arc_challenge": { + "acc": 0.26535836177474403, + "acc_stderr": 0.012902554762313967, + "acc_norm": 0.2858361774744027, + "acc_norm_stderr": 0.013203196088537369 + }, + "sciq": { + "acc": 0.882, + "acc_stderr": 0.01020686926438179, + "acc_norm": 0.875, + "acc_norm_stderr": 0.010463483381956722 + }, + "piqa": { + "acc": 0.7268770402611534, + "acc_stderr": 0.010395730264453267, + "acc_norm": 0.7317736670293797, + "acc_norm_stderr": 0.010336761992404485 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/evaluation/rankeval/lm1-2b8-55b-oscardedup25_3.csv b/evaluation/rankeval/lm1-2b8-55b-oscardedup25_3.csv new file mode 100644 index 0000000000000000000000000000000000000000..f2957de38269e9c94586eff293e5c58fb206e897 --- /dev/null +++ b/evaluation/rankeval/lm1-2b8-55b-oscardedup25_3.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.318,0.014734079309311901,0 +anli_r2,acc,0.355,0.01513949154378053,0 +anli_r3,acc,0.33916666666666667,0.013672343491681813,0 +arc_challenge,acc,0.2773037542662116,0.013082095839059374,0 +arc_challenge,acc_norm,0.29436860068259385,0.013318528460539427,0 +arc_easy,acc,0.5896464646464646,0.010093531255765457,0 +arc_easy,acc_norm,0.5879629629629629,0.010099765857562764,0 +boolq,acc,0.5253822629969419,0.008733779541853497,1 +cb,acc,0.5714285714285714,0.06672848092813058,1 +cb,f1,0.5334753619757763,,1 +copa,acc,0.76,0.04292346959909283,0 +hellaswag,acc,0.4195379406492731,0.004924748500639342,0 +hellaswag,acc_norm,0.54371639115714,0.004970672651595843,0 +piqa,acc,0.7301414581066377,0.0103565954218522,0 +piqa,acc_norm,0.7290533188248096,0.010369718937426846,0 +rte,acc,0.5342960288808665,0.030025579819366426,0 +sciq,acc,0.879,0.01031821038094609,0 +sciq,acc_norm,0.88,0.01028132801274739,0 +storycloze_2016,acc,0.6798503474078034,0.010788532546733105,0 +winogrande,acc,0.5524861878453039,0.013974847640536203,0 diff --git a/evaluation/rankeval/lm1-2b8-55b-oscardedup25_3.json b/evaluation/rankeval/lm1-2b8-55b-oscardedup25_3.json new file mode 100644 index 0000000000000000000000000000000000000000..8a5c450284279e4a1521c33acccb59e2cac9f2b2 --- /dev/null +++ b/evaluation/rankeval/lm1-2b8-55b-oscardedup25_3.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.318, + "acc_stderr": 0.014734079309311901 + }, + "anli_r2": { + "acc": 0.355, + "acc_stderr": 0.01513949154378053 + }, + "anli_r3": { + "acc": 0.33916666666666667, + "acc_stderr": 0.013672343491681813 + }, + "cb": { + "acc": 0.5714285714285714, + "acc_stderr": 0.06672848092813058, + "f1": 0.5334753619757763 + }, + "copa": { + "acc": 0.76, + "acc_stderr": 0.04292346959909283 + }, + "hellaswag": { + "acc": 0.4195379406492731, + "acc_stderr": 0.004924748500639342, + "acc_norm": 0.54371639115714, + "acc_norm_stderr": 0.004970672651595843 + }, + "rte": { + "acc": 0.5342960288808665, + "acc_stderr": 0.030025579819366426 + }, + "winogrande": { + "acc": 0.5524861878453039, + "acc_stderr": 0.013974847640536203 + }, + "storycloze_2016": { + "acc": 0.6798503474078034, + "acc_stderr": 0.010788532546733105 + }, + "boolq": { + "acc": 0.5253822629969419, + "acc_stderr": 0.008733779541853497 + }, + "arc_easy": { + "acc": 0.5896464646464646, + "acc_stderr": 0.010093531255765457, + "acc_norm": 0.5879629629629629, + "acc_norm_stderr": 0.010099765857562764 + }, + "arc_challenge": { + "acc": 0.2773037542662116, + "acc_stderr": 0.013082095839059374, + "acc_norm": 0.29436860068259385, + "acc_norm_stderr": 0.013318528460539427 + }, + "sciq": { + "acc": 0.879, + "acc_stderr": 0.01031821038094609, + "acc_norm": 0.88, + "acc_norm_stderr": 0.01028132801274739 + }, + "piqa": { + "acc": 0.7301414581066377, + "acc_stderr": 0.0103565954218522, + "acc_norm": 0.7290533188248096, + "acc_norm_stderr": 0.010369718937426846 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/evaluation/rankeval/lm1-2b8-55b-oscardedup25_4.csv b/evaluation/rankeval/lm1-2b8-55b-oscardedup25_4.csv new file mode 100644 index 0000000000000000000000000000000000000000..f14b47f628516081733979e10ffc351be0cbfdd8 --- /dev/null +++ b/evaluation/rankeval/lm1-2b8-55b-oscardedup25_4.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.317,0.014721675438880215,0 +anli_r2,acc,0.35,0.015090650341444233,0 +anli_r3,acc,0.3425,0.013704669762934732,0 +arc_challenge,acc,0.2781569965870307,0.013094469919538804,0 +arc_challenge,acc_norm,0.2858361774744027,0.013203196088537369,0 +arc_easy,acc,0.5959595959595959,0.01006906164954955,0 +arc_easy,acc_norm,0.5871212121212122,0.010102837421104668,0 +boolq,acc,0.4969418960244648,0.008744891407361649,1 +cb,acc,0.44642857142857145,0.06703189227942398,1 +cb,f1,0.2833268050659355,,1 +copa,acc,0.74,0.04408440022768077,0 +hellaswag,acc,0.4208325034853615,0.004926837572202164,0 +hellaswag,acc_norm,0.5416251742680741,0.004972460206842309,0 +piqa,acc,0.720892274211099,0.010465657948498228,0 +piqa,acc_norm,0.7263329706202394,0.010402184206229218,0 +rte,acc,0.516245487364621,0.030080573208738064,0 +sciq,acc,0.888,0.00997775303139724,0 +sciq,acc_norm,0.879,0.010318210380946094,0 +storycloze_2016,acc,0.6862640299305185,0.01073017911931762,0 +winogrande,acc,0.5714285714285714,0.013908353814606703,0 diff --git a/evaluation/rankeval/lm1-2b8-55b-oscardedup25_4.json b/evaluation/rankeval/lm1-2b8-55b-oscardedup25_4.json new file mode 100644 index 0000000000000000000000000000000000000000..bbfb410bcee85fd70047f152917c699e4bf7c5a9 --- /dev/null +++ b/evaluation/rankeval/lm1-2b8-55b-oscardedup25_4.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.317, + "acc_stderr": 0.014721675438880215 + }, + "anli_r2": { + "acc": 0.35, + "acc_stderr": 0.015090650341444233 + }, + "anli_r3": { + "acc": 0.3425, + "acc_stderr": 0.013704669762934732 + }, + "cb": { + "acc": 0.44642857142857145, + "acc_stderr": 0.06703189227942398, + "f1": 0.2833268050659355 + }, + "copa": { + "acc": 0.74, + "acc_stderr": 0.04408440022768077 + }, + "hellaswag": { + "acc": 0.4208325034853615, + "acc_stderr": 0.004926837572202164, + "acc_norm": 0.5416251742680741, + "acc_norm_stderr": 0.004972460206842309 + }, + "rte": { + "acc": 0.516245487364621, + "acc_stderr": 0.030080573208738064 + }, + "winogrande": { + "acc": 0.5714285714285714, + "acc_stderr": 0.013908353814606703 + }, + "storycloze_2016": { + "acc": 0.6862640299305185, + "acc_stderr": 0.01073017911931762 + }, + "boolq": { + "acc": 0.4969418960244648, + "acc_stderr": 0.008744891407361649 + }, + "arc_easy": { + "acc": 0.5959595959595959, + "acc_stderr": 0.01006906164954955, + "acc_norm": 0.5871212121212122, + "acc_norm_stderr": 0.010102837421104668 + }, + "arc_challenge": { + "acc": 0.2781569965870307, + "acc_stderr": 0.013094469919538804, + "acc_norm": 0.2858361774744027, + "acc_norm_stderr": 0.013203196088537369 + }, + "sciq": { + "acc": 0.888, + "acc_stderr": 0.00997775303139724, + "acc_norm": 0.879, + "acc_norm_stderr": 0.010318210380946094 + }, + "piqa": { + "acc": 0.720892274211099, + "acc_stderr": 0.010465657948498228, + "acc_norm": 0.7263329706202394, + "acc_norm_stderr": 0.010402184206229218 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/evaluation/rankeval/lm1-2b8-55b-oscardedup25_5.csv b/evaluation/rankeval/lm1-2b8-55b-oscardedup25_5.csv new file mode 100644 index 0000000000000000000000000000000000000000..86fca467d563a7512396d9d0dc27306bdae81e4f --- /dev/null +++ b/evaluation/rankeval/lm1-2b8-55b-oscardedup25_5.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.334,0.01492201952373297,0 +anli_r2,acc,0.351,0.015100563798316403,0 +anli_r3,acc,0.34833333333333333,0.013759437498874066,0 +arc_challenge,acc,0.2773037542662116,0.013082095839059374,0 +arc_challenge,acc_norm,0.28754266211604096,0.01322671905626613,0 +arc_easy,acc,0.5841750841750841,0.01011334824464787,0 +arc_easy,acc_norm,0.5816498316498316,0.010122061470742853,0 +boolq,acc,0.4889908256880734,0.008742934884517653,1 +cb,acc,0.5535714285714286,0.06703189227942395,1 +cb,f1,0.3695324283559578,,1 +copa,acc,0.74,0.0440844002276808,0 +hellaswag,acc,0.4206333399721171,0.004926518439372265,0 +hellaswag,acc_norm,0.5425214100776737,0.0049717049172677535,0 +piqa,acc,0.7257889009793254,0.010408618664933382,0 +piqa,acc_norm,0.7252448313384113,0.010415033676676065,0 +rte,acc,0.5270758122743683,0.030052303463143706,0 +sciq,acc,0.892,0.009820001651345693,0 +sciq,acc_norm,0.896,0.009658016218524305,0 +storycloze_2016,acc,0.689470871191876,0.010700112173178448,0 +winogrande,acc,0.5698500394632992,0.0139146850947167,0 diff --git a/evaluation/rankeval/lm1-2b8-55b-oscardedup25_5.json b/evaluation/rankeval/lm1-2b8-55b-oscardedup25_5.json new file mode 100644 index 0000000000000000000000000000000000000000..a7c39cc27b97039fd30304248631ee6af250205b --- /dev/null +++ b/evaluation/rankeval/lm1-2b8-55b-oscardedup25_5.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.334, + "acc_stderr": 0.01492201952373297 + }, + "anli_r2": { + "acc": 0.351, + "acc_stderr": 0.015100563798316403 + }, + "anli_r3": { + "acc": 0.34833333333333333, + "acc_stderr": 0.013759437498874066 + }, + "cb": { + "acc": 0.5535714285714286, + "acc_stderr": 0.06703189227942395, + "f1": 0.3695324283559578 + }, + "copa": { + "acc": 0.74, + "acc_stderr": 0.0440844002276808 + }, + "hellaswag": { + "acc": 0.4206333399721171, + "acc_stderr": 0.004926518439372265, + "acc_norm": 0.5425214100776737, + "acc_norm_stderr": 0.0049717049172677535 + }, + "rte": { + "acc": 0.5270758122743683, + "acc_stderr": 0.030052303463143706 + }, + "winogrande": { + "acc": 0.5698500394632992, + "acc_stderr": 0.0139146850947167 + }, + "storycloze_2016": { + "acc": 0.689470871191876, + "acc_stderr": 0.010700112173178448 + }, + "boolq": { + "acc": 0.4889908256880734, + "acc_stderr": 0.008742934884517653 + }, + "arc_easy": { + "acc": 0.5841750841750841, + "acc_stderr": 0.01011334824464787, + "acc_norm": 0.5816498316498316, + "acc_norm_stderr": 0.010122061470742853 + }, + "arc_challenge": { + "acc": 0.2773037542662116, + "acc_stderr": 0.013082095839059374, + "acc_norm": 0.28754266211604096, + "acc_norm_stderr": 0.01322671905626613 + }, + "sciq": { + "acc": 0.892, + "acc_stderr": 0.009820001651345693, + "acc_norm": 0.896, + "acc_norm_stderr": 0.009658016218524305 + }, + "piqa": { + "acc": 0.7257889009793254, + "acc_stderr": 0.010408618664933382, + "acc_norm": 0.7252448313384113, + "acc_norm_stderr": 0.010415033676676065 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/global_step52452/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1830b0bef385a8267430608e9052f869db04b599 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bebcea3d977537b6f4717088f7d2501a97b782aa1d52397eb1605636b3a7da9 +size 131677719 diff --git a/global_step52452/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ee7d011113872fca11961d659f1fc907f368851 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ae98d50a6bab4f7c3b08e9f40b0932ab9a195ae3296907e72c95e3f5a115255 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4a1ebc314a2ae6395babefeb13c20c0dda11a154 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bac64b7c53a3310d4bc5d2f0319b1a19ea5ddaa836d334aa413dbcd81e702f9 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b46da0f09e420bdbc89ed82fddcebca3cbc79c2f --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96dc22a5eb621a7443ca26ef4ed380b0f52c9ed70fc150c54ebad220dc8fc95d +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..151d362295f35f2fd4b8db286eea8556f9868785 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:120e4662d8efc6cf1c6e28f52d5ffe01b80ad559f5eff5a6a4e96dc262c492f7 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3c48ad4e98e26a03c53daf6051b8752aba90c106 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c9e38140b6503c44dc317c860d83fb65fdd674758a56c03d7d7bd0d91546ea2 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b7f5fcc4299eecb145b26fb434246d902c40d925 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d2e3bbf829bcbee2f5e4206a4c9d5eb781df56962d93a502c342e27a08c8c4b +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f53daa3f6430da81cf9249cf265d3772ca428e5f --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fcac30842078fbbce8be10c4330c8411c159b3716834dad499a2597b9b2dc49 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..983a26d4a2e1267c442d70347c521483ffaa4904 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f7496174d386d20a57e79c4e1594a0b3a9737a8780fcf2be86f8dede058c631 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a3fb8eb235b834f16b82bb3a69009aa4d4129ee9 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33d520f2f2eee2312a1b03a6e9e98c32d7d866d21fa112a7ed8d950f72ab8b1f +size 131677869 diff --git a/global_step52452/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..55ad5f0ff266cc77c296c228b671674a51065ae1 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35fd96e2b98c91435f666cf6b404a61397b68b764d0ca9d74fc706d7ebbadaf1 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2179d16ce0da5ef4a657d3eee35349045db4bf39 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcac4ea62c499fcae0737f2ef998d40e88066c9d7a28333c23ef908c8bf352b0 +size 131677794 diff --git a/global_step52452/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e4751515fb532b7e109a932f418a46102f6fbd9c --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7b309864afb2d3b6e783ec76c395e5a56aae68d9c1cff0d1a881b5a143de196 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a5f31e967c35b6f106b6354614ea0fc36c3a5526 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:579fb2c4a6641dc35a22b4b77522aabd212c5b2c6bf3819a6ca7ede065b776b0 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c1f8a95996ad8a9355391252721361fa9227014e --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfb8ba5ff3ddbd7015efb1288fe990a67d6d58d530fd6e2c975c2c2d8a0a5bac +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..73c9839339990261c324ed69f385141147acb26c --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abf6be2b9651a6747aa89cc9e1d3f18d0086baf6e1e690c3ac90fd4dad88dc01 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1a904365c4f1286be105bcd86a0a88eaf3909fc7 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f04d9b1562c9cfce35906282a88f769191d13a3803a061639b88b1745bad2f6f +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dd2d95ebc62fb70553ac510fd0b9abf281122695 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a651e30fd4d8a40911071bf16ce274e16a5af4f9d7112bd4f2a854130ad0be98 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..205139e56a75239d128c646c294250399baf9d36 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac1278b8cfb9a038d61932229e87a3767490b17f73e01247b2ab5ce133cc6ac8 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8281422c5f1400c83bfdcb007fc25b5bcff31c4f --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f1b72a20641e996f0968677b9428589803c403215744cd48d3043c8b9595a96 +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e91e44edfed017106ef3c7de324eef6e1e14908f --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b37f8c039cf6ae871aeb758f9270ae1badd411d39b1088a3a8955e462e2a5bfe +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f8786d592ab47e8c157a5112995ba86b1b1ee5e --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ac1abda0adc66f3d497af57be0f31239cc2cf28376de1ac8537f69253645e8b +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..120a82c566a5176ad179f451718936eebf17f304 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21a2d1642413a72e1bd7492f85bbad87a346bc898bd9f865f049fb104f14888e +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0853fea627aeb6449253cdc59db5aa85594a2bff --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34a2af30b2298f2629e8272b4c5532cf3c0de7715f595d8e599563087da1c7ea +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fdb661b5f2bb2dd67528b2d68ac2dfe91fc2da37 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec74f24c85ece31ea36051b9d82da810d3e59e93bb2b7d611200e63f40e000f0 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eb7b607990b52205a06069e0d2f4f701e96e2593 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16b836b0d7a6a14d42ec2d27265ca6cc73637deb83afa1edee3cdc9c0f63fe14 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4afec75d6c2a9f7956f5e626b96984ba3aff2044 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62b74825a065ea4db506883d34cf501f91f5215c278c12f5db11b3243e159510 +size 131677869 diff --git a/global_step52452/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..042f4a2227f47bd5f1d44bb7d0dd51a404131a6b --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4297d250e8f0fc2f18b9a028b20537d7584ec09716e1d62dbb5cdf20046e2ae +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..875eb499a4a60095225065540c0f92987368cacf --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5e34e4f3dd68e14319974cbc925d5194da1488d6d93b633fb9ddf8b9b55ba79 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a1198fd7da24b4fa0dbaf48e9fa8e8b8280623f2 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0048f405172f8c90db315689348e19fc2a306efdd5b0753e39bde4cf05e101c2 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e8b78bd3f20acfccfe473212a5d4c6dc677d7884 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:197245304dbeb000f6513e99e1106f8d16e9047d93a88ab51b3fddb8ca2e30b4 +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_128_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_128_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e55c2dd6bf8ff34b48976ad1daea9aabb41c753a --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_128_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b960f087ee1fa436614520e59488c4f3f0b99d29c70ef9d0f65d9a9263c127a3 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_129_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_129_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..838d491743f385dc24d26e5a2596a45e7f4f2063 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_129_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3c3eb85ed7181622d151cd66f87c813c057eb712424236f72b83c82fe7fa183 +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f3b9615cc04568ef8645bd8e4b117efbd34cd07 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d821264cccb2ceda1c0b48b019d0f722cd871ee4f8a8221e167f99d303596cd4 +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_130_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_130_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..72956c994694440c20e813d83ed8ae0ecd65651f --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_130_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86427cd570831e2fa0dead7a1642e495ee66627499c4e2881d4e0fb894cf4ea3 +size 131677869 diff --git a/global_step52452/bf16_zero_pp_rank_131_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_131_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..66cc3ee7fad4f5846324644025df48d4b36ab908 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_131_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d0b454d7bae199f3bf83dc82085fde3e5a72cd58962129f4c696f20913a1fb6 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_132_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_132_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e54cc302dfd62231bef0805dd1d624808b673438 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_132_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f93a0d17a0717b9ec1b7090001f9c0879ff08e2c43196e98928d3468083bce6 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_133_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_133_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c881565090e77b4421526d73a859c047fa073eb2 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_133_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c050349bfd04d3b7c00ac045bd2b058a26e84ef5c2780ecee0cb432d78b270b +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_134_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_134_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aa2829a172377891ed628fa89adcb8c41a18e1cb --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_134_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a71482c31a694df4ab183444b470c5fc51ae02f84480501d145c742106a57512 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_135_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_135_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9695d1b1c5d90e77117d08e77e5f0f573f47db58 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_135_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5293dbd2f51e8594032540521e86386bb6d98caa5029260a482443f28986e025 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_136_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_136_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..379ba241e9cb7d3bd723f16b58cdf5cdb0d6f1a3 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_136_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b1cb3987e9588f4cfd4f01cb964ac23481ca7128608b84fa8010d5294571605 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_137_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_137_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e823bbf8235f4991a788f3379bd287ddff8ae096 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_137_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:108d3a352ec2fcf046522f1f646dc08c70df420c580382338958502c282fd9f7 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_138_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_138_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0dc9ac2615c870d915424b96eff2507a89cc49f2 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_138_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d7651eb25d9933c89f442a2c0722d52de63999b73675a1ee2733975f262592d +size 131677869 diff --git a/global_step52452/bf16_zero_pp_rank_139_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_139_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c9a6ce22cc0a127874edee691795be7fb0b91e34 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_139_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56102666ba319e61b28d073efc6cb81ff4da61322c7b44aef238cbb216c2c870 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..368ad68c3db95a95caf65930105ba55ae5739170 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b16635158b0b078e30389edad5cfd561e38c1166bcd2911996feae729513c2cd +size 131677666 diff --git a/global_step52452/bf16_zero_pp_rank_140_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_140_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a8c10582d450f442efba085615de379a2d6478ec --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_140_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23ea000ebdb6ad61ddadb5315d6e7ee527cc786a1ec86b3e7f672dbad4593202 +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_141_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_141_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5c4eca08f09094402e9cbb594fa54599864b15f --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_141_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af9d3ae0e37e78f10954d0d16e0b50345484161d81c6ce70a01e08a60c299131 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_142_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_142_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..277c54d67f468afb845e9a935b8b3d79fc66ffa9 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_142_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51d324cf5bfca8819b0bf83c817df027f2caea55e0f61b7a7715364457f57ce7 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_143_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_143_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..78dca189207fafa1f2c7294a2b8ab3af6a684bcf --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_143_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c518390f13d4ee97c72de7a577d4f370d7b0517701a1d549732031f751cfb4fe +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_144_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_144_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b0b3ee8b51e9093216e22a1662cd315fbf9d26ba --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_144_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cd4eb6f47b3a52b970bd79928cd64a8ab8e9cadfa4c6cdaae96da157ce6bdd1 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_145_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_145_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b2d48344559bc1082edffe51d9c349e947770688 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_145_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05b324996f243c90de01813fc872f0159d5206f3d53d18b994a2c19bde260344 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_146_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_146_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..465de7f89eca6564845c60c232f58ed8bce8fc46 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_146_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0b109b0612aa4b4c613e7839d2a9e45938f221b556e8ec8a7b8d28e9e5faed3 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_147_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_147_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3102ef26aa602bc09dda7867cac97f728c05bdb0 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_147_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1919c56ebd4c13999e86824d3457b4c54da4a9ae3e84f2d2758c8b99a0ea6ae +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_148_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_148_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..578057989d20a41da66550acb7ebf77af9df5a56 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_148_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd89f1e2ce694e41346bdd27f8ada1ba331e0f4d70344add1114899ff5d47da0 +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_149_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_149_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f2aa13176410f28a2df527bc579b542f3317151 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_149_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e958c7be69092c0330ece383986739ca5931373def97998c6fc64b75726a9742 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..62f6a45f7d54161e78840293093cc9c6d83b4ea0 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:879b9a52e4aa6c5202b49b206df7c6ba3b05316a0743bf20cda55f340c9cb992 +size 131677794 diff --git a/global_step52452/bf16_zero_pp_rank_150_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_150_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6c9d57311840b120e3558f82198507987156e110 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_150_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2195d85998abacfaa18b2537d5367fffb40d38b37a6758543d9124ba322fecfc +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_151_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_151_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d110337efd728ab9afd1aa1e2e742ef02cadc6d --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_151_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3645c86f824eb3165e04720b26a95023a86f14c1824ede1b93a4eb20d36a6a59 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_152_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_152_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..97af5748d0b710ab51f7ee484429fb919fec71bf --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_152_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:682ec7cd6f58f9fc7a352aeb48c428271cb0ab9a7629272d18e858ffa6fd8b56 +size 131677869 diff --git a/global_step52452/bf16_zero_pp_rank_153_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_153_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5327188a496254625b59e91dbd0a2e053b4d621f --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_153_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52079b7a2e05fc22a011a53355c5ae6f4d8e77380a8ff430be809501cc17e898 +size 131677869 diff --git a/global_step52452/bf16_zero_pp_rank_154_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_154_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..da2c154d7c58794ee30dfd04520c2ac811ad090b --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_154_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f110208650eeff637df777ca1ea49ab85b9be86b2c2a621fa347709d7a1ffe6f +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_155_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_155_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf647fcde39f458f7d90cb6f0a906eec15f211a6 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_155_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e396b16a96723f1de2cd5aedb29e2008eb375f86241951e2608d2703dd95affc +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_156_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_156_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cbbcedf7a0a14a639fcdcb82c081ceabe2d994bd --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_156_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:baf9242d95dcef120afa1d8f6c60bb0fafd3b07f64c83e40afa010d30a40e227 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_157_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_157_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1fafbc271ac48fa18c5a732d5fee4b179552f2b6 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_157_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:801ac1b487d82219dea0059a1e607481e0108faf21fbbb5fba2d8b6cc5b3d65d +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_158_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_158_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a8c561b9ad961768cd6d48c841c1ed7a0978e30 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_158_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d39aadda07496a060bb21011a5524532e8095be7e727420a5a39a63febe67803 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_159_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_159_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ab4f7e4059aa64d0be2ff4e31f8d8fec2efe0409 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_159_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a39b6de3107bc0f413dd3680ea3e8d738bdd6c45bab48299fb54671108a214c +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1363250f07155f2397f06690f0b474ae67ca46d5 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:239eab93a066a3ce171859d1dde0babd61b501daa860b50f9709dab3a8d74cbf +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_160_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_160_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..665de78bacf90880a1687e3cf28a3703770817dc --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_160_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bb03d2a46bf7910c3fa6f422a79e635f3a8697031e8023ddf9e66ed4c7af3c7 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_161_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_161_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..50e0472f240e6f5cc749a318937f36be3474ef65 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_161_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bedc20da4b62287b15f1584cef55f46a920b8e196216c48f6535a1fdec03be4 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_162_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_162_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8547f04da99383ffa87fbcd4adc93eb9c0cac7f7 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_162_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b955bbde00ff0ba4ad53425ec8966087b77273df5c7b8aef773345b78b18f854 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_163_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_163_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c3d091057149bd6471b931868e27d42d6c58f77 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_163_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:703dbe8353a4bb27ea0ea8b31c13b5b7a337903f4bb1418aaffdbcf0ed6f8fd3 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_164_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_164_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f152f6ee12ea7b92393f9eede4cbb36b6507355 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_164_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d108714fec39bf7ab415b6a85b039e59380a6eca786fb046bc99e16bc4682272 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_165_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_165_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e376ce09e2f460959b804686bdbd6a15f25eab2a --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_165_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89e1891c0b7b9ff8394c322cdeddbbc8e29a611aebae7ba60eec17fe8c0e6a22 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_166_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_166_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f0cbcf345d14e022c8718971df72e3a51ab1f1f --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_166_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e49b694b0c5b8c11b8563bdcb722e44f35e2d88e0768e019a172d3a5bce41455 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_167_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_167_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0e906a775e6078d6556c37686b490c82691dbd54 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_167_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:803e563a5de3223f34baa0bc73b2e4952eecebb634babb23a07cf4b8615bbbe2 +size 131677869 diff --git a/global_step52452/bf16_zero_pp_rank_168_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_168_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..72ce145ebb0ab59e813fa741c7be66223b25480a --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_168_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8aa6f68b9bb8c38d483d4e0b26f0fadef2a683346d2a2400326f0c381906b8c2 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_169_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_169_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e0ced9bc0250c559d0e6071f4efaa1e27b80773a --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_169_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2877e4fef9651e2be25ffcd40031559af0955349f7d0ced8ad98ef9e1952671e +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..203c0da738b26c6dde7b5a70defbf1d719d7fb96 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2241e84a765e11677affadc3c48574c05c2d794976bc5aaba973bcbfe44409e +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_170_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_170_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cfae980c4c2790fb2e12f65e9ae4a01d65bee73a --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_170_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a63535b5d74ba3a0564a612b10bcd7eabf95e08e0e0750d10296b7a3089e92ef +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_171_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_171_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5e5f2c899f6f72bc363dd244f40da42d55b6caa --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_171_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48874d09935dae99b9ca8be51593a7d2a22603166713c9d734d1b3767be2b413 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_172_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_172_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..417e249be776eaea149d03f14a1593810682fd6f --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_172_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9b79653b881f75a33e140c6c24002c692a7a428e5d10c101b2bd9822c066fde +size 131677869 diff --git a/global_step52452/bf16_zero_pp_rank_173_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_173_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f27a2dd65d8881124fedded3896f24c608177dd --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_173_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cff314ee92018e65d464e01f90ed113e265f873e7e32f513edebf3da16903d06 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_174_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_174_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4cf5fb231bfe3dcb1bd17e1b23f987d93652db17 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_174_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abb217e47c405b811679e373cd0acb0791e37f5b07c65b05a5c48ac061bf66d4 +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_175_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_175_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3164788c585ecd2b10946b60eebd49f0e7bf5cdb --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_175_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15052657bb96edbf315b446e679be570a43d8600e057d8aee989bee11ccfaf43 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_176_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_176_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7ff8209ebc222a0ca5babf5a3e255a2735bd282c --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_176_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fde0e0f9c96d4a29984f9ed3524869893127719ae5cea78b901018db69a7fa2 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_177_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_177_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ab98f3caeff98470b1b92bffd98488aa18e2eb3 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_177_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c2e9f8dc31773c82e67159dce6571de6e8cfcd471998026ef229c1f0915d4ba +size 131677869 diff --git a/global_step52452/bf16_zero_pp_rank_178_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_178_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..88f3372c482314ee5c71a0d33fb2261ac58897a8 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_178_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f7d3503de97595228c08ba0809ae63321e4eb14decbcf420bf7eda73a8721b1 +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_179_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_179_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3f2ab13985f12f6c2333b9605a1a553f1180a81 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_179_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9107351b2a3549c1498c381b3e722453e68fcfd717817902c1fac9dd22ea0a3c +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..46a44abd8b2997509f1b30da04ae15c9a2b40239 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28ae7aa031197bf584415a971a4a0861d0765a59ffcebbd7b3ccbb5efee357d9 +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_180_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_180_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ed5dbcf25a65d03ff78b7e55e0e69e038c985caa --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_180_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c50bc2c9dd97a30519553a3e1d51f773fb210751b896245b12bbdf0293da10f0 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_181_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_181_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7598ab83cd203ca529101b75f2ea511d65540e44 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_181_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35f46f657cb84e868ec59ba79bfa10b2356d00da712f38aaccc1285d21e08cce +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_182_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_182_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f9726321dda5bebc9b7857342bcca6ca97fc70f --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_182_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d83e2020f7e16d863fa04e30b7f17f784679eb1582058aaef3c354b636f09c3 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_183_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_183_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..15a9d744610fbabd9555ac8879a59c14ee428cec --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_183_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f43652273ec826bac76465f56bf154a31dfd3a91016e9f3f467d7853fda1ef3a +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_184_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_184_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..741b1877dd196ef97aab0be2938a75786366c5d3 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_184_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63a42924b2b047af45ecefd955888804a90e7d671e8c41b5ed90bee757fd8559 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_185_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_185_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8f1bb4e95486cfba4f242ebc89361a8d2a4e70f2 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_185_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:231699f129a5f7588798791a55d05daf97c6f32af5409db512e1dcff984916e5 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_186_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_186_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8881b3bddf755b06a90199477f3ce398c78f7fc7 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_186_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a179bf8923913d98d2f5d0ebe8b6072aed78cd2941ea1b395d4a5b333ebe5df +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_187_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_187_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d79f7f4ee99f291688ebfa45f7abdee2c0fe7e90 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_187_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb4a4fdf8089a3f47c1532bff5c2c4e5cfd41473bd8319c00155ca14e209f64e +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_188_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_188_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe2bddf0c8d185d349578285a6feafa999e0fade --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_188_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6862821e73ad3fac3b0a62a43d29fdf9f8df2201d32ac94500837bc8bd41a212 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_189_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_189_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f924c7edd5ef45c3fffe3f07adf2f7c91d9f01d --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_189_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a22537613014acafb4dc4d8e4d357eb1388ad57c735ac9acc3acbfde620565a +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f41d32afbbed27a9259c4dae859653e086d53b8 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2caa19436eff69a9ccc3a631fd64df46be1384016e84215d0c123a09ac8e30dc +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_190_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_190_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e7073a01e11ef818ac0e3809979fcc5b3330dc2e --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_190_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d5886c0c16ae027316f12c204bd55ecf38e6a2d53e0055ef2984e76726f9ae4 +size 131677869 diff --git a/global_step52452/bf16_zero_pp_rank_191_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_191_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b4247f4c022a685b9e358c083669ee04fd5d1b0a --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_191_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a1491ed921f298cd10c9fd8bdbd1dffd2f7a63df2a20295d962805cb0f145ae +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_192_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_192_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dc2635e081f16347a7e5855f7d048ad44f1bd24b --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_192_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c4d5ca9936d51ba883e7e2617102d844296f18c171473a0d47177915b28f279 +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_193_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_193_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b5af7c93d5b6692b4573fd2b74ea29e6cc67071 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_193_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d40ea4a1e2616bb135a6d28911391afbdc3384c5cad26de3e605e6f94171748 +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_194_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_194_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..647e6a66079be6ddaa3355788aa3eb4d3235fb0b --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_194_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1a602fc76cae5a093ee81132355e2ca7035ad773035edf98da97c12d91eb7a2 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_195_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_195_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cafae33ea810f6afbc8adbdbd8247c8c85f4899b --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_195_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e0024b7d2eb71aee77bdf0e58c8796631065916e9c658d59d02dfec630b5b1e +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_196_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_196_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ad22bd9296128c06c0d0afff50f2696d2360015 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_196_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d90d657bff18f8fd4f64d8d4781b456d3a26fb15ecf6cfd2cbfc2f962e336d51 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_197_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_197_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..806ce724eb33a7af2ac07480ee5981470ff7f9cf --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_197_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4c6bc85c5ae58dd7d6af160c7f628bc8cc41931396e1319fd4f5b103ebfa7ab +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_198_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_198_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..41d26f270c1b04b1e1e6d0ade94cd7e944b7ea4a --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_198_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f3269846670ff6d16ec1c9de375a9985239ba2f03cfde9aa3a7f8bf31219b54 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_199_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_199_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..696e3f26e301b26691b6191260292696bc500517 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_199_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5c6dc3243b0a2329c16e441a31f05c0f77d28a9c3c75334a57afe49cc5d24f4 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d1aeb3c557738670d08fe1b867ee2180c28e5c3b --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2290ac782ff1969361729e90f674021268c914b2839de89c552aaf319422eab4 +size 131677858 diff --git a/global_step52452/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d3d08aafc75b2455fb994fac9e7dc9d2d1a4ba20 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5c190131dc51e1b8c1faf465c4edf6e52322d8af82ab0ddda7a4d0e2fa46f2c +size 131677719 diff --git a/global_step52452/bf16_zero_pp_rank_200_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_200_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..44f14f850d78528969d68e9e80246b48e7632f03 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_200_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:382f3e66084ea61084ea15236b819ac11651a3d3cc04b629dffe15d6ed6d7224 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_201_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_201_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e8bdd56c97c909f1f7f6e0a2025f767731f96662 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_201_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9304b5cf201318e09e73a90ec5d542f0a070984699a0df277c104b328e883f0e +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_202_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_202_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5dab617cb31be95afc904f9b12326072fcc1ccec --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_202_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4e2fb7359969e8311af063c1476778c0d1cea23d2a267eb7ba4b8f82e974745 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_203_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_203_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1344f06b5a768c692810f1e42ed7b3a00d108c30 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_203_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f930af0939a5d74a1e316d741bb666aabf0ecd6df7c2e0e0133412045ecfb551 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_204_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_204_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..22ad562f353b2912cfd28ad44c8c05012b051059 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_204_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6bb45c744a3cb83c759a3610897edbe09f43cb01c3d243dad81a7e0710be5af +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_205_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_205_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f310e3d760b02ba3983248e40a5c0e0afa73adb5 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_205_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:840c89dbe8ca85bf69bfb770fee23ddcfc5e62ed4d21b3397caf97c2488b8434 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_206_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_206_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7ada1fb010b3c26a323f45d63e3cadea620320b6 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_206_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b53d8cf053bc298d2eac7ee4df95027b234cd467f19db36369cea30809ca59a8 +size 131677869 diff --git a/global_step52452/bf16_zero_pp_rank_207_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_207_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..051227995dbee7f46a9ad22e80fb451add3eb0fa --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_207_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e417cabb5834e33e89bc2918cc28afc1ddd3a8434b5b29de955ed3f88f29fcc +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_208_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_208_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..343162e28f8051a8413ac991471f6e3bed69d18c --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_208_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd91d0a0a60054bc86359fdb09b7583b2d4c1efdf8eee98ba0a2400321434707 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_209_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_209_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..50bfb6b3dd225cdd8dbd07e3fcfb830d91f51e3e --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_209_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44fe36f2327b6efc2d4fb71f40743b044f88d5857468936d73995a467ff85b0f +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c92b9bd3b97c970996d32cee5542cefc111436ea --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67d10f0a4ea2fa17819be26495d52732f8650a40f5bc96a0f762aa1f985eb902 +size 131677666 diff --git a/global_step52452/bf16_zero_pp_rank_210_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_210_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2c04576f2a48a14d7d55474ecffd51ba54607a80 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_210_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0c62a2baba7c43b00f7d23c572e6d681e9fc46f2e26a0fedb8a23d3242b796c +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_211_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_211_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7e2c8d4adc1a4ee321c8e467c5aaba1995e3a832 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_211_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ec6ff673e972c82e36d0c8f912a0589a8bb77a0f705eae57b81cb8412b360e4 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_212_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_212_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..54efab23e73271bb03d59dc019fec59a6e9121eb --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_212_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c331d9ee8fcee2229daff201b1a47719848833009c06bea5a13b9ba1c67924e +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_213_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_213_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3838125e490401835cde702c3a7742ba17e1a97f --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_213_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e3181474011001b35d54ed94f33911333c20529d49e2f0a6d071025e2c9e2df +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_214_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_214_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eb74e35303f414ff0c1ff7c495993c484a9d69d4 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_214_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bc3cdc12a513651b0be14ad4e6a46b826d8f6ffe232305d8cb212dcdfa097a4 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_215_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_215_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1bb03b33e7357d467570aa3017161b87bc925662 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_215_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d1e52422f93e0733a88312f821d88a1f923b2415984fd165bb6f535c2b2a445 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_216_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_216_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fd5660522586b011523ee4709ac5f101da9ce87f --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_216_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22e155543a18af4a5e8bc728ee8e4749c4812ce3fbd19aaf5fc7a5745f9d7a91 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_217_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_217_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..76adfc942bde9421d0294ff025fdd81f6fcc2db8 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_217_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbea2ff96179cb6f53e484b4c58a172959aa5cd388a3ec50fa2e223f5a27a388 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_218_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_218_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..17021e01f1756f06842d7a004d71a0482606b75d --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_218_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98db2d69e726ef66067a5b78a6e34b7551543936ddac35b4e9f83d8522476b9a +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_219_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_219_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1c30f172f69159a4df4a5f7528b312b6d6e213df --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_219_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c25af1110d1b6cef361680b58c2e139e3c4b1cc3474c712ad1abd72a07b5490b +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..99ae0a8d3fff7b30469b1d77ec25d84339c70935 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8eb31472548f16f47f429f8bd9b8de68611fb24f2a3c5d08a8b442ff3e072e48 +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_220_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_220_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..41c6bcc38e5ff862995ea840a4505b7c94072e56 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_220_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d85a3ffeeef91b854b750c2e385051460241c6dee640d02df6772e0b94eebfb1 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_221_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_221_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6187df195f064c5ce6b0574b53d9466d6242295e --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_221_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddee3f5cdfbd014521781ac4e36ce59fec1117e5031ff178a5d6a55851dfe99f +size 131677869 diff --git a/global_step52452/bf16_zero_pp_rank_222_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_222_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4282c0c4e05201c8fe42d7ac308da68bc161f24c --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_222_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfc8beef87516f75e005ba60b9bd5025dc265e6641d162fb05767a45292a0013 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_223_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_223_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..829d329ef367ba41e764680bb5d8e96617627504 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_223_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cba9b48b3eacedf65a7c2a2f39e5ef1cb330fcf25879a8bc1268920ce0a57b2 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_224_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_224_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7a3dbf6f2700d7cf817641abb731e69223dade6b --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_224_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79e64560ff5e98bc117b785a68c5fa53d74132d392e16200b4cfe6b5fe502c96 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_225_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_225_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dde5e0ddcf311e1a967377159e13224f2e72f4a7 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_225_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43e56068fade1d0fd3fc50420fb9a96f625768c73ed3b999a748abe62f6b53bc +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_226_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_226_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3313393d7ce263ff892a15c65283d2618530f1e6 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_226_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:243d13eea04a907d993682a3f85c6869946a9911ae55cf58ee27bd34d641a3b2 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_227_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_227_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b9eca64698b321b1c74ed0f9415a9acc46937a7a --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_227_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfa1d661924a4f588f8defc9426853f61dc5634d0f3ba5ba9d3f10f60e549929 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_228_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_228_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5912c7ea94e39ea08829de3332de693655de3566 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_228_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b0d96691e8679b60c74a7b6be8300990289cc7ff15faf20b47447b63774083b +size 131677869 diff --git a/global_step52452/bf16_zero_pp_rank_229_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_229_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bb7a2e0be26bb5b41efd25d40724c872d092e30b --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_229_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7552e5454b0c70f7f203faeb1fbcabd81d899d345851624117da96e002c7f514 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1a3cb0658444ab9c2ed1593eadd5c59177d33021 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a9cbbe1cdfddafe22a7da4d0b4744db4cc53902ee9baffd2d9740b499e55dd4 +size 131677794 diff --git a/global_step52452/bf16_zero_pp_rank_230_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_230_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..745e9082d656342204171834a5c737ce8efd71cd --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_230_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7b1e7fb23cd18534df877c79a4aa933f6ae53f33680e53ba18a586a4610e3de +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_231_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_231_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..518a38a70c7fa6ef55785a6dba2e3c488ff32c98 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_231_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:955ea9411ddc0ae6c7ed53d127c489b561d70a3d3f6d129662e1ecd862020c09 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_232_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_232_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..af39a4474e1153eae6aa956ad427a119b1d939f6 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_232_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b47e448f18994d3da1bcef52926bc9312907cf4360406f153d238772ad00137 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_233_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_233_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..209a1685361757ca157af7e40fc1386b2d18a00b --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_233_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b44ede2088950e1458dfad9978f4e3b9bd706c7ab14a6749b06bb9ebb20fd40 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_234_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_234_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1a47b60c05ca7db7d5bb0e4409a415122e485d15 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_234_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c2045cad4e25a3e79f0b9929f16dc545aaee0d122fba62b73ae5594f549f1ea +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_235_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_235_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a38c517080156ad564d496d1fa65069341654879 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_235_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f9df9edba10479e06ae57004dbdc91604482ab2dac904a3f8946749f6ac3d32 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_236_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_236_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d43dc04a0e372fe4eab6e37c1da695550b5b2d3 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_236_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e832553489c14e079cd6aca1c1405f617703606d871e6c76d3a38934a5001412 +size 131677933 diff --git a/global_step52452/bf16_zero_pp_rank_237_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_237_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..967f5b0cc4e25a54726e54fd0e8c11fed6648d34 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_237_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90b174a7637824242bf234b8369f8368f94fd5f93f233a97b68bd6be5593b95f +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_238_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_238_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..83b0d5e4eec12b1c6e5811c2b61b0a56cb693461 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_238_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ed9b33c12ca1c529af8977063f77c24167eec292df5f340dda33a54d1012c7f +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_239_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_239_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..503a6fef8f379044f94d45a9d98e7e4f28cebb86 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_239_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91bd9a7bfc13544960c734662f103bdee438cf5273f7b3da0ba3b14ce92ba84f +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..96e4f48a2a55a6959c502d8da8abb545b0ebbcfa --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:466b772b0433c569712964abae46a359a4a414c08b895b5b3bc19493f0489f96 +size 131677922 diff --git a/global_step52452/bf16_zero_pp_rank_240_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_240_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0225926015244fb406d88970c53017cd852f1ada --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_240_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:082e0b3b471b6c704ff30bfc92038f9b56c11edc20f74cae0820bbcd9a15d3b6 +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_241_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_241_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ccf8bc5cb37ac6bf4c286c6d29eb0799d925e89 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_241_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b89164d9a02088a7dce4cc5fde80c8e53e7ada89f3bcdac4026789fc36e39e21 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_242_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_242_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..237f561d5f9a810e080ad5d6e56e2cabda237b8e --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_242_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ad8407b9c3c80624edcda2791ac4b6427c40896a1d1944c77fa2d79d3a0c405 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_243_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_243_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1e116770e9ef6e828f0cf15a091322513f4fc44 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_243_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbc632d33b2bdc67306319d58546c486e96b2e8ef10a1d86c53cececa3b7fc15 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_244_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_244_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..757b645fbabdb4318065483c906ca291411090e1 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_244_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4af788b5b4f7d813a0c0e49675d89a1ef972a668adecc0742241517dcf26bf50 +size 131677805 diff --git a/global_step52452/bf16_zero_pp_rank_245_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_245_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe609f2adc9998bea2d487355835e853b7689835 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_245_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1b6eea4ef8d91f696517b25c7af592ba0c900751747be1beeb863814fd90ff7 +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_246_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_246_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1ae945b7fe6f5cc8bc97b24cc75cc9a6cfcf7294 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_246_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de95b20aa6661371e58f26264e2bd2bf41c183a19756a2159cec84497ab57ef1 +size 131677869 diff --git a/global_step52452/bf16_zero_pp_rank_247_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_247_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ae3412dbe9d890dba76aa82e9595bfcc65f432b --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_247_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4a86d7c0f1b8e3765c365d39c78a4167b5b07d96383b44918ca226d2c75c132 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_248_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_248_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0028c621c2f989be69fdeef1ec696b113cedb99b --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_248_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c29d51ba9a784df498bd374b67e3565315bde132e5cb920305a87e770c36648 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_249_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_249_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..635ec22a4b98c1e88708cfab9011ff385b0c52dd --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_249_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4381097a4446df0d4a30a780b652fa4310d0f8a46f9f3a8b8dec9b7c71ff7e65 +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e5ee0dbaa6701df3df2bbf913394c65cabb0bdab --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:437b08dd8cb085b472e49e7fac65e51ec67c4c95be332e354aacb7c02bc0d2cf +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_250_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_250_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..43c7fe67197d1e75539a7fc654ffb23477231dc7 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_250_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01766d14747a9b9cbd87574eec5d9276585061ac22dcb6f5c8b626296aae5c4a +size 131677869 diff --git a/global_step52452/bf16_zero_pp_rank_251_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_251_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..797e5e692704df71587d6d8df66cf9835e1525b5 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_251_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3484c589c7ff5b04117c217f919b565c2e5744b06e9f00795f4e3c08780b972 +size 131677869 diff --git a/global_step52452/bf16_zero_pp_rank_252_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_252_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..636ecf957a9a6bfcb64855cdb368c8e08cb79948 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_252_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf28b854c428a6f7fad7457749b115eb0826ac301cf7a369e95658ae7930e068 +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_253_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_253_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4dd8c722085fee37ffd911c86d7c550d601985a8 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_253_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8082b2553035490f3f0a0628e92d38ddd24c558703415dada44ff2a6c8b3dbb +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_254_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_254_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..710bb9849fdacc8f04088b1405d9f572537bde2c --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_254_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d60b77ef0f848b7f63c5e61b1d358ecf0ced6c2160e8e0b27f8cf17d22ee4d29 +size 131677741 diff --git a/global_step52452/bf16_zero_pp_rank_255_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_255_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8ec25da14bdf1ef1cf40735543bdc3c08f0646e4 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_255_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef06dde5b62dcb519bb35f177efc30c7a6bfe48e25c62f1deea226e4858f6911 +size 131677677 diff --git a/global_step52452/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ddc12ccc024894c127f86b5c07d4780aff1a1846 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cde9837c7d5a678f50d7848fefab5324fe7f9d8360ef2b8c8aa4dbe9ee1dccb9 +size 131677794 diff --git a/global_step52452/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d23785c45fe3a711bd13a64f08a88859499f8a83 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7636c336068b094f0c3521a7d78179ac473ba5d5b6d4c502eb81f50d5e8703bd +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1304024365a5f2c6a3129295e5f7cc349d3b0349 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c2e8463a227a11bda1c45d3b96ea7f70185ca401470f1b23f67c1d6f086f9f8 +size 131677794 diff --git a/global_step52452/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a1f5e631de408c29f876c66059534b76c21abea7 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2ee516504d8bfe1a9f885400210af7c3fbc0c1766a40dae30e27705789dd6de +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3eec31af63958aabdb364119fc82135309815f6e --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5adb9c0a3a50ed5057f5827e4b47a16a174f4475213a7443a732ccc469178b1d +size 131677794 diff --git a/global_step52452/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..41261cfda5a77288dc755bca74490248013876e8 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:096cdecbfbf9694535625fce90b848898295e939cc6323f07de29eba5d73ce26 +size 131677719 diff --git a/global_step52452/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5b7c868c854f2002decd3098327a0f3ee2ae1b6b --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd3472173d3e91ff9a7f82e552354292a3c117b6099e19e380d905f4012dedb4 +size 131677666 diff --git a/global_step52452/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..15f2f44840b25c2a55e86e885d1452d207afeab4 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37ca88b765aefb048043645294faf4a2bf19bca42eee2707dd3d1ee4f0c43faf +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3427caf595305be75e58d2e74ac30394a6cc37bb --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0733f870d36fc2910ab3b7c980d66c95941d44cc0a6c1ab8661abe5095350781 +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..afa6289eef436199c6cc4215e17ef313cb36fb3a --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac5de80ba69cdab12dd98360e3c1f81785e7bb807193d9e680d3ce27f31ef853 +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ac58de9a80653f235714d94aae2c1b04b87b43ed --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90eba8b2ce5dd964212b290b7cde727664e7af4e29d67460fb0b023257b7f34d +size 131677858 diff --git a/global_step52452/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5639cd0070b89748e3ec6df0cdc40ce1f0cc908 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d930ef24e6de988dada706c06355acae9f4a6c2cf71f2395115541739b6e7e2b +size 131677666 diff --git a/global_step52452/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c35fa32a66964ba26c6d79e77c2c9d58aff1f6a5 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3533bf46e5f8734d83b8e9baef9a4ebb06f01e2b1e1b78a427eaf37bb764a2aa +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7daa4850a7665cf95d211b9fc41cefad5560090d --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c97edf23ae1fa42ff667d4ab75442c0e365f88c14259700521090581f40b325 +size 131677794 diff --git a/global_step52452/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b04d2fd0c0fdfb2b3c67a33908094791eeec442a --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81a12ac6e9601022d80a25454bab2d3f737250a5f9d6ed4b5d9da69c52daa095 +size 131677858 diff --git a/global_step52452/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c99e9f84cbf231db793bd5c78de646e9f8f4596d --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c630cac515100476499f6043cd909482a0e4764c925f0e1f5f2bd61ece7d56df +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe84b5b70a003c418c30808318c14d8026149bfd --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c73a7b23c9ed46b4d3ddf33d5eaa678d679905ec0562a0db5885ab195736c1a +size 131677719 diff --git a/global_step52452/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..06222bf842f7385246f6122c9bce3dc83cfb6929 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed93fc2494a587ee826d4bc1e12fc29be3e0b3c82aed485e60f7bc0b9aa44410 +size 131677794 diff --git a/global_step52452/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..46cfe0787eb73639974f7931421d21f9d7599143 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e396d299c8769e37bab51224842f09fa962e5dff814fc64f09fd527d3f753bdc +size 131677794 diff --git a/global_step52452/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4a16a32c0f96ff121eedaed39379ae5a085248a3 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c94d5b7e43f8df926b9278c9b597aefd8dc4865bb2180de406277dd7b38dfd7 +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4e1f139cd7f79e0e1f7d9cc1f718027201ca8f99 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3213b19b6eaae62df5b9059a166fea809ec5ee8cf4118bd8f68e6634126a2093 +size 131677666 diff --git a/global_step52452/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..436a2417125010354e0eac2dad3d7b8c642d3c0b --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8e86dbb106a609a45ee9a2e84a2e4b299f71c7f71614ebf2665c28692fcdf91 +size 131677794 diff --git a/global_step52452/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4bfcd05086f6e34c9cb8fae11ddbbeb40e829ccb --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a33df8543e0f9fa44fd6ff8afc0199f22f9739b42e425329472425d460b933fd +size 131677666 diff --git a/global_step52452/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..65bb898b9d7fe1ab81129dc9c9bf456da53e5ce1 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d264c9661024373b1a0d2ce59e037ba7a20cdab1e6f2eaa875a55861d2ef5a7 +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3d0f5e18c5c8d67e406ab3d5bebc601b3a7b793f --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de3e223803d09aff7040dfdc23ae1b9debd546e9398c8c5cdca2fca0bba022d0 +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..012d3a0e2d368b6eadc7053445da55f5d2fb3eb3 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91eee7d1618ace432f5ac77a6cf0e6f54b6433af8f0a81939d5d40f4eb178c05 +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..18b01280e03dedb49710ed678325db9b1f5471eb --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3baac9646d9932efa2ae0ce644b3edd9ad28e7a0719d6d9f281809ae604441a1 +size 131677858 diff --git a/global_step52452/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f4cf4403127170fb0f4ebc63b19e7dfaf7cc8e25 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39e7c4324ac19393847e4e4f71b0cc89edf7737262c14703c64720458e526bf5 +size 131677847 diff --git a/global_step52452/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a7e070cb80db6def07c176318664c6e5ca9f1b9e --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5dd6a80c42c9fbed956e1aaaaa14a5984bf6aa64f07be696825763dd7de69801 +size 131677666 diff --git a/global_step52452/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c316a37e0369fa8144f2b0d964ba3fc9001f074c --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:609b6a563e2743641f0511cc0acae44771f8355e307031d87e5142d04a9b5e57 +size 131677794 diff --git a/global_step52452/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eca3499f95e609b65660544d83e2ecb823a3a692 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c70a382a4b39a54b1abca840119a6e0e0423d64103bf3a8e22749601af31005e +size 131677794 diff --git a/global_step52452/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..05ae9eea5a23e3dc9238f64742fb873a532ed991 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8023456542f8e15e20a50dc39b8f07bf0fc248842761761063948230d4feb3dd +size 131677794 diff --git a/global_step52452/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d1825ca8f8ed327424a7472ac619827a0a40c1fb --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:074964ee5fbf24df1b79a419e631cc17e69be9f77fbb7c6f70ddef2568d87f7b +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2c472da2a3fa1d0b67eb0fdb105fac157818ebb6 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63bd5cdb03492c6b19b9d4f172d057cfc989475884cbaab34a24a7e191eb5929 +size 131677858 diff --git a/global_step52452/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f235601556f8e5590f84ba4256f74c59ea15c53a --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:201c8a654c205e0d582c92d332ff65d70630476d47a2fe7a39d3e6132bc36efc +size 131677794 diff --git a/global_step52452/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4fccbf85fd7be5f1ebfc9485719992fd6c17ed67 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18dab8476c81bb2720ae8f759804ae7d39bd155b5eb073d6a575eff7f4a973b7 +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad9b923d532ee5acb9b3d7dc2f241afb44bd13f9 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da6d5abfb97d5491157de3b6317ea1a83adf3599f918e1120f0e2dba612e62e7 +size 131677666 diff --git a/global_step52452/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..39cc06cd03e78c195100810468a563f0762a88ee --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9058f3cbbc212e79f3e0b485bf943af2b504c77dd0e4d74bbcfd20a542b0862b +size 131677794 diff --git a/global_step52452/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..74b4d63d211b54e0dfb10e9012c3ace104e837eb --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73f4188a330e7aae4197cbbca531446ed3c36a20d3242b8083461adbf29caec7 +size 131677655 diff --git a/global_step52452/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b709b993deabf493205220c9994924323a23a6e8 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69b2cb218693c21e0cc1505557c19e673d70bfbbd70c170b5c2ed8ebef1b6b33 +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec19b24a0659aece11c25cdbc93fe8e531f0307d --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7730b83004ae3f312274549026c836cb8759b37ee8c6f9cbbe49f6a51b9037d2 +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b7309c7d6791601d31eaa28e989e14f24df37365 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:021ab536f568249d9d2eed881af1b1fde188ccadcc6ba9f3d387ab7d0fff4b0a +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a5469f9629b718ad29c4a023b94a0bd4a201c44f --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5489084a739c3c7e23083ff866ccb0d785ecee54e723ebff3a25d7f8a3f583d +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c8c200a8be7cfff23d841586afddf8fab7291a12 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d1d16928bc4d4b24973a9f3b9ae30748e7e27893f773ccc682767cd25d72b59 +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a7d0c8a5a07c472d267450ca4ac30b458e609eb1 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc239ac14765b73027b4bcf35b98db665827b2c0f25097a703a3411dae4c640a +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3aff2623586f80e2c43e5287f2aed014c3266338 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b68aad08890933d991018411d170e414c335e024c5c6c55b563b4cfcad49d792 +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a47c4ba582fd614eaaf03b87871db107304eea6e --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e261b9d865d8ab78bd58d7bfa403cda0043b43d20a217d9010b91bfec5eddca +size 131677794 diff --git a/global_step52452/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..59bc4dbe30531c61032eb34a81b039c81665b441 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1cd8541b473f98573c24d72991d4c4f7b9f844048fac51141be8072f3890eec +size 131677858 diff --git a/global_step52452/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f96d318e04ad6a5e960b748ee6960398e3ffbe2 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e5ba9d64f0416b3cf6917ac7f550c2a4aabd97b4c518742b271b3d5573d4493 +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c7d11b8170c1bf0bc10b99569dc4e281203c6ab2 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee6c365a73fa3564155b1d77b566eed38bc1a14386830cd8717a3d3132ba1a2c +size 131677719 diff --git a/global_step52452/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7798552b2a02b1cd5e5f4546fdcc511ceb539682 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab4758bf49ce628e336c3a49f013c81c09c849c4c7e8a6b48acbfd7b96c8d4aa +size 131677858 diff --git a/global_step52452/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..95fe254285f4c8294d68a72c2728a37c67b7b4fb --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8afa8167edfe606e98012f05166fe60c8f8721237ecc3bf49407e23a917bc7f7 +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..60620ad1dff0c7480a00941facfca4c5029e8994 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5e41693033cef983fd6db49b93169190d8c0ef5e4ab06595da12fc0695c6e7b +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..512d2e9c4c11abebb943326a4915b227771d67ac --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f1006bbc399deaaffe9d82be3d0156032a31d0643faff8cee155d981d078e4b +size 131677666 diff --git a/global_step52452/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..27a5a232cbe0a6b0494239c8b70ce6ca86d6c898 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b66c8e48079728bf8f29231ab88ae0379a3e832391a43fc4b0ccdcf08ffa28f +size 131677794 diff --git a/global_step52452/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aa793b52076c65f073ecae9fd61dbf00e5332b1d --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7977cdc0bda58c6fd1cb8321281bdb4b3bf29f0db660afb633cdbff0df56384 +size 131677666 diff --git a/global_step52452/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f3d17fd6101abbf5cb2d738a548660419da59c6d --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4930adb2708af0e996db0dee225d6c4553a413f0cc1b9f9c7f5dc56df64006c0 +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d3b038568d6cfc5f56d77a971985030c2b17959f --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d970384f753d19b03d197b65b5f8ee8a67447357283a47a0d1d1dbef29cb4ce +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d738ad46ffa1bf9781011a024f5d1225bba2416 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d0aa2afece928147d941631923d6b87d266c7b595d57949256f7583028625ad +size 131677858 diff --git a/global_step52452/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cada26d9bc5fb001d5763ffcff00650d5aadf016 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b748d9ffc661769614b84ea5c68fabb89446dd4227bec1aa049246cba9ff2b97 +size 131677794 diff --git a/global_step52452/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..95dfc5339882a13788a3c4a51a8e4448433f38b9 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb38db313c70bda61925522aeb1deabbf21655d9f849ec90d480c76fa664565f +size 131677719 diff --git a/global_step52452/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..40e4986252fd6e14e5405f7d8e4b80c5f14749c8 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28208754c0a0f5acdcfb9f267aef1ef710c0fcea8fe77eb651fd91580b4c2f64 +size 131677666 diff --git a/global_step52452/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..94dbcab776a406c029a4515280ea2acfbec04250 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09d093fa44a06cd2fdd9da0e45d489620c76b6db7c5d60688a569a0ccac56acd +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d26c6868c279cf655222f38ddff108727cb4fc9a --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f8e78ea45d6e8a2119c800a5fa8abbc24dbdf5f8c6451c21c3157a102e09b26 +size 131677858 diff --git a/global_step52452/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1d88393c1ca1c048d24d241a8241ccb349450206 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd690529429afc068dd3f220d4ff0e7d1ad520d591456d41be17c4291cc65377 +size 131677794 diff --git a/global_step52452/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ffd2c72688601b4c04d8f2e86da6402b6a6893f5 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f7a7e8dd4bb0029aa2f07f0eeb9057abc87dccb7a886ad730f69b8effca994f +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..837f88ab3493178c1849243726e6865ce9ad90d7 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9528efa3fd31e44329d23d364b9e2777c11052eafe7f48d157b242c71eeda865 +size 131677794 diff --git a/global_step52452/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fc22e07860d50e27c373bace69c9e4e31a2d1000 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eaa4d61533254f6157cad1b4dbb2f417259d60e70bac34c034c6e4ccb985dba2 +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..44f3a1dae80a49734746b14888b6d3312496e63c --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e3ad4b9c96132eee126555986ff7cd7c1ddadedca48cb402219f00e3a560e93 +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..18f242e914e86b0b9377be9ce6b76cfa407a3966 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41f0a9af597986dd15b237d546477ce16c55ee68e5c8a01861607bc2fa9baf15 +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3c49c330fed193c0b22b2cb11a4f2446eb911ec9 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ad22adacd53f01a500d18f721755f5da78acb1b833af59dae49df4b9a9bb509 +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..daee1f06f13bf78123c57812cd02af9ef710daa6 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:576d9544698acd59237b82d35cd0e28947815ede4f971fac55fa2b4c37d37ad6 +size 131677847 diff --git a/global_step52452/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b543e89585cad8e07e34cf9fa1bff4d59bd8e9a5 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:754cbdc2892e2c6c5bee049d65dd96f966d8238e8e2fb093a92968bdc9cb8a53 +size 131677666 diff --git a/global_step52452/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6b4f5d67f4ecf0ed3856b68c7bfc14cb44b94fd6 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d8762f6a36a8bae9f000971a8963accc062ba2316d97350efa502e22aae7981 +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d732bbc92cacaae54acc975e7f95150299baf64 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81093f013c0f30bdf2d97c4fb76a5d6882935e47225f4e8b2b766e5a2a675ed8 +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2054af9534ca877876358904f0a82ee1a4aa8ccb --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dae8cb4bd3fff255bfa80e78133fc0cbc46c409ac3ef990a644467fb2ec59cc6 +size 131677922 diff --git a/global_step52452/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e6c7a9686f133e9e7b2ed9aaf4dc1a48f1dafcde --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a0a2982d2f106b480a27a4fe26be82306272291c1494cd23c6fabc2611726b7 +size 131677730 diff --git a/global_step52452/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d4067cf0b7158d59e1bf9c8814eb10633af7c9e6 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2275c7fe618fe0e1fc9560e5684accd7e7dc0523759dd9da6c08e669c870279f +size 131677666 diff --git a/global_step52452/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c8c2b638856ea279061f090c553f106075d97207 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:983cf3759f5cf26ae96d84a68b1f3711b58d055732588e4a8e25008ad2589df0 +size 131677794 diff --git a/global_step52452/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..93f404989ecbfe4a750ac6b97f3e443abb307cf9 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:793d4b73a0f76b3446aef0591d612741ca944d7ba5f2c22c4372936fe3ace7b6 +size 131677794 diff --git a/global_step52452/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0aea54886467232bea94c83527c5a82d6edce6c5 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ddf6e8ff473af35b6cf762e6c512faec96d2ef1bb57db07dbcdae6fc505c773 +size 131677858 diff --git a/global_step52452/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2293109a279bad8db2110a7020d4b66063724fa6 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65d4dd1e7a3a09808875b477afe6f38944d8112a4dd18e0eb7d91b2f47c46eb9 +size 131677666 diff --git a/global_step52452/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/global_step52452/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0e01853c90d7ed1f2ae3f6ad1ef220e7feadbd21 --- /dev/null +++ b/global_step52452/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76eaac5e747a8395255eb326bce2be35f9dfe27381714137ede5adf4e4ec3117 +size 131677719 diff --git a/global_step52452/layer_01-model_00-model_states.pt b/global_step52452/layer_01-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..be04e0fd1c981ea7ccb36165467591f6be39a828 --- /dev/null +++ b/global_step52452/layer_01-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e08fca5e25b2a83c827c258992fdfdd937b4e30a1b26cb836403cf43d36ec7d +size 268043523 diff --git a/global_step52452/layer_03-model_00-model_states.pt b/global_step52452/layer_03-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..922fab7e05f235e801b044431e361fee76ddb80f --- /dev/null +++ b/global_step52452/layer_03-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbc2524cd5ac32cb1cbe3a6bc846fe5de172bf186a8710482f514fe1206c0908 +size 157357315 diff --git a/global_step52452/layer_04-model_00-model_states.pt b/global_step52452/layer_04-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc49073b9edde8a13fb6b5e8b40d724efebc06a7 --- /dev/null +++ b/global_step52452/layer_04-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36270e954b33dba78802c58d365451d37a3a4ab144698a687508f93a9195471b +size 157357315 diff --git a/global_step52452/layer_05-model_00-model_states.pt b/global_step52452/layer_05-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3c8f86a01f9e989c6878d655e14b542b8dd0703b --- /dev/null +++ b/global_step52452/layer_05-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e284e3500ef11f517d2b8a8166e29e0e4fbed9d74ff3314474d9fe307cc6ccd1 +size 157357315 diff --git a/global_step52452/layer_06-model_00-model_states.pt b/global_step52452/layer_06-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..83c1cf6a791317f175d0fe9f8778d2992676cfcd --- /dev/null +++ b/global_step52452/layer_06-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7726e0e9ca08782e43ed995f80802bd5361d2587c168d7ba1a59a7d6e8ecff8d +size 157357315 diff --git a/global_step52452/layer_07-model_00-model_states.pt b/global_step52452/layer_07-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..088e001562042263edfa43c9304bd495087fa213 --- /dev/null +++ b/global_step52452/layer_07-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e122d5a2a7a594677379dd5cb4518aa02abfeafdd224612eda6be37ccca662af +size 157357315 diff --git a/global_step52452/layer_08-model_00-model_states.pt b/global_step52452/layer_08-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e71fc177c7fd31c1b6b24e6e00169ad39147663 --- /dev/null +++ b/global_step52452/layer_08-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6f8d72f9305bb43ed1f568e763a4a6915c14be8d26211e642ce7ba2c8201730 +size 157357315 diff --git a/global_step52452/layer_09-model_00-model_states.pt b/global_step52452/layer_09-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..10d20b2f2b15dbc8358d0797eb2c0308928458ca --- /dev/null +++ b/global_step52452/layer_09-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fab5a8b777f8701250653a172e5467ed045d3319163969a941263201648404f1 +size 157357315 diff --git a/global_step52452/layer_10-model_00-model_states.pt b/global_step52452/layer_10-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..410b3bfcfb19fad90251a37895d711777d2e60f9 --- /dev/null +++ b/global_step52452/layer_10-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35178e406bb51442cf6025eabd538cb704fa913e0b14f9a86ce35f153ba5d47d +size 157357315 diff --git a/global_step52452/layer_11-model_00-model_states.pt b/global_step52452/layer_11-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6bd3de329bd5f6b7b8954e39f07eb19a84f39969 --- /dev/null +++ b/global_step52452/layer_11-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc3169930aad760c0794e206cf9141810cf6b2502fa9e4eebaab388b254b27b2 +size 157357315 diff --git a/global_step52452/layer_12-model_00-model_states.pt b/global_step52452/layer_12-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a8ce6f9ad4ade17a4ccfbb8f92190ef7ad56820 --- /dev/null +++ b/global_step52452/layer_12-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77361e98da1942d9f53fc04f9e967267e8b21889b617597bc34e18d2c57e3788 +size 157357315 diff --git a/global_step52452/layer_13-model_00-model_states.pt b/global_step52452/layer_13-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad61e3e30a0de12bda50caf6b37127764c55b4dc --- /dev/null +++ b/global_step52452/layer_13-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0decb22c5f179b1adde671d41d76d58dfcfb7788f5a0ff23fe54305dc549d50e +size 157357315 diff --git a/global_step52452/layer_14-model_00-model_states.pt b/global_step52452/layer_14-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8c45f3481beee07b9c2c7349230703f1fb04b923 --- /dev/null +++ b/global_step52452/layer_14-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a50112dc49d7715f2e026fbe64cc31b134abf16ff3831c30409871fea3f6ca97 +size 157357315 diff --git a/global_step52452/layer_15-model_00-model_states.pt b/global_step52452/layer_15-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3a5351f2a0fb75c1982fc0fa18d77602e24cdee5 --- /dev/null +++ b/global_step52452/layer_15-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbda8d954c34e9168f85220707836a7c13ae91fe3a32fa4d2b9cd575039d2d5d +size 157357315 diff --git a/global_step52452/layer_16-model_00-model_states.pt b/global_step52452/layer_16-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..116717fc5bab1704287188ccdab40465cc5a2a15 --- /dev/null +++ b/global_step52452/layer_16-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4faf48f9defd6b80cb47fa8c821dab7b8dd62c39547da312ebdac0fd78524ef +size 157357315 diff --git a/global_step52452/layer_17-model_00-model_states.pt b/global_step52452/layer_17-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..705f3cec70ef91e5926f06b58a2ee9fd5816c2fd --- /dev/null +++ b/global_step52452/layer_17-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4039407420b96b90449270135a2ea91d01594693a365ee07e1e463944c5b7fe1 +size 157357315 diff --git a/global_step52452/layer_18-model_00-model_states.pt b/global_step52452/layer_18-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6764410d8bd2c850559eceb11002bf0727df436 --- /dev/null +++ b/global_step52452/layer_18-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edea95dbc09913fced98f55d28188c6ac89baa10c3734ea4809de5f0db0f95b7 +size 157357315 diff --git a/global_step52452/layer_19-model_00-model_states.pt b/global_step52452/layer_19-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b8fddc184c23497f8d97c6d65c13fa6367cfaf8d --- /dev/null +++ b/global_step52452/layer_19-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:874d2e1534a1438d99458b505f562e42f83ddd60d96d7c1583c0bf559a6bd7f5 +size 157357315 diff --git a/global_step52452/layer_20-model_00-model_states.pt b/global_step52452/layer_20-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7d64b09b63b4764240b06cf4e475960e90ceff78 --- /dev/null +++ b/global_step52452/layer_20-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:966d514bffef86d1b88fe57d101426bfe1d8f29ebb3cc9f0c10543668b7008d5 +size 157357315 diff --git a/global_step52452/layer_21-model_00-model_states.pt b/global_step52452/layer_21-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d46bccc5a83cc34cf977f9e895ab21bf9f36542a --- /dev/null +++ b/global_step52452/layer_21-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ac219fdffb989bf2ddb5aae24ef0fa79616fe47d143e041581e1b8a26d9e200 +size 157357315 diff --git a/global_step52452/layer_22-model_00-model_states.pt b/global_step52452/layer_22-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eae1be0f08e77088101b0fa5c5e39b113cec3abd --- /dev/null +++ b/global_step52452/layer_22-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4b1ba2e5b775640b317239e50ff0d47ea08304793902d98e96d9144fcbcea8d +size 157357315 diff --git a/global_step52452/layer_23-model_00-model_states.pt b/global_step52452/layer_23-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..968f6eb009a88e2dd0a582552a64672f6cc17de0 --- /dev/null +++ b/global_step52452/layer_23-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f77c7a432ba5d31472b91d318592366c9dade065e9412e1bce04a403a84de5f1 +size 157357315 diff --git a/global_step52452/layer_24-model_00-model_states.pt b/global_step52452/layer_24-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ee815d9a45c7d61ac87eae7b1b9b96e123017eb8 --- /dev/null +++ b/global_step52452/layer_24-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02401345320559195921380f08789d7815d1365b32f2b75e644482bbabe6af98 +size 157357315 diff --git a/global_step52452/layer_25-model_00-model_states.pt b/global_step52452/layer_25-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..63a0046719bcb9db24a266cd6ef4e538fb578935 --- /dev/null +++ b/global_step52452/layer_25-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f8337d6465aa4ed03f6c3791b7bff1c8cbe6ccffd18a662bf6a5b33f2c8ace7 +size 157357315 diff --git a/global_step52452/layer_26-model_00-model_states.pt b/global_step52452/layer_26-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..092d2f0f4b39656a932db6eee9311523893694b7 --- /dev/null +++ b/global_step52452/layer_26-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e689880b7a74b9450c0f5fd6488619d0b60ae2b340d58b1958ad20d8b8fe883 +size 157357315 diff --git a/global_step52452/layer_27-model_00-model_states.pt b/global_step52452/layer_27-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9e511ded60185994278c146c6ed61b3b21da564e --- /dev/null +++ b/global_step52452/layer_27-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4eb47ea681aa319518eee83ce509fbbb0d20a4ee6d71844b6f82ca1f0386432 +size 157357315 diff --git a/global_step52452/layer_28-model_00-model_states.pt b/global_step52452/layer_28-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..079077ae16524aabd82b9effdaa6fa8290523034 --- /dev/null +++ b/global_step52452/layer_28-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21b289f80ededc083af42dc2475a91bcdd013e5216d026e1f4ffd37f6df1b258 +size 157357315 diff --git a/global_step52452/layer_29-model_00-model_states.pt b/global_step52452/layer_29-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fa33ad24a1f7748e09521df01db0996bdd506796 --- /dev/null +++ b/global_step52452/layer_29-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32edda7168ec4a6d67d5ab134bf345067a058212155d98f232a4f2681106b5a4 +size 157357315 diff --git a/global_step52452/layer_30-model_00-model_states.pt b/global_step52452/layer_30-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6c0acc0493eb94bdcdf30c8dc184e3227c3583f2 --- /dev/null +++ b/global_step52452/layer_30-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3f6a72190838ec982e2aa442f0ca93d33fc8a47ee619657ac8b12a7fb852eb8 +size 157357315 diff --git a/global_step52452/layer_31-model_00-model_states.pt b/global_step52452/layer_31-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..88f1a140a7c05339d19356b9946489f2c53632d4 --- /dev/null +++ b/global_step52452/layer_31-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22c502eb878c4db627d9596ca0ff7ea49776616732051d5e412af0618a9201d0 +size 157357315 diff --git a/global_step52452/layer_32-model_00-model_states.pt b/global_step52452/layer_32-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f7cb89c08ab1c9eedbaef021789805eff4dd8acb --- /dev/null +++ b/global_step52452/layer_32-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83555cf6c5e28ba02bf4afc3a5ab990c2413f1e13f93a98df9f9b233b4f7048b +size 157357315 diff --git a/global_step52452/layer_33-model_00-model_states.pt b/global_step52452/layer_33-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dda306e034da412cb73d62967ff2c9573531bb45 --- /dev/null +++ b/global_step52452/layer_33-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d04123f5b5f3dfa86bd50ead4d4e7e812b3301d068000f8cb1a0e7d7b7d4c237 +size 157357315 diff --git a/global_step52452/layer_34-model_00-model_states.pt b/global_step52452/layer_34-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..94ef95993b7e6dc7034eb846708278748ddc27e0 --- /dev/null +++ b/global_step52452/layer_34-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e7b06e0d5694c575e7dd62ed88c9c3c46b6dc461a87aa444743f8707ff71deb +size 157357315 diff --git a/global_step52452/layer_35-model_00-model_states.pt b/global_step52452/layer_35-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d529a85587ddff38a5fc6350105fb8e1f6f665b5 --- /dev/null +++ b/global_step52452/layer_35-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65bc7a40c1af4cfd0e3902bef5918c4fe6408e88a8f3395d43664e4336506d8b +size 157357315 diff --git a/global_step52452/layer_36-model_00-model_states.pt b/global_step52452/layer_36-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f1fbcc576d5aa02bf707e4cca381e70069ddb88 --- /dev/null +++ b/global_step52452/layer_36-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1699abcafffa6ab1535a44c06d10d817aeda593d68c59ac82101c429b1bdf207 +size 157357315 diff --git a/global_step52452/layer_38-model_00-model_states.pt b/global_step52452/layer_38-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4106ebbc0f5c09b6ba3a11dcda9a69c061614144 --- /dev/null +++ b/global_step52452/layer_38-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:928f52273668009619d0f8a2b31e50c380c6a5b4307b2b030124df5988ff6277 +size 11459 diff --git a/global_step52452/mp_rank_00_model_states.pt b/global_step52452/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..731a3f3cd31d8fa449f6dd311a65fac8300ea496 --- /dev/null +++ b/global_step52452/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28c09fe55b6bb9649ce64435176c3fd159b9cc7e7cdd13dbe5527065c8f4133a +size 50163 diff --git a/tensorboard_2b855boscardedup25/events.out.tfevents.1675558512.nid006726.61448.0 b/tensorboard_2b855boscardedup25/events.out.tfevents.1675558512.nid006726.61448.0 new file mode 100644 index 0000000000000000000000000000000000000000..17325380ec49404646632b9439be141b092f23cc --- /dev/null +++ b/tensorboard_2b855boscardedup25/events.out.tfevents.1675558512.nid006726.61448.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21636808503ef58d89ec185b3488676c1199f72d8edade028b66f445a4c0b462 +size 1005003 diff --git a/tensorboard_2b855boscardedup25/events.out.tfevents.1675956275.nid007208.82606.0 b/tensorboard_2b855boscardedup25/events.out.tfevents.1675956275.nid007208.82606.0 new file mode 100644 index 0000000000000000000000000000000000000000..c6b09363f107a080599924c58d23c73d9c31dfab --- /dev/null +++ b/tensorboard_2b855boscardedup25/events.out.tfevents.1675956275.nid007208.82606.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56db27096f2684781c42ecd182740cb4a372b022d3a293358b326223902d1d9c +size 43155268 diff --git a/tensorboard_2b855boscardedup25/events.out.tfevents.1676018692.nid005411.93520.0 b/tensorboard_2b855boscardedup25/events.out.tfevents.1676018692.nid005411.93520.0 new file mode 100644 index 0000000000000000000000000000000000000000..7876c320fa41634661b1149a2427782e2f76be82 --- /dev/null +++ b/tensorboard_2b855boscardedup25/events.out.tfevents.1676018692.nid005411.93520.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97823583ad8a2ab605fb2d9cd01b998e38a531d47d591e39e2a2514f8fc5c34d +size 58906225 diff --git a/transformers/config.json b/transformers/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2debb3b510ba26c4e6693cd4c6be848955e93cec --- /dev/null +++ b/transformers/config.json @@ -0,0 +1 @@ +{"vocab_size": 50304, "n_positions": 2048, "n_embd": 2560, "n_layer": 34, "n_head": 20, "n_inner": 10240, "activation_function": "gelu", "resid_pdrop": 0.1, "embd_pdrop": 0.1, "attn_pdrop": 0.1, "layer_norm_epsilon": 1e-05, "initializer_range": 0.02, "summary_type": "cls_index", "summary_use_proj": true, "summary_activation": null, "summary_first_dropout": 0.1, "summary_proj_to_labels": true, "scale_attn_weights": true, "use_cache": true, "scale_attn_by_inverse_layer_idx": false, "reorder_and_upcast_attn": false, "bos_token_id": 50256, "eos_token_id": 50256, "return_dict": true, "output_hidden_states": false, "output_attentions": false, "torchscript": false, "torch_dtype": null, "use_bfloat16": false, "tf_legacy_loss": false, "pruned_heads": {}, "tie_word_embeddings": true, "is_encoder_decoder": false, "is_decoder": false, "cross_attention_hidden_size": null, "add_cross_attention": false, "tie_encoder_decoder": false, "max_length": 20, "min_length": 0, "do_sample": false, "early_stopping": false, "num_beams": 1, "num_beam_groups": 1, "diversity_penalty": 0.0, "temperature": 1.0, "top_k": 50, "top_p": 1.0, "typical_p": 1.0, "repetition_penalty": 1.0, "length_penalty": 1.0, "no_repeat_ngram_size": 0, "encoder_no_repeat_ngram_size": 0, "bad_words_ids": null, "num_return_sequences": 1, "chunk_size_feed_forward": 0, "output_scores": false, "return_dict_in_generate": false, "forced_bos_token_id": null, "forced_eos_token_id": null, "remove_invalid_values": false, "exponential_decay_length_penalty": null, "suppress_tokens": null, "begin_suppress_tokens": null, "architectures": ["GPT2LMHeadModel"], "finetuning_task": null, "id2label": {"0": "LABEL_0", "1": "LABEL_1"}, "label2id": {"LABEL_0": 0, "LABEL_1": 1}, "tokenizer_class": null, "prefix": null, "pad_token_id": null, "sep_token_id": null, "decoder_start_token_id": null, "task_specific_params": null, "problem_type": null, "_name_or_path": "", "transformers_version": "4.25.0.dev0", "n_ctx": 1024, "gradient_checkpointing": false, "model_type": "gpt2"} \ No newline at end of file diff --git a/transformers/pytorch_model.bin b/transformers/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..a5db1d9ced705cd4f13b60b4eb6440e2846ef81a --- /dev/null +++ b/transformers/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c3e64c2744426e335f6d61c2720c6bb1148a1c106a29f64cca170f0adb7101c +size 5903417421