Eval

8b7178b13b/evaluation/8b7178b13b_0_babi.json

@@ -0,0 +1,22 @@
+{
+  "results": {
+    "babi": {
+      "em": 0.0,
+      "em_stderr": 0.0
+    }
+  },
+  "versions": {
+    "babi": 0
+  },
+  "config": {
+    "model": "gpt2",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b13b/transformers",
+    "num_fewshot": 0,
+    "batch_size": null,
+    "device": null,
+    "no_cache": true,
+    "limit": 3000,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

8b7178b13b/evaluation/8b7178b13b_1_babi.json

@@ -0,0 +1,22 @@
+{
+  "results": {
+    "babi": {
+      "em": 0.0,
+      "em_stderr": 0.0
+    }
+  },
+  "versions": {
+    "babi": 0
+  },
+  "config": {
+    "model": "gpt2",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b13b/transformers",
+    "num_fewshot": 1,
+    "batch_size": null,
+    "device": null,
+    "no_cache": true,
+    "limit": 3000,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

8b7178b13b/evaluation/8b7178b13b_2_babi.json

@@ -0,0 +1,22 @@
+{
+  "results": {
+    "babi": {
+      "em": 0.0,
+      "em_stderr": 0.0
+    }
+  },
+  "versions": {
+    "babi": 0
+  },
+  "config": {
+    "model": "gpt2",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b13b/transformers",
+    "num_fewshot": 2,
+    "batch_size": null,
+    "device": null,
+    "no_cache": true,
+    "limit": 3000,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

8b7178b13b/evaluation/8b7178b13b_3_babi.json

@@ -0,0 +1,22 @@
+{
+  "results": {
+    "babi": {
+      "em": 0.0,
+      "em_stderr": 0.0
+    }
+  },
+  "versions": {
+    "babi": 0
+  },
+  "config": {
+    "model": "gpt2",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b13b/transformers",
+    "num_fewshot": 3,
+    "batch_size": null,
+    "device": null,
+    "no_cache": true,
+    "limit": 3000,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

8b7178b13b/evaluation/8b7178b13b_4_babi.json

@@ -0,0 +1,22 @@
+{
+  "results": {
+    "babi": {
+      "em": 0.0,
+      "em_stderr": 0.0
+    }
+  },
+  "versions": {
+    "babi": 0
+  },
+  "config": {
+    "model": "gpt2",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b13b/transformers",
+    "num_fewshot": 4,
+    "batch_size": null,
+    "device": null,
+    "no_cache": true,
+    "limit": 3000,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

8b7178b13b/evaluation/8b7178b13b_5_babi.json

@@ -0,0 +1,22 @@
+{
+  "results": {
+    "babi": {
+      "em": 0.0,
+      "em_stderr": 0.0
+    }
+  },
+  "versions": {
+    "babi": 0
+  },
+  "config": {
+    "model": "gpt2",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b13b/transformers",
+    "num_fewshot": 5,
+    "batch_size": null,
+    "device": null,
+    "no_cache": true,
+    "limit": 3000,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

8b7178b13b/evaluation/generation/agg.8b7178b13b_GEM-wiki_lingua_en_tldr_en_2.json

@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.19198287608229725, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0021092410953077365}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.32589953688933776, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002753157025629824}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.22344837173188767, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018933971382747686}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.048137852198349124, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0010124293758406351}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.08482162860321114, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0017481381213950958}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.05600464749019431, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001035304930307184}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.13481580311964722, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0014071194011638184}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.23733416551629913, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.00219502068666546}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.15854970566001372, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0012774189115518982}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.18117711700554173, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0019909544088953742}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.30882558318305736, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0026454321668364913}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.21107970658856343, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017866890110014018}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 3.013745331324814, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.06592229302797037}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b13b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 8, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}

8b7178b13b/evaluation/generation/agg.8b7178b13b_GEM-wiki_lingua_en_tldr_en_3.json

@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.16688833624223554, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0024001395431129838}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.27404204769144364, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003375962083519381}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.1883894874399809, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0022518375274135087}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.04136653299690803, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0010174651332227344}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.07172155304298297, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0017383368313820028}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.04708204631655152, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010069752068567668}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.11810699062222797, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001714836740267086}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.19959382723616914, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002605476332476844}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.13391288059874437, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001564486351392238}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.15789252138792423, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0022764600108588808}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.26023963410461465, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.003240567155894993}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.17834159419543702, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002133544799751138}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 2.986263870634216, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.0869488541072268}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b13b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 8, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}

8b7178b13b/evaluation/generation/agg.8b7178b13b_GEM-wiki_lingua_en_tldr_en_4.json

@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.05571162128505982, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0019790843122430768}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.09490155511879607, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003216109108055619}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.06229352605061771, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002059973094043126}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.013746904154847127, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.000658207908545434}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.02605352352304377, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0013317962968417952}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.01596233535299914, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0007332643966253175}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.04028867827357096, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0014354449068175981}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.07013160574213247, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0024380518058086703}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.04502172063067562, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0014806563009305842}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.0526446584524354, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0018735818695187819}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.08977749314111799, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0030541817569144}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.05884434373920439, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0019486127272992522}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.7653537900871676, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.049323102026813614}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b13b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 8, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}

8b7178b13b/evaluation/generation/agg.8b7178b13b_GEM-wiki_lingua_en_tldr_en_5.json

@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.009989872401580602, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0009952828000827124}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.015276156161555617, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.001444671711873694}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.010262809104423124, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.000941433577191994}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.0024167502513949756, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00033798462070527304}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.004213844763908219, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0006183240014712593}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.0025848086028049965, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0003301678198930521}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.007350732687047713, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.000736612195363616}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.011512337859684543, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0011311946838375108}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.007530769725575652, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0006918159097560924}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.009401823549618816, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0009477935776177291}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.014304994804442096, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0013529102153531946}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.009601062566883777, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0008823539930029628}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 2.313122283850011e-06, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 4.438365983452618e-06}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b13b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 8, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}

8b7178b13b/evaluation/generation/agg.8b7178b13b_gem_xsum_article_DOC_summary_3.json

@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.14077378985561326, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002167627071465062}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.3365881713882994, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004904093832075129}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.19482488018157684, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0028449440307477085}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.03346907677060051, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00115993469618386}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.08385926778368435, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0030304591639245464}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.04702347182983231, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001623530160402657}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.10579758272401656, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0016080916023179224}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.25509688514423123, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.003826588022233898}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.14665273300314152, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.00212580713201847}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.11263878026618175, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.00176128619109869}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.2715816804884895, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004199504898315826}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.15621401458929957, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.00234420229722605}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 2.03791695157546, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.1342348737065576}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b13b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 8, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}

8b7178b13b/evaluation/generation/agg.8b7178b13b_gem_xsum_article_DOC_summary_4.json

@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.04332722979741195, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0025397484135024503}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.08638484320409213, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004855712915740289}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.05341124486563334, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0029118807193791816}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.009371982127377696, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007761604812612375}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.02197790550576286, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0018737167719494205}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.0126600494371876, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010393927624132967}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.03308111037957417, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002037483672167313}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.06547067873377617, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0037402890017668913}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.04031062592059575, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0022065055222514843}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.03547102946933422, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002145817702406172}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.07066149952031933, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004048133700252855}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.04344922023460101, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002384615926685988}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.9907623400783704, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.12144441541342028}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b13b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 8, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}

8b7178b13b/evaluation/generation/agg.8b7178b13b_gem_xsum_article_DOC_summary_5.json

@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.002432491256372859, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0006531802890059347}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.0035688165030904754, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.000969634171191206}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.0026267053439145405, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0006851925438686376}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.0002462802129838755, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00010248150275091556}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.00045873937383371344, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0002027676865696476}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.0003000575858304317, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00012455541465929192}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.001824459823195743, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.000493813415204384}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.002706254470313614, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0007319861753860738}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.001963800127619678, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0005077215919641312}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.001947339725356528, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0005292516418451348}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.0028418243363945105, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0007704086076680828}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.002094899399917077, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0005500941719652507}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 1.959625061740418e-19, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 1.1984703371139495e-15}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b13b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 8, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}

8b7178b13b/evaluation/generation/examples.8b7178b13b_GEM-wiki_lingua_en_tldr_en_2.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e1dbeb121b2aa6b21362cc27760dfac9df5f0c4392de6dc66b55afa2d2ddcdb
+size 18917037

8b7178b13b/evaluation/generation/examples.8b7178b13b_GEM-wiki_lingua_en_tldr_en_3.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9170e4bf6e70cb0474e5cc61a22d09edf230e43fbfc24971c3ca80e1f9303ace
+size 24337079

8b7178b13b/evaluation/generation/examples.8b7178b13b_GEM-wiki_lingua_en_tldr_en_4.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7121ccb97fa1421eb5d54fd4c30f575c47407e8e1d22029847bd1bc796605b98
+size 29478626

8b7178b13b/evaluation/generation/examples.8b7178b13b_GEM-wiki_lingua_en_tldr_en_5.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d4f84f2bf12c2c7f4343c7a75d12d4f3b26590608f8e73f6feb6f4760f1adf8
+size 34801616

8b7178b13b/evaluation/generation/examples.8b7178b13b_gem_xsum_article_DOC_summary_3.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbbd5b38d4479e236de19fb18b0426fc754c283c908c6551a1c23132a8bec707
+size 9647191

8b7178b13b/evaluation/generation/examples.8b7178b13b_gem_xsum_article_DOC_summary_4.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efb257030811a985c3b288620cf2719b5f2f7b4cf73388bcdb5057bd80ab93d9
+size 11673906

8b7178b13b/evaluation/generation/examples.8b7178b13b_gem_xsum_article_DOC_summary_5.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc87da448769f33c6956c2aade6eb6ad69e8c8db403d47f83be5244751597f78
+size 13899064

8b7178b13b/evaluation/generation/merged.csv

@@ -0,0 +1,53 @@
+dataset,fewshots,prompt,metric,value
+e2e_nlg_cleaned,0,generate_text_restaurant,rouge2_fmeasure,0.013098749477153882
+e2e_nlg_cleaned,0,median,rouge2_fmeasure,0.013098749477153882
+e2e_nlg_cleaned,1,generate_text_restaurant,rouge2_fmeasure,0.17675735102863532
+e2e_nlg_cleaned,1,median,rouge2_fmeasure,0.17675735102863532
+e2e_nlg_cleaned,2,generate_text_restaurant,rouge2_fmeasure,0.20791813280930055
+e2e_nlg_cleaned,2,median,rouge2_fmeasure,0.20791813280930055
+e2e_nlg_cleaned,3,generate_text_restaurant,rouge2_fmeasure,0.2138339925008178
+e2e_nlg_cleaned,3,median,rouge2_fmeasure,0.2138339925008178
+e2e_nlg_cleaned,4,generate_text_restaurant,rouge2_fmeasure,0.21707781052843164
+e2e_nlg_cleaned,4,median,rouge2_fmeasure,0.21707781052843164
+e2e_nlg_cleaned,5,generate_text_restaurant,rouge2_fmeasure,0.21807002700335165
+e2e_nlg_cleaned,5,median,rouge2_fmeasure,0.21807002700335165
+e2e_nlg_cleaned,5,average,multiple,0.17445934389128182
+gem_xsum,0,article_DOC_summary,rouge2_fmeasure,0.05279448511938101
+gem_xsum,0,median,rouge2_fmeasure,0.05279448511938101
+gem_xsum,1,article_DOC_summary,rouge2_fmeasure,0.03829439148207006
+gem_xsum,1,median,rouge2_fmeasure,0.03829439148207006
+gem_xsum,2,article_DOC_summary,rouge2_fmeasure,0.04760922765448965
+gem_xsum,2,median,rouge2_fmeasure,0.04760922765448965
+gem_xsum,3,article_DOC_summary,rouge2_fmeasure,0.04702347182983231
+gem_xsum,3,median,rouge2_fmeasure,0.04702347182983231
+gem_xsum,4,article_DOC_summary,rouge2_fmeasure,0.0126600494371876
+gem_xsum,4,median,rouge2_fmeasure,0.0126600494371876
+gem_xsum,5,article_DOC_summary,rouge2_fmeasure,0.0003000575858304317
+gem_xsum,5,median,rouge2_fmeasure,0.0003000575858304317
+gem_xsum,5,average,multiple,0.033113613851465176
+web_nlg_en,0,PALM_prompt,rouge2_fmeasure,0.051001464150726095
+web_nlg_en,0,median,rouge2_fmeasure,0.051001464150726095
+web_nlg_en,1,PALM_prompt,rouge2_fmeasure,0.05455324665630359
+web_nlg_en,1,median,rouge2_fmeasure,0.05455324665630359
+web_nlg_en,2,PALM_prompt,rouge2_fmeasure,0.055583861631680276
+web_nlg_en,2,median,rouge2_fmeasure,0.055583861631680276
+web_nlg_en,3,PALM_prompt,rouge2_fmeasure,0.05769919253564089
+web_nlg_en,3,median,rouge2_fmeasure,0.05769919253564089
+web_nlg_en,4,PALM_prompt,rouge2_fmeasure,0.06050302495157815
+web_nlg_en,4,median,rouge2_fmeasure,0.06050302495157815
+web_nlg_en,5,PALM_prompt,rouge2_fmeasure,0.06252454636994373
+web_nlg_en,5,median,rouge2_fmeasure,0.06252454636994373
+web_nlg_en,5,average,multiple,0.05697755604931212
+wiki_lingua_en,0,tldr_en,rouge2_fmeasure,0.04191366659253858
+wiki_lingua_en,0,median,rouge2_fmeasure,0.04191366659253858
+wiki_lingua_en,1,tldr_en,rouge2_fmeasure,0.04827166271383884
+wiki_lingua_en,1,median,rouge2_fmeasure,0.04827166271383884
+wiki_lingua_en,2,tldr_en,rouge2_fmeasure,0.05600464749019431
+wiki_lingua_en,2,median,rouge2_fmeasure,0.05600464749019431
+wiki_lingua_en,3,tldr_en,rouge2_fmeasure,0.04708204631655152
+wiki_lingua_en,3,median,rouge2_fmeasure,0.04708204631655152
+wiki_lingua_en,4,tldr_en,rouge2_fmeasure,0.01596233535299914
+wiki_lingua_en,4,median,rouge2_fmeasure,0.01596233535299914
+wiki_lingua_en,5,tldr_en,rouge2_fmeasure,0.0025848086028049965
+wiki_lingua_en,5,median,rouge2_fmeasure,0.0025848086028049965
+wiki_lingua_en,5,average,multiple,0.0353031945114879

8b7178b13b/evaluation/generation/merged.json

@@ -0,0 +1 @@
+{"GEM/web_nlg_en": {"0": {"PALM_prompt": {"bleu": 0.4373157998206226, "bleu_stderr": 0.056741486001354216, "rouge1_fmeasure": 0.10947380559420297, "rouge1_fmeasure_stderr": 0.0022511546576554292, "rouge1_precision": 0.07630675677312203, "rouge1_precision_stderr": 0.0021365872076980082, "rouge1_recall": 0.2947830429134024, "rouge1_recall_stderr": 0.0047665492114514684, "rouge2_fmeasure": 0.051001464150726095, "rouge2_fmeasure_stderr": 0.0014146940985046523, "rouge2_precision": 0.03437396040114199, "rouge2_precision_stderr": 0.0011646295650624351, "rouge2_recall": 0.13907462394894776, "rouge2_recall_stderr": 0.0031715068123442996, "rougeL_fmeasure": 0.10417993615217146, "rougeL_fmeasure_stderr": 0.002043414019393663, "rougeL_precision": 0.07227253003813615, "rougeL_precision_stderr": 0.001955466598501427, "rougeL_recall": 0.28336049631237586, "rougeL_recall_stderr": 0.004569573466829386, "rougeLsum_fmeasure": 0.10462419732455051, "rougeLsum_fmeasure_stderr": 0.002115527416565905, "rougeLsum_precision": 0.07290032072974349, "rougeLsum_precision_stderr": 0.002034641349821296, "rougeLsum_recall": 0.2825727583996548, "rougeLsum_recall_stderr": 0.004541613699421234}}, "1": {"PALM_prompt": {"bleu": 0.5816764130686213, "bleu_stderr": 0.054147923323820346, "rouge1_fmeasure": 0.11727270364668571, "rouge1_fmeasure_stderr": 0.0021476260743326565, "rouge1_precision": 0.07783086970109386, "rouge1_precision_stderr": 0.001926390801136041, "rouge1_recall": 0.36711372145643695, "rouge1_recall_stderr": 0.005207087210896133, "rouge2_fmeasure": 0.05455324665630359, "rouge2_fmeasure_stderr": 0.0013665856855669073, "rouge2_precision": 0.035941971570071736, "rouge2_precision_stderr": 0.0011116447277521812, "rouge2_recall": 0.17604536516651662, "rouge2_recall_stderr": 0.0036353372642678756, "rougeL_fmeasure": 0.10871994085882164, "rougeL_fmeasure_stderr": 0.0019153699814912425, "rougeL_precision": 0.07209000868611247, "rougeL_precision_stderr": 0.0017559014471748775, "rougeL_recall": 0.33901675935925657, "rougeL_recall_stderr": 0.004687996190515175, "rougeLsum_fmeasure": 0.11053867463797286, "rougeLsum_fmeasure_stderr": 0.0020173857832705686, "rougeLsum_precision": 0.0735230628158407, "rougeLsum_precision_stderr": 0.0018435905984185187, "rougeLsum_recall": 0.34433610068323095, "rougeLsum_recall_stderr": 0.004792623835429767}}, "2": {"PALM_prompt": {"bleu": 0.6801735381749798, "bleu_stderr": 0.04514863078336761, "rouge1_fmeasure": 0.11972585427910692, "rouge1_fmeasure_stderr": 0.001963983917319172, "rouge1_precision": 0.07666591908388082, "rouge1_precision_stderr": 0.0015425382902699712, "rouge1_recall": 0.40028519820196484, "rouge1_recall_stderr": 0.0053218199498182555, "rouge2_fmeasure": 0.055583861631680276, "rouge2_fmeasure_stderr": 0.0012687258801338563, "rouge2_precision": 0.03562073396147935, "rouge2_precision_stderr": 0.0009954366405729978, "rouge2_recall": 0.19724892000097954, "rouge2_recall_stderr": 0.004019172908229666, "rougeL_fmeasure": 0.10999138993564177, "rougeL_fmeasure_stderr": 0.0017648420950003858, "rougeL_precision": 0.07037367893965144, "rougeL_precision_stderr": 0.0013558489579657599, "rougeL_recall": 0.3648716666842121, "rougeL_recall_stderr": 0.004723267009132289, "rougeLsum_fmeasure": 0.1122905686653009, "rougeLsum_fmeasure_stderr": 0.0018425170251459826, "rougeLsum_precision": 0.07198504559682002, "rougeLsum_precision_stderr": 0.0014532243063151847, "rougeLsum_recall": 0.3737640227903558, "rougeLsum_recall_stderr": 0.004892612299725926}}, "3": {"PALM_prompt": {"bleu": 0.8393015493701177, "bleu_stderr": 0.049868510474836, "rouge1_fmeasure": 0.12332290364001403, "rouge1_fmeasure_stderr": 0.0020213255478620834, "rouge1_precision": 0.07934496701755703, "rouge1_precision_stderr": 0.0017108996646152186, "rouge1_recall": 0.41957311539245606, "rouge1_recall_stderr": 0.005383968460349908, "rouge2_fmeasure": 0.05769919253564089, "rouge2_fmeasure_stderr": 0.001315453385968732, "rouge2_precision": 0.03700401446806371, "rouge2_precision_stderr": 0.0010754080423489075, "rouge2_recall": 0.20785188427731804, "rouge2_recall_stderr": 0.0039850136524860235, "rougeL_fmeasure": 0.11171273283361995, "rougeL_fmeasure_stderr": 0.0017620828691461753, "rougeL_precision": 0.07186075331384528, "rougeL_precision_stderr": 0.0015065682717614625, "rougeL_recall": 0.3792930717112658, "rougeL_recall_stderr": 0.004756348746075166, "rougeLsum_fmeasure": 0.11543602475589927, "rougeLsum_fmeasure_stderr": 0.001888123216509075, "rougeLsum_precision": 0.07439776049493438, "rougeLsum_precision_stderr": 0.0016078356678010679, "rougeLsum_recall": 0.39147213524665503, "rougeLsum_recall_stderr": 0.004968164479358761}}, "4": {"PALM_prompt": {"bleu": 0.8436391155168642, "bleu_stderr": 0.049789928911136136, "rouge1_fmeasure": 0.12951808355992003, "rouge1_fmeasure_stderr": 0.0019403013558157668, "rouge1_precision": 0.08188000384427346, "rouge1_precision_stderr": 0.0015429926759057174, "rouge1_recall": 0.45225212783083296, "rouge1_recall_stderr": 0.005452056426418837, "rouge2_fmeasure": 0.06050302495157815, "rouge2_fmeasure_stderr": 0.0012369724150331927, "rouge2_precision": 0.03789712819393572, "rouge2_precision_stderr": 0.0009152136077591718, "rouge2_recall": 0.22689495723762804, "rouge2_recall_stderr": 0.004067188852506108, "rougeL_fmeasure": 0.11592286526905457, "rougeL_fmeasure_stderr": 0.0016906240203803347, "rougeL_precision": 0.07330729359090057, "rougeL_precision_stderr": 0.0013644419423480823, "rougeL_recall": 0.4045943171397325, "rougeL_recall_stderr": 0.0048320778663538955, "rougeLsum_fmeasure": 0.12060522562383363, "rougeLsum_fmeasure_stderr": 0.0018022149452879743, "rougeLsum_precision": 0.07637812548771711, "rougeLsum_precision_stderr": 0.0014540942217052375, "rougeLsum_recall": 0.4207633762253156, "rougeLsum_recall_stderr": 0.005005614020197891}}, "5": {"PALM_prompt": {"bleu": 0.968801645723095, "bleu_stderr": 0.05462868115593731, "rouge1_fmeasure": 0.13468892691608006, "rouge1_fmeasure_stderr": 0.0019240731337134673, "rouge1_precision": 0.08408541082259362, "rouge1_precision_stderr": 0.001415159971957011, "rouge1_recall": 0.4730441639084742, "rouge1_recall_stderr": 0.005431978662817184, "rouge2_fmeasure": 0.06252454636994373, "rouge2_fmeasure_stderr": 0.0012377179103821956, "rouge2_precision": 0.038803485654806714, "rouge2_precision_stderr": 0.0008788058278028121, "rouge2_recall": 0.2379631167113304, "rouge2_recall_stderr": 0.004152278672512853, "rougeL_fmeasure": 0.11896213494013362, "rougeL_fmeasure_stderr": 0.001650634135835398, "rougeL_precision": 0.07423644311431606, "rougeL_precision_stderr": 0.0012142160071926726, "rougeL_recall": 0.4194644730807224, "rougeL_recall_stderr": 0.0047799777603664715, "rougeLsum_fmeasure": 0.12479160570822044, "rougeLsum_fmeasure_stderr": 0.0017844920178026805, "rougeLsum_precision": 0.07800322392434518, "rougeLsum_precision_stderr": 0.0013202039315876929, "rougeLsum_recall": 0.43851085762444947, "rougeLsum_recall_stderr": 0.004968418620592047}}}, "GEM/wiki_lingua_en": {"0": {"tldr_en": {"bleu": 1.8310798418084737, "bleu_stderr": 0.06119305662626742, "rouge1_fmeasure": 0.19188833167040015, "rouge1_fmeasure_stderr": 0.0018702943944022418, "rouge1_precision": 0.1628352267958049, "rouge1_precision_stderr": 0.0019076289743005436, "rouge1_recall": 0.28137034403026634, "rouge1_recall_stderr": 0.002726139405286867, "rouge2_fmeasure": 0.04191366659253858, "rouge2_fmeasure_stderr": 0.000917658619128283, "rouge2_precision": 0.035168122195033014, "rouge2_precision_stderr": 0.0008051014260490183, "rouge2_recall": 0.06371619685684557, "rouge2_recall_stderr": 0.0015314676134498058, "rougeL_fmeasure": 0.14614932555210797, "rougeL_fmeasure_stderr": 0.0012956973040530264, "rougeL_precision": 0.12246239102421737, "rougeL_precision_stderr": 0.001285311648356421, "rougeL_recall": 0.2201424968967575, "rougeL_recall_stderr": 0.0022006617198564397, "rougeLsum_fmeasure": 0.17662543049894286, "rougeLsum_fmeasure_stderr": 0.001716560474974557, "rougeLsum_precision": 0.14960133985735732, "rougeLsum_precision_stderr": 0.0017445585328390875, "rougeLsum_recall": 0.2600296050390341, "rougeLsum_recall_stderr": 0.002552101400202146}}, "1": {"tldr_en": {"bleu": 2.463691563435929, "bleu_stderr": 0.056238669871763534, "rouge1_fmeasure": 0.20607218590152535, "rouge1_fmeasure_stderr": 0.0019348192676270253, "rouge1_precision": 0.17507251962841255, "rouge1_precision_stderr": 0.0019979193178610104, "rouge1_recall": 0.3008600636001003, "rouge1_recall_stderr": 0.002760929751246215, "rouge2_fmeasure": 0.04827166271383884, "rouge2_fmeasure_stderr": 0.000981638580890638, "rouge2_precision": 0.040814840621502924, "rouge2_precision_stderr": 0.0008769075552676463, "rouge2_recall": 0.07239534384858555, "rouge2_recall_stderr": 0.0015745819458817655, "rougeL_fmeasure": 0.14993882097099004, "rougeL_fmeasure_stderr": 0.0013143528095662098, "rougeL_precision": 0.12604513290190442, "rougeL_precision_stderr": 0.001331288983523961, "rougeL_recall": 0.22410236240350495, "rougeL_recall_stderr": 0.0021252698749924166, "rougeLsum_fmeasure": 0.19267274247494454, "rougeLsum_fmeasure_stderr": 0.0018042087124662393, "rougeLsum_precision": 0.1634506211607229, "rougeLsum_precision_stderr": 0.0018573519091366366, "rougeLsum_recall": 0.282240921650429, "rougeLsum_recall_stderr": 0.002619133620350951}}, "2": {"tldr_en": {"bleu": 3.013745331324814, "bleu_stderr": 0.06592229302797037, "rouge1_fmeasure": 0.22344837173188767, "rouge1_fmeasure_stderr": 0.0018933971382747686, "rouge1_precision": 0.19198287608229725, "rouge1_precision_stderr": 0.0021092410953077365, "rouge1_recall": 0.32589953688933776, "rouge1_recall_stderr": 0.002753157025629824, "rouge2_fmeasure": 0.05600464749019431, "rouge2_fmeasure_stderr": 0.001035304930307184, "rouge2_precision": 0.048137852198349124, "rouge2_precision_stderr": 0.0010124293758406351, "rouge2_recall": 0.08482162860321114, "rouge2_recall_stderr": 0.0017481381213950958, "rougeL_fmeasure": 0.15854970566001372, "rougeL_fmeasure_stderr": 0.0012774189115518982, "rougeL_precision": 0.13481580311964722, "rougeL_precision_stderr": 0.0014071194011638184, "rougeL_recall": 0.23733416551629913, "rougeL_recall_stderr": 0.00219502068666546, "rougeLsum_fmeasure": 0.21107970658856343, "rougeLsum_fmeasure_stderr": 0.0017866890110014018, "rougeLsum_precision": 0.18117711700554173, "rougeLsum_precision_stderr": 0.0019909544088953742, "rougeLsum_recall": 0.30882558318305736, "rougeLsum_recall_stderr": 0.0026454321668364913}}, "3": {"tldr_en": {"bleu": 2.986263870634216, "bleu_stderr": 0.0869488541072268, "rouge1_fmeasure": 0.1883894874399809, "rouge1_fmeasure_stderr": 0.0022518375274135087, "rouge1_precision": 0.16688833624223554, "rouge1_precision_stderr": 0.0024001395431129838, "rouge1_recall": 0.27404204769144364, "rouge1_recall_stderr": 0.003375962083519381, "rouge2_fmeasure": 0.04708204631655152, "rouge2_fmeasure_stderr": 0.0010069752068567668, "rouge2_precision": 0.04136653299690803, "rouge2_precision_stderr": 0.0010174651332227344, "rouge2_recall": 0.07172155304298297, "rouge2_recall_stderr": 0.0017383368313820028, "rougeL_fmeasure": 0.13391288059874437, "rougeL_fmeasure_stderr": 0.001564486351392238, "rougeL_precision": 0.11810699062222797, "rougeL_precision_stderr": 0.001714836740267086, "rougeL_recall": 0.19959382723616914, "rougeL_recall_stderr": 0.002605476332476844, "rougeLsum_fmeasure": 0.17834159419543702, "rougeLsum_fmeasure_stderr": 0.002133544799751138, "rougeLsum_precision": 0.15789252138792423, "rougeLsum_precision_stderr": 0.0022764600108588808, "rougeLsum_recall": 0.26023963410461465, "rougeLsum_recall_stderr": 0.003240567155894993}}, "4": {"tldr_en": {"bleu": 0.7653537900871676, "bleu_stderr": 0.049323102026813614, "rouge1_fmeasure": 0.06229352605061771, "rouge1_fmeasure_stderr": 0.002059973094043126, "rouge1_precision": 0.05571162128505982, "rouge1_precision_stderr": 0.0019790843122430768, "rouge1_recall": 0.09490155511879607, "rouge1_recall_stderr": 0.003216109108055619, "rouge2_fmeasure": 0.01596233535299914, "rouge2_fmeasure_stderr": 0.0007332643966253175, "rouge2_precision": 0.013746904154847127, "rouge2_precision_stderr": 0.000658207908545434, "rouge2_recall": 0.02605352352304377, "rouge2_recall_stderr": 0.0013317962968417952, "rougeL_fmeasure": 0.04502172063067562, "rougeL_fmeasure_stderr": 0.0014806563009305842, "rougeL_precision": 0.04028867827357096, "rougeL_precision_stderr": 0.0014354449068175981, "rougeL_recall": 0.07013160574213247, "rougeL_recall_stderr": 0.0024380518058086703, "rougeLsum_fmeasure": 0.05884434373920439, "rougeLsum_fmeasure_stderr": 0.0019486127272992522, "rougeLsum_precision": 0.0526446584524354, "rougeLsum_precision_stderr": 0.0018735818695187819, "rougeLsum_recall": 0.08977749314111799, "rougeLsum_recall_stderr": 0.0030541817569144}}, "5": {"tldr_en": {"bleu": 2.313122283850011e-06, "bleu_stderr": 4.438365983452618e-06, "rouge1_fmeasure": 0.010262809104423124, "rouge1_fmeasure_stderr": 0.000941433577191994, "rouge1_precision": 0.009989872401580602, "rouge1_precision_stderr": 0.0009952828000827124, "rouge1_recall": 0.015276156161555617, "rouge1_recall_stderr": 0.001444671711873694, "rouge2_fmeasure": 0.0025848086028049965, "rouge2_fmeasure_stderr": 0.0003301678198930521, "rouge2_precision": 0.0024167502513949756, "rouge2_precision_stderr": 0.00033798462070527304, "rouge2_recall": 0.004213844763908219, "rouge2_recall_stderr": 0.0006183240014712593, "rougeL_fmeasure": 0.007530769725575652, "rougeL_fmeasure_stderr": 0.0006918159097560924, "rougeL_precision": 0.007350732687047713, "rougeL_precision_stderr": 0.000736612195363616, "rougeL_recall": 0.011512337859684543, "rougeL_recall_stderr": 0.0011311946838375108, "rougeLsum_fmeasure": 0.009601062566883777, "rougeLsum_fmeasure_stderr": 0.0008823539930029628, "rougeLsum_precision": 0.009401823549618816, "rougeLsum_precision_stderr": 0.0009477935776177291, "rougeLsum_recall": 0.014304994804442096, "rougeLsum_recall_stderr": 0.0013529102153531946}}}, "e2e_nlg_cleaned": {"0": {"generate_text_restaurant": {"bleu": 0.3016960813220789, "bleu_stderr": 0.04082243698870125, "rouge1_fmeasure": 0.08446489060239885, "rouge1_fmeasure_stderr": 0.000832880994211257, "rouge1_precision": 0.0628362517057776, "rouge1_precision_stderr": 0.0006647851313912593, "rouge1_recall": 0.137799699754724, "rouge1_recall_stderr": 0.0013399199497499206, "rouge2_fmeasure": 0.013098749477153882, "rouge2_fmeasure_stderr": 0.00036393407871382073, "rouge2_precision": 0.009897778438587298, "rouge2_precision_stderr": 0.0002813167869752497, "rouge2_recall": 0.02030644827526888, "rouge2_recall_stderr": 0.0005626168002295672, "rougeL_fmeasure": 0.08185345358329085, "rougeL_fmeasure_stderr": 0.0007758077715250189, "rougeL_precision": 0.060894085509621707, "rougeL_precision_stderr": 0.0006165662598640652, "rougeL_recall": 0.1334600824136164, "rougeL_recall_stderr": 0.0012622226683087598, "rougeLsum_fmeasure": 0.07821933484300712, "rougeLsum_fmeasure_stderr": 0.000742258462877918, "rougeLsum_precision": 0.05822489106803366, "rougeLsum_precision_stderr": 0.00059732648786741, "rougeLsum_recall": 0.12759747263850774, "rougeLsum_recall_stderr": 0.0011920315451119745}}, "1": {"generate_text_restaurant": {"bleu": 9.014911506469076, "bleu_stderr": 0.10639404635916568, "rouge1_fmeasure": 0.4071860181343587, "rouge1_fmeasure_stderr": 0.002172401493926826, "rouge1_precision": 0.4220547738164344, "rouge1_precision_stderr": 0.0029747898853502135, "rouge1_recall": 0.44890682882262095, "rouge1_recall_stderr": 0.0030482256607117985, "rouge2_fmeasure": 0.17675735102863532, "rouge2_fmeasure_stderr": 0.0016983524897307275, "rouge2_precision": 0.18398717679998686, "rouge2_precision_stderr": 0.002070314538395242, "rouge2_recall": 0.19695708046780447, "rouge2_recall_stderr": 0.0021253468809232486, "rougeL_fmeasure": 0.2901955244768342, "rougeL_fmeasure_stderr": 0.0017445755551563752, "rougeL_precision": 0.301979337271323, "rougeL_precision_stderr": 0.002426170411020153, "rougeL_recall": 0.32120889976820555, "rougeL_recall_stderr": 0.0024556761259021593, "rougeLsum_fmeasure": 0.3366538075182426, "rougeLsum_fmeasure_stderr": 0.0020840116286440125, "rougeLsum_precision": 0.3498631569304881, "rougeLsum_precision_stderr": 0.00275693533807641, "rougeLsum_recall": 0.3704033862486897, "rougeLsum_recall_stderr": 0.00277359280896307}}, "2": {"generate_text_restaurant": {"bleu": 11.1355912496663, "bleu_stderr": 0.13104161087327487, "rouge1_fmeasure": 0.4449026725503146, "rouge1_fmeasure_stderr": 0.0020303864410380614, "rouge1_precision": 0.45419518183344554, "rouge1_precision_stderr": 0.0027266345121146297, "rouge1_recall": 0.48044140772752547, "rouge1_recall_stderr": 0.002817950150203246, "rouge2_fmeasure": 0.20791813280930055, "rouge2_fmeasure_stderr": 0.0017732074653438918, "rouge2_precision": 0.2124884501715128, "rouge2_precision_stderr": 0.0020421137596173576, "rouge2_recall": 0.22663601203825093, "rouge2_recall_stderr": 0.0021799430214164035, "rougeL_fmeasure": 0.32200623778436555, "rougeL_fmeasure_stderr": 0.0017788348099423016, "rougeL_precision": 0.32873823752244813, "rougeL_precision_stderr": 0.002262430136729236, "rougeL_recall": 0.34880236417169236, "rougeL_recall_stderr": 0.0024064753259342344, "rougeLsum_fmeasure": 0.37259609212766276, "rougeLsum_fmeasure_stderr": 0.0020305961255707534, "rougeLsum_precision": 0.38047010773438, "rougeLsum_precision_stderr": 0.002559842275120574, "rougeLsum_recall": 0.40224234667337105, "rougeLsum_recall_stderr": 0.002660545444016772}}, "3": {"generate_text_restaurant": {"bleu": 12.10470203084112, "bleu_stderr": 0.160895558301053, "rouge1_fmeasure": 0.4492382386896765, "rouge1_fmeasure_stderr": 0.0019699301602619833, "rouge1_precision": 0.45456971607102353, "rouge1_precision_stderr": 0.0023960763531178423, "rouge1_recall": 0.47953391372338766, "rouge1_recall_stderr": 0.0027865520920584822, "rouge2_fmeasure": 0.2138339925008178, "rouge2_fmeasure_stderr": 0.0018445417129634963, "rouge2_precision": 0.2163784140584202, "rouge2_precision_stderr": 0.001980036007401307, "rouge2_recall": 0.23028758666497792, "rouge2_recall_stderr": 0.0022756293438486903, "rougeL_fmeasure": 0.3303919839118561, "rougeL_fmeasure_stderr": 0.001783388701966232, "rougeL_precision": 0.33426681511660167, "rougeL_precision_stderr": 0.002069523940604892, "rougeL_recall": 0.35354783483337526, "rougeL_recall_stderr": 0.0024228991972543336, "rougeLsum_fmeasure": 0.378239342272828, "rougeLsum_fmeasure_stderr": 0.002043462221499771, "rougeLsum_precision": 0.3827360863233357, "rougeLsum_precision_stderr": 0.002359678766192712, "rougeLsum_recall": 0.40399025031746394, "rougeLsum_recall_stderr": 0.002710450542079631}}, "4": {"generate_text_restaurant": {"bleu": 12.44085622955195, "bleu_stderr": 0.13067375364343567, "rouge1_fmeasure": 0.4511875093908521, "rouge1_fmeasure_stderr": 0.001948885960105966, "rouge1_precision": 0.45479182860495637, "rouge1_precision_stderr": 0.0023107278825576553, "rouge1_recall": 0.48017725960351876, "rouge1_recall_stderr": 0.0027316537664281945, "rouge2_fmeasure": 0.21707781052843164, "rouge2_fmeasure_stderr": 0.00185164014349205, "rouge2_precision": 0.21833473257724795, "rouge2_precision_stderr": 0.0019502917664114363, "rouge2_recall": 0.23333710855434647, "rouge2_recall_stderr": 0.002269432186215575, "rougeL_fmeasure": 0.33434874026181494, "rougeL_fmeasure_stderr": 0.0017786642192651214, "rougeL_precision": 0.3366370287941819, "rougeL_precision_stderr": 0.0019965287497770777, "rougeL_recall": 0.35684028344910274, "rougeL_recall_stderr": 0.0024057773390550254, "rougeLsum_fmeasure": 0.38297835064075053, "rougeLsum_fmeasure_stderr": 0.0020438066934495387, "rougeLsum_precision": 0.38571523251054607, "rougeLsum_precision_stderr": 0.0022865314323742805, "rougeLsum_recall": 0.40803304829345277, "rougeLsum_recall_stderr": 0.0027038368009947755}}, "5": {"generate_text_restaurant": {"bleu": 12.352678257477072, "bleu_stderr": 0.1229858870702429, "rouge1_fmeasure": 0.4520495564262539, "rouge1_fmeasure_stderr": 0.0019091091728356577, "rouge1_precision": 0.4536679355694742, "rouge1_precision_stderr": 0.002282805690194143, "rouge1_recall": 0.48257839264210245, "rouge1_recall_stderr": 0.002694452362144022, "rouge2_fmeasure": 0.21807002700335165, "rouge2_fmeasure_stderr": 0.0018243298667958761, "rouge2_precision": 0.2184937204533077, "rouge2_precision_stderr": 0.0019084340007320115, "rouge2_recall": 0.2348008779716612, "rouge2_recall_stderr": 0.0022303214323899125, "rougeL_fmeasure": 0.336313193439231, "rougeL_fmeasure_stderr": 0.001791714678974818, "rougeL_precision": 0.33692930278278704, "rougeL_precision_stderr": 0.0019923203298548866, "rougeL_recall": 0.36002100624003847, "rougeL_recall_stderr": 0.0024053149539646364, "rougeLsum_fmeasure": 0.38490335194628533, "rougeLsum_fmeasure_stderr": 0.002018489316229454, "rougeLsum_precision": 0.38599958036488224, "rougeLsum_precision_stderr": 0.002272531904720447, "rougeLsum_recall": 0.4113378384062774, "rougeLsum_recall_stderr": 0.002667546056299814}}}, "gem_xsum": {"0": {"article_DOC_summary": {"bleu": 2.194933977793404, "bleu_stderr": 0.09650169688197711, "rouge1_fmeasure": 0.2157374708631877, "rouge1_fmeasure_stderr": 0.002508783327476019, "rouge1_precision": 0.15954668519186138, "rouge1_precision_stderr": 0.002100617390256336, "rouge1_recall": 0.36309739763041776, "rouge1_recall_stderr": 0.004368615445871372, "rouge2_fmeasure": 0.05279448511938101, "rouge2_fmeasure_stderr": 0.001664044690700437, "rouge2_precision": 0.03857312620535916, "rouge2_precision_stderr": 0.0012905906114606838, "rouge2_recall": 0.09177566784631346, "rouge2_recall_stderr": 0.0029866334470645636, "rougeL_fmeasure": 0.1604319788556621, "rougeL_fmeasure_stderr": 0.0019556284658093183, "rougeL_precision": 0.11839547499241576, "rougeL_precision_stderr": 0.001646949604960197, "rougeL_recall": 0.27198898577078495, "rougeL_recall_stderr": 0.0035222426961979128, "rougeLsum_fmeasure": 0.1697976651408494, "rougeLsum_fmeasure_stderr": 0.002135233558113895, "rougeLsum_precision": 0.1250746587410693, "rougeLsum_precision_stderr": 0.0017464873888176696, "rougeLsum_recall": 0.28828909491111, "rougeLsum_recall_stderr": 0.0038785736556512756}}, "1": {"article_DOC_summary": {"bleu": 1.638275312642142, "bleu_stderr": 0.11197811159644402, "rouge1_fmeasure": 0.18261422772647087, "rouge1_fmeasure_stderr": 0.00258517997013667, "rouge1_precision": 0.130085376048313, "rouge1_precision_stderr": 0.0019273226219467348, "rouge1_recall": 0.31899443410148204, "rouge1_recall_stderr": 0.004375864955378675, "rouge2_fmeasure": 0.03829439148207006, "rouge2_fmeasure_stderr": 0.0015268296315399182, "rouge2_precision": 0.027035766317831913, "rouge2_precision_stderr": 0.0010834329723279196, "rouge2_recall": 0.06843246331973844, "rouge2_recall_stderr": 0.002783755254166952, "rougeL_fmeasure": 0.13872448607191418, "rougeL_fmeasure_stderr": 0.0019214547149178605, "rougeL_precision": 0.09860861520075329, "rougeL_precision_stderr": 0.0014180838773530536, "rougeL_recall": 0.2439616017337858, "rougeL_recall_stderr": 0.003388978319546548, "rougeLsum_fmeasure": 0.14673205236069722, "rougeLsum_fmeasure_stderr": 0.002156691093638724, "rougeLsum_precision": 0.10431817474650688, "rougeLsum_precision_stderr": 0.0015880489605249845, "rougeLsum_recall": 0.25778306821268704, "rougeLsum_recall_stderr": 0.0037812899736781526}}, "2": {"article_DOC_summary": {"bleu": 2.0210914476360786, "bleu_stderr": 0.12454216962651855, "rouge1_fmeasure": 0.1978001315070318, "rouge1_fmeasure_stderr": 0.002669936318830053, "rouge1_precision": 0.14059986141007405, "rouge1_precision_stderr": 0.001980010991959053, "rouge1_recall": 0.34704590818514364, "rouge1_recall_stderr": 0.004609783640663682, "rouge2_fmeasure": 0.04760922765448965, "rouge2_fmeasure_stderr": 0.001702755511678876, "rouge2_precision": 0.033448694107769246, "rouge2_precision_stderr": 0.0011984145262489562, "rouge2_recall": 0.08605872654054372, "rouge2_recall_stderr": 0.003191665944600775, "rougeL_fmeasure": 0.1511855886298728, "rougeL_fmeasure_stderr": 0.0020261890172370246, "rougeL_precision": 0.10721276455620249, "rougeL_precision_stderr": 0.0014796729903083395, "rougeL_recall": 0.26727650609085896, "rougeL_recall_stderr": 0.0036995069904560843, "rougeLsum_fmeasure": 0.16004925294103853, "rougeLsum_fmeasure_stderr": 0.0022364087298009884, "rougeLsum_precision": 0.11346372550782761, "rougeLsum_precision_stderr": 0.0016303825401644061, "rougeLsum_recall": 0.28288787023938955, "rougeLsum_recall_stderr": 0.004038388892533831}}, "3": {"article_DOC_summary": {"bleu": 2.03791695157546, "bleu_stderr": 0.1342348737065576, "rouge1_fmeasure": 0.19482488018157684, "rouge1_fmeasure_stderr": 0.0028449440307477085, "rouge1_precision": 0.14077378985561326, "rouge1_precision_stderr": 0.002167627071465062, "rouge1_recall": 0.3365881713882994, "rouge1_recall_stderr": 0.004904093832075129, "rouge2_fmeasure": 0.04702347182983231, "rouge2_fmeasure_stderr": 0.001623530160402657, "rouge2_precision": 0.03346907677060051, "rouge2_precision_stderr": 0.00115993469618386, "rouge2_recall": 0.08385926778368435, "rouge2_recall_stderr": 0.0030304591639245464, "rougeL_fmeasure": 0.14665273300314152, "rougeL_fmeasure_stderr": 0.00212580713201847, "rougeL_precision": 0.10579758272401656, "rougeL_precision_stderr": 0.0016080916023179224, "rougeL_recall": 0.25509688514423123, "rougeL_recall_stderr": 0.003826588022233898, "rougeLsum_fmeasure": 0.15621401458929957, "rougeLsum_fmeasure_stderr": 0.00234420229722605, "rougeLsum_precision": 0.11263878026618175, "rougeLsum_precision_stderr": 0.00176128619109869, "rougeLsum_recall": 0.2715816804884895, "rougeLsum_recall_stderr": 0.004199504898315826}}, "4": {"article_DOC_summary": {"bleu": 0.9907623400783704, "bleu_stderr": 0.12144441541342028, "rouge1_fmeasure": 0.05341124486563334, "rouge1_fmeasure_stderr": 0.0029118807193791816, "rouge1_precision": 0.04332722979741195, "rouge1_precision_stderr": 0.0025397484135024503, "rouge1_recall": 0.08638484320409213, "rouge1_recall_stderr": 0.004855712915740289, "rouge2_fmeasure": 0.0126600494371876, "rouge2_fmeasure_stderr": 0.0010393927624132967, "rouge2_precision": 0.009371982127377696, "rouge2_precision_stderr": 0.0007761604812612375, "rouge2_recall": 0.02197790550576286, "rouge2_recall_stderr": 0.0018737167719494205, "rougeL_fmeasure": 0.04031062592059575, "rougeL_fmeasure_stderr": 0.0022065055222514843, "rougeL_precision": 0.03308111037957417, "rougeL_precision_stderr": 0.002037483672167313, "rougeL_recall": 0.06547067873377617, "rougeL_recall_stderr": 0.0037402890017668913, "rougeLsum_fmeasure": 0.04344922023460101, "rougeLsum_fmeasure_stderr": 0.002384615926685988, "rougeLsum_precision": 0.03547102946933422, "rougeLsum_precision_stderr": 0.002145817702406172, "rougeLsum_recall": 0.07066149952031933, "rougeLsum_recall_stderr": 0.004048133700252855}}, "5": {"article_DOC_summary": {"bleu": 1.959625061740418e-19, "bleu_stderr": 1.1984703371139495e-15, "rouge1_fmeasure": 0.0026267053439145405, "rouge1_fmeasure_stderr": 0.0006851925438686376, "rouge1_precision": 0.002432491256372859, "rouge1_precision_stderr": 0.0006531802890059347, "rouge1_recall": 0.0035688165030904754, "rouge1_recall_stderr": 0.000969634171191206, "rouge2_fmeasure": 0.0003000575858304317, "rouge2_fmeasure_stderr": 0.00012455541465929192, "rouge2_precision": 0.0002462802129838755, "rouge2_precision_stderr": 0.00010248150275091556, "rouge2_recall": 0.00045873937383371344, "rouge2_recall_stderr": 0.0002027676865696476, "rougeL_fmeasure": 0.001963800127619678, "rougeL_fmeasure_stderr": 0.0005077215919641312, "rougeL_precision": 0.001824459823195743, "rougeL_precision_stderr": 0.000493813415204384, "rougeL_recall": 0.002706254470313614, "rougeL_recall_stderr": 0.0007319861753860738, "rougeLsum_fmeasure": 0.002094899399917077, "rougeLsum_fmeasure_stderr": 0.0005500941719652507, "rougeLsum_precision": 0.001947339725356528, "rougeLsum_precision_stderr": 0.0005292516418451348, "rougeLsum_recall": 0.0028418243363945105, "rougeLsum_recall_stderr": 0.0007704086076680828}}}}

8b7178b13b/evaluation/generation/slim.8b7178b13b_GEM-wiki_lingua_en_tldr_en_2.json

@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_precision": 0.19198287608229725,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0021092410953077365
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_recall": 0.32589953688933776,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.002753157025629824
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_fmeasure": 0.22344837173188767,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0018933971382747686
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_precision": 0.048137852198349124,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0010124293758406351
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_recall": 0.08482162860321114,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0017481381213950958
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_fmeasure": 0.05600464749019431,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.001035304930307184
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_precision": 0.13481580311964722,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0014071194011638184
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_recall": 0.23733416551629913,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.00219502068666546
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_fmeasure": 0.15854970566001372,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0012774189115518982
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_precision": 0.18117711700554173,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0019909544088953742
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_recall": 0.30882558318305736,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0026454321668364913
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_fmeasure": 0.21107970658856343,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0017866890110014018
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "bleu": 3.013745331324814,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.06592229302797037
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b13b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 8,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}

8b7178b13b/evaluation/generation/slim.8b7178b13b_GEM-wiki_lingua_en_tldr_en_3.json

@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_precision": 0.16688833624223554,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0024001395431129838
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_recall": 0.27404204769144364,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.003375962083519381
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_fmeasure": 0.1883894874399809,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0022518375274135087
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_precision": 0.04136653299690803,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0010174651332227344
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_recall": 0.07172155304298297,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0017383368313820028
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_fmeasure": 0.04708204631655152,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0010069752068567668
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_precision": 0.11810699062222797,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.001714836740267086
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_recall": 0.19959382723616914,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.002605476332476844
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_fmeasure": 0.13391288059874437,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.001564486351392238
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_precision": 0.15789252138792423,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0022764600108588808
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_recall": 0.26023963410461465,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.003240567155894993
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_fmeasure": 0.17834159419543702,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.002133544799751138
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "bleu": 2.986263870634216,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.0869488541072268
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b13b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 8,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}

8b7178b13b/evaluation/generation/slim.8b7178b13b_GEM-wiki_lingua_en_tldr_en_4.json

@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_precision": 0.05571162128505982,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0019790843122430768
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_recall": 0.09490155511879607,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.003216109108055619
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_fmeasure": 0.06229352605061771,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.002059973094043126
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_precision": 0.013746904154847127,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.000658207908545434
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_recall": 0.02605352352304377,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0013317962968417952
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_fmeasure": 0.01596233535299914,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0007332643966253175
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_precision": 0.04028867827357096,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0014354449068175981
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_recall": 0.07013160574213247,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0024380518058086703
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_fmeasure": 0.04502172063067562,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0014806563009305842
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_precision": 0.0526446584524354,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0018735818695187819
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_recall": 0.08977749314111799,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0030541817569144
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_fmeasure": 0.05884434373920439,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0019486127272992522
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "bleu": 0.7653537900871676,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.049323102026813614
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b13b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 8,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}

8b7178b13b/evaluation/generation/slim.8b7178b13b_GEM-wiki_lingua_en_tldr_en_5.json

@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_precision": 0.009989872401580602,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0009952828000827124
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_recall": 0.015276156161555617,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.001444671711873694
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_fmeasure": 0.010262809104423124,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.000941433577191994
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_precision": 0.0024167502513949756,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.00033798462070527304
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_recall": 0.004213844763908219,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0006183240014712593
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_fmeasure": 0.0025848086028049965,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0003301678198930521
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_precision": 0.007350732687047713,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.000736612195363616
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_recall": 0.011512337859684543,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0011311946838375108
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_fmeasure": 0.007530769725575652,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0006918159097560924
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_precision": 0.009401823549618816,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0009477935776177291
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_recall": 0.014304994804442096,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0013529102153531946
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_fmeasure": 0.009601062566883777,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0008823539930029628
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "bleu": 2.313122283850011e-06,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 4.438365983452618e-06
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b13b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 8,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}

8b7178b13b/evaluation/generation/slim.8b7178b13b_gem_xsum_article_DOC_summary_3.json

@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_precision": 0.14077378985561326,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.002167627071465062
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_recall": 0.3365881713882994,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.004904093832075129
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_fmeasure": 0.19482488018157684,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.0028449440307477085
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_precision": 0.03346907677060051,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.00115993469618386
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_recall": 0.08385926778368435,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0030304591639245464
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_fmeasure": 0.04702347182983231,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.001623530160402657
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_precision": 0.10579758272401656,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0016080916023179224
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_recall": 0.25509688514423123,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.003826588022233898
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_fmeasure": 0.14665273300314152,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.00212580713201847
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_precision": 0.11263878026618175,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.00176128619109869
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_recall": 0.2715816804884895,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.004199504898315826
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_fmeasure": 0.15621401458929957,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.00234420229722605
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "bleu": 2.03791695157546,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.1342348737065576
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b13b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 8,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}

8b7178b13b/evaluation/generation/slim.8b7178b13b_gem_xsum_article_DOC_summary_4.json

@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_precision": 0.04332722979741195,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0025397484135024503
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_recall": 0.08638484320409213,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.004855712915740289
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_fmeasure": 0.05341124486563334,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.0029118807193791816
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_precision": 0.009371982127377696,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0007761604812612375
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_recall": 0.02197790550576286,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0018737167719494205
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_fmeasure": 0.0126600494371876,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0010393927624132967
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_precision": 0.03308111037957417,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.002037483672167313
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_recall": 0.06547067873377617,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.0037402890017668913
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_fmeasure": 0.04031062592059575,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0022065055222514843
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_precision": 0.03547102946933422,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.002145817702406172
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_recall": 0.07066149952031933,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.004048133700252855
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_fmeasure": 0.04344922023460101,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.002384615926685988
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "bleu": 0.9907623400783704,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.12144441541342028
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b13b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 8,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}

8b7178b13b/evaluation/generation/slim.8b7178b13b_gem_xsum_article_DOC_summary_5.json

@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_precision": 0.002432491256372859,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0006531802890059347
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_recall": 0.0035688165030904754,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.000969634171191206
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_fmeasure": 0.0026267053439145405,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.0006851925438686376
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_precision": 0.0002462802129838755,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.00010248150275091556
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_recall": 0.00045873937383371344,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0002027676865696476
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_fmeasure": 0.0003000575858304317,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.00012455541465929192
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_precision": 0.001824459823195743,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.000493813415204384
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_recall": 0.002706254470313614,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.0007319861753860738
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_fmeasure": 0.001963800127619678,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0005077215919641312
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_precision": 0.001947339725356528,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0005292516418451348
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_recall": 0.0028418243363945105,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.0007704086076680828
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_fmeasure": 0.002094899399917077,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0005500941719652507
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "bleu": 1.959625061740418e-19,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 1.1984703371139495e-15
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b13b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 8,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}

8b7178b13b/evaluation/rankeval/8b7178b13b_0.csv

@@ -0,0 +1,21 @@
+task,metric,value,err,version
+anli_r1,acc,0.35,0.01509065034144423,0
+anli_r2,acc,0.346,0.015050266127564448,0
+anli_r3,acc,0.355,0.013819249004047296,0
+arc_challenge,acc,0.29180887372013653,0.013284525292403496,0
+arc_challenge,acc_norm,0.3216723549488055,0.013650488084494162,0
+arc_easy,acc,0.6220538720538721,0.009949405744045469,0
+arc_easy,acc_norm,0.5395622895622896,0.010227616386289017,0
+boolq,acc,0.6376146788990825,0.008407308655864048,1
+cb,acc,0.26785714285714285,0.05971290310957636,1
+cb,f1,0.2374338624338624,,1
+copa,acc,0.81,0.03942772444036623,0
+hellaswag,acc,0.5215096594303924,0.004985162074336112,0
+hellaswag,acc_norm,0.6843258315076678,0.004638339207348913,0
+piqa,acc,0.7627856365614799,0.009924694933586364,0
+piqa,acc_norm,0.7747551686615887,0.00974664347103214,0
+rte,acc,0.5270758122743683,0.030052303463143706,0
+sciq,acc,0.87,0.010640169792499361,0
+sciq,acc_norm,0.807,0.012486268734370145,0
+storycloze_2016,acc,0.7455905932656334,0.010071542492663043,0
+winogrande,acc,0.5659037095501184,0.013929882555694054,0

8b7178b13b/evaluation/rankeval/8b7178b13b_0_lm-eval_global_step84877_2023-05-15-10-07-19_0shots_backup.json

@@ -1,87 +0,0 @@
-{
-    "results": {
-        "anli_r1": {
-            "acc": 0.35,
-            "acc_stderr": 0.01509065034144423
-        },
-        "anli_r2": {
-            "acc": 0.346,
-            "acc_stderr": 0.015050266127564448
-        },
-        "anli_r3": {
-            "acc": 0.355,
-            "acc_stderr": 0.013819249004047296
-        },
-        "cb": {
-            "acc": 0.26785714285714285,
-            "acc_stderr": 0.05971290310957636,
-            "f1": 0.2374338624338624
-        },
-        "copa": {
-            "acc": 0.81,
-            "acc_stderr": 0.03942772444036623
-        },
-        "hellaswag": {
-            "acc": 0.5215096594303924,
-            "acc_stderr": 0.004985162074336112,
-            "acc_norm": 0.6843258315076678,
-            "acc_norm_stderr": 0.004638339207348913
-        },
-        "rte": {
-            "acc": 0.5270758122743683,
-            "acc_stderr": 0.030052303463143706
-        },
-        "winogrande": {
-            "acc": 0.5659037095501184,
-            "acc_stderr": 0.013929882555694054
-        },
-        "storycloze_2016": {
-            "acc": 0.7455905932656334,
-            "acc_stderr": 0.010071542492663043
-        },
-        "boolq": {
-            "acc": 0.6376146788990825,
-            "acc_stderr": 0.008407308655864048
-        },
-        "arc_easy": {
-            "acc": 0.6220538720538721,
-            "acc_stderr": 0.009949405744045469,
-            "acc_norm": 0.5395622895622896,
-            "acc_norm_stderr": 0.010227616386289017
-        },
-        "arc_challenge": {
-            "acc": 0.29180887372013653,
-            "acc_stderr": 0.013284525292403496,
-            "acc_norm": 0.3216723549488055,
-            "acc_norm_stderr": 0.013650488084494162
-        },
-        "sciq": {
-            "acc": 0.87,
-            "acc_stderr": 0.010640169792499361,
-            "acc_norm": 0.807,
-            "acc_norm_stderr": 0.012486268734370145
-        },
-        "piqa": {
-            "acc": 0.7627856365614799,
-            "acc_stderr": 0.009924694933586364,
-            "acc_norm": 0.7747551686615887,
-            "acc_norm_stderr": 0.00974664347103214
-        }
-    },
-    "versions": {
-        "anli_r1": 0,
-        "anli_r2": 0,
-        "anli_r3": 0,
-        "cb": 1,
-        "copa": 0,
-        "hellaswag": 0,
-        "rte": 0,
-        "winogrande": 0,
-        "storycloze_2016": 0,
-        "boolq": 1,
-        "arc_easy": 0,
-        "arc_challenge": 0,
-        "sciq": 0,
-        "piqa": 0
-    }
-}

8b7178b13b/evaluation/rankeval/8b7178b13b_1.csv

@@ -0,0 +1,21 @@
+task,metric,value,err,version
+anli_r1,acc,0.328,0.014853842487270334,0
+anli_r2,acc,0.316,0.01470919305605713,0
+anli_r3,acc,0.3591666666666667,0.013855141559780364,0
+arc_challenge,acc,0.3054607508532423,0.0134600804780025,0
+arc_challenge,acc_norm,0.3319112627986348,0.013760988200880538,0
+arc_easy,acc,0.6422558922558923,0.009835772757343361,0
+arc_easy,acc_norm,0.6035353535353535,0.010037412763064529,0
+boolq,acc,0.6477064220183486,0.00835476049390613,1
+cb,acc,0.26785714285714285,0.05971290310957635,1
+cb,f1,0.21294539321104786,,1
+copa,acc,0.79,0.040936018074033256,0
+hellaswag,acc,0.5201155148376817,0.004985741706385719,0
+hellaswag,acc_norm,0.6825333598884684,0.004645393477680675,0
+piqa,acc,0.764961915125136,0.009893146688805326,0
+piqa,acc_norm,0.7725788900979326,0.009779850767847232,0
+rte,acc,0.4657039711191336,0.030025579819366426,0
+sciq,acc,0.905,0.009276910103103329,0
+sciq,acc_norm,0.88,0.0102813280127474,0
+storycloze_2016,acc,0.7365045430251203,0.010187168219156485,0
+winogrande,acc,0.5872138910812944,0.013837060648682103,0

8b7178b13b/evaluation/rankeval/8b7178b13b_1_lm-eval_global_step84877_2023-05-15-10-07-19_1shots_backup.json

@@ -1,87 +0,0 @@
-{
-    "results": {
-        "anli_r1": {
-            "acc": 0.328,
-            "acc_stderr": 0.014853842487270334
-        },
-        "anli_r2": {
-            "acc": 0.316,
-            "acc_stderr": 0.01470919305605713
-        },
-        "anli_r3": {
-            "acc": 0.3591666666666667,
-            "acc_stderr": 0.013855141559780364
-        },
-        "cb": {
-            "acc": 0.26785714285714285,
-            "acc_stderr": 0.05971290310957635,
-            "f1": 0.21294539321104786
-        },
-        "copa": {
-            "acc": 0.79,
-            "acc_stderr": 0.040936018074033256
-        },
-        "hellaswag": {
-            "acc": 0.5201155148376817,
-            "acc_stderr": 0.004985741706385719,
-            "acc_norm": 0.6825333598884684,
-            "acc_norm_stderr": 0.004645393477680675
-        },
-        "rte": {
-            "acc": 0.4657039711191336,
-            "acc_stderr": 0.030025579819366426
-        },
-        "winogrande": {
-            "acc": 0.5872138910812944,
-            "acc_stderr": 0.013837060648682103
-        },
-        "storycloze_2016": {
-            "acc": 0.7365045430251203,
-            "acc_stderr": 0.010187168219156485
-        },
-        "boolq": {
-            "acc": 0.6477064220183486,
-            "acc_stderr": 0.00835476049390613
-        },
-        "arc_easy": {
-            "acc": 0.6422558922558923,
-            "acc_stderr": 0.009835772757343361,
-            "acc_norm": 0.6035353535353535,
-            "acc_norm_stderr": 0.010037412763064529
-        },
-        "arc_challenge": {
-            "acc": 0.3054607508532423,
-            "acc_stderr": 0.0134600804780025,
-            "acc_norm": 0.3319112627986348,
-            "acc_norm_stderr": 0.013760988200880538
-        },
-        "sciq": {
-            "acc": 0.905,
-            "acc_stderr": 0.009276910103103329,
-            "acc_norm": 0.88,
-            "acc_norm_stderr": 0.0102813280127474
-        },
-        "piqa": {
-            "acc": 0.764961915125136,
-            "acc_stderr": 0.009893146688805326,
-            "acc_norm": 0.7725788900979326,
-            "acc_norm_stderr": 0.009779850767847232
-        }
-    },
-    "versions": {
-        "anli_r1": 0,
-        "anli_r2": 0,
-        "anli_r3": 0,
-        "cb": 1,
-        "copa": 0,
-        "hellaswag": 0,
-        "rte": 0,
-        "winogrande": 0,
-        "storycloze_2016": 0,
-        "boolq": 1,
-        "arc_easy": 0,
-        "arc_challenge": 0,
-        "sciq": 0,
-        "piqa": 0
-    }
-}

8b7178b13b/evaluation/rankeval/8b7178b13b_2.csv

@@ -0,0 +1,21 @@
+task,metric,value,err,version
+anli_r1,acc,0.336,0.014944140233795018,0
+anli_r2,acc,0.329,0.01486539538592835,0
+anli_r3,acc,0.35083333333333333,0.013782212417178195,0
+arc_challenge,acc,0.310580204778157,0.013522292098053059,0
+arc_challenge,acc_norm,0.33532423208191126,0.013796182947785562,0
+arc_easy,acc,0.6481481481481481,0.009799078929868706,0
+arc_easy,acc_norm,0.6212121212121212,0.00995373765654204,0
+boolq,acc,0.6382262996941896,0.008404238796949254,1
+cb,acc,0.19642857142857142,0.05357142857142858,1
+cb,f1,0.1668300653594771,,1
+copa,acc,0.78,0.04163331998932263,0
+hellaswag,acc,0.5200159330810595,0.004985781620467012,0
+hellaswag,acc_norm,0.6863174666401115,0.004630407476835209,0
+piqa,acc,0.7573449401523396,0.0100020025697087,0
+piqa,acc_norm,0.764961915125136,0.009893146688805319,0
+rte,acc,0.48014440433212996,0.0300727231673172,0
+sciq,acc,0.913,0.00891686663074591,0
+sciq,acc_norm,0.889,0.009938701010583726,0
+storycloze_2016,acc,0.743452699091395,0.01009926092771917,0
+winogrande,acc,0.5864246250986582,0.013840971763195306,0

8b7178b13b/evaluation/rankeval/8b7178b13b_2_lm-eval_global_step84877_2023-05-15-10-07-19_2shots_backup.json

@@ -1,87 +0,0 @@
-{
-    "results": {
-        "anli_r1": {
-            "acc": 0.336,
-            "acc_stderr": 0.014944140233795018
-        },
-        "anli_r2": {
-            "acc": 0.329,
-            "acc_stderr": 0.01486539538592835
-        },
-        "anli_r3": {
-            "acc": 0.35083333333333333,
-            "acc_stderr": 0.013782212417178195
-        },
-        "cb": {
-            "acc": 0.19642857142857142,
-            "acc_stderr": 0.05357142857142858,
-            "f1": 0.1668300653594771
-        },
-        "copa": {
-            "acc": 0.78,
-            "acc_stderr": 0.04163331998932263
-        },
-        "hellaswag": {
-            "acc": 0.5200159330810595,
-            "acc_stderr": 0.004985781620467012,
-            "acc_norm": 0.6863174666401115,
-            "acc_norm_stderr": 0.004630407476835209
-        },
-        "rte": {
-            "acc": 0.48014440433212996,
-            "acc_stderr": 0.0300727231673172
-        },
-        "winogrande": {
-            "acc": 0.5864246250986582,
-            "acc_stderr": 0.013840971763195306
-        },
-        "storycloze_2016": {
-            "acc": 0.743452699091395,
-            "acc_stderr": 0.01009926092771917
-        },
-        "boolq": {
-            "acc": 0.6382262996941896,
-            "acc_stderr": 0.008404238796949254
-        },
-        "arc_easy": {
-            "acc": 0.6481481481481481,
-            "acc_stderr": 0.009799078929868706,
-            "acc_norm": 0.6212121212121212,
-            "acc_norm_stderr": 0.00995373765654204
-        },
-        "arc_challenge": {
-            "acc": 0.310580204778157,
-            "acc_stderr": 0.013522292098053059,
-            "acc_norm": 0.33532423208191126,
-            "acc_norm_stderr": 0.013796182947785562
-        },
-        "sciq": {
-            "acc": 0.913,
-            "acc_stderr": 0.00891686663074591,
-            "acc_norm": 0.889,
-            "acc_norm_stderr": 0.009938701010583726
-        },
-        "piqa": {
-            "acc": 0.7573449401523396,
-            "acc_stderr": 0.0100020025697087,
-            "acc_norm": 0.764961915125136,
-            "acc_norm_stderr": 0.009893146688805319
-        }
-    },
-    "versions": {
-        "anli_r1": 0,
-        "anli_r2": 0,
-        "anli_r3": 0,
-        "cb": 1,
-        "copa": 0,
-        "hellaswag": 0,
-        "rte": 0,
-        "winogrande": 0,
-        "storycloze_2016": 0,
-        "boolq": 1,
-        "arc_easy": 0,
-        "arc_challenge": 0,
-        "sciq": 0,
-        "piqa": 0
-    }
-}

8b7178b13b/evaluation/rankeval/8b7178b13b_3.csv

@@ -0,0 +1,21 @@
+task,metric,value,err,version
+anli_r1,acc,0.348,0.01507060460376841,0
+anli_r2,acc,0.344,0.015029633724408947,0
+anli_r3,acc,0.3325,0.013605417345710528,0
+arc_challenge,acc,0.3148464163822526,0.01357265770308495,0
+arc_challenge,acc_norm,0.3250853242320819,0.013688147309729119,0
+arc_easy,acc,0.648989898989899,0.009793703885101042,0
+arc_easy,acc_norm,0.6393097643097643,0.009853512108416748,0
+boolq,acc,0.6363914373088685,0.008413404209789989,1
+cb,acc,0.3392857142857143,0.06384226561930825,1
+cb,f1,0.3185837135128588,,1
+copa,acc,0.78,0.04163331998932263,0
+hellaswag,acc,0.5206134236207927,0.00498553915978342,0
+hellaswag,acc_norm,0.6902011551483768,0.00461465517501001,0
+piqa,acc,0.7584330794341676,0.009986718001804467,0
+piqa,acc_norm,0.7665941240478781,0.009869247889520991,0
+rte,acc,0.48014440433212996,0.0300727231673172,0
+sciq,acc,0.917,0.008728527206074792,0
+sciq,acc_norm,0.911,0.009008893392651521,0
+storycloze_2016,acc,0.7504008551576697,0.010008002459430848,0
+winogrande,acc,0.601420678768745,0.01376035717687383,0

8b7178b13b/evaluation/rankeval/8b7178b13b_3_lm-eval_global_step84877_2023-05-15-10-07-25_3shots_backup.json

@@ -1,87 +0,0 @@
-{
-    "results": {
-        "anli_r1": {
-            "acc": 0.348,
-            "acc_stderr": 0.01507060460376841
-        },
-        "anli_r2": {
-            "acc": 0.344,
-            "acc_stderr": 0.015029633724408947
-        },
-        "anli_r3": {
-            "acc": 0.3325,
-            "acc_stderr": 0.013605417345710528
-        },
-        "cb": {
-            "acc": 0.3392857142857143,
-            "acc_stderr": 0.06384226561930825,
-            "f1": 0.3185837135128588
-        },
-        "copa": {
-            "acc": 0.78,
-            "acc_stderr": 0.04163331998932263
-        },
-        "hellaswag": {
-            "acc": 0.5206134236207927,
-            "acc_stderr": 0.00498553915978342,
-            "acc_norm": 0.6902011551483768,
-            "acc_norm_stderr": 0.00461465517501001
-        },
-        "rte": {
-            "acc": 0.48014440433212996,
-            "acc_stderr": 0.0300727231673172
-        },
-        "winogrande": {
-            "acc": 0.601420678768745,
-            "acc_stderr": 0.01376035717687383
-        },
-        "storycloze_2016": {
-            "acc": 0.7504008551576697,
-            "acc_stderr": 0.010008002459430848
-        },
-        "boolq": {
-            "acc": 0.6363914373088685,
-            "acc_stderr": 0.008413404209789989
-        },
-        "arc_easy": {
-            "acc": 0.648989898989899,
-            "acc_stderr": 0.009793703885101042,
-            "acc_norm": 0.6393097643097643,
-            "acc_norm_stderr": 0.009853512108416748
-        },
-        "arc_challenge": {
-            "acc": 0.3148464163822526,
-            "acc_stderr": 0.01357265770308495,
-            "acc_norm": 0.3250853242320819,
-            "acc_norm_stderr": 0.013688147309729119
-        },
-        "sciq": {
-            "acc": 0.917,
-            "acc_stderr": 0.008728527206074792,
-            "acc_norm": 0.911,
-            "acc_norm_stderr": 0.009008893392651521
-        },
-        "piqa": {
-            "acc": 0.7584330794341676,
-            "acc_stderr": 0.009986718001804467,
-            "acc_norm": 0.7665941240478781,
-            "acc_norm_stderr": 0.009869247889520991
-        }
-    },
-    "versions": {
-        "anli_r1": 0,
-        "anli_r2": 0,
-        "anli_r3": 0,
-        "cb": 1,
-        "copa": 0,
-        "hellaswag": 0,
-        "rte": 0,
-        "winogrande": 0,
-        "storycloze_2016": 0,
-        "boolq": 1,
-        "arc_easy": 0,
-        "arc_challenge": 0,
-        "sciq": 0,
-        "piqa": 0
-    }
-}

8b7178b13b/evaluation/rankeval/8b7178b13b_4.csv

@@ -0,0 +1,21 @@
+task,metric,value,err,version
+anli_r1,acc,0.344,0.015029633724408947,0
+anli_r2,acc,0.345,0.015039986742055238,0
+anli_r3,acc,0.34833333333333333,0.013759437498874072,0
+arc_challenge,acc,0.3199658703071672,0.013631345807016196,0
+arc_challenge,acc_norm,0.3447098976109215,0.01388881628678211,0
+arc_easy,acc,0.6632996632996633,0.009697166595752475,0
+arc_easy,acc_norm,0.6447811447811448,0.009820245899287124,0
+boolq,acc,0.636085626911315,0.00841491890912884,1
+cb,acc,0.21428571428571427,0.055328333517248834,1
+cb,f1,0.1997113997113997,,1
+copa,acc,0.79,0.040936018074033256,0
+hellaswag,acc,0.5206134236207927,0.004985539159783419,0
+hellaswag,acc_norm,0.6900019916351324,0.004615472210316043,0
+piqa,acc,0.7665941240478781,0.009869247889521007,0
+piqa,acc_norm,0.7682263329706203,0.00984514377279404,0
+rte,acc,0.48736462093862815,0.030086851767188564,0
+sciq,acc,0.927,0.008230354715244062,0
+sciq,acc_norm,0.908,0.009144376393151108,0
+storycloze_2016,acc,0.7509353287012293,0.010000841162740146,0
+winogrande,acc,0.6077348066298343,0.013722400462000888,0

8b7178b13b/evaluation/rankeval/8b7178b13b_4_lm-eval_global_step84877_2023-05-15-10-06-46_4shots_backup.json

@@ -1,87 +0,0 @@
-{
-    "results": {
-        "anli_r1": {
-            "acc": 0.344,
-            "acc_stderr": 0.015029633724408947
-        },
-        "anli_r2": {
-            "acc": 0.345,
-            "acc_stderr": 0.015039986742055238
-        },
-        "anli_r3": {
-            "acc": 0.34833333333333333,
-            "acc_stderr": 0.013759437498874072
-        },
-        "cb": {
-            "acc": 0.21428571428571427,
-            "acc_stderr": 0.055328333517248834,
-            "f1": 0.1997113997113997
-        },
-        "copa": {
-            "acc": 0.79,
-            "acc_stderr": 0.040936018074033256
-        },
-        "hellaswag": {
-            "acc": 0.5206134236207927,
-            "acc_stderr": 0.004985539159783419,
-            "acc_norm": 0.6900019916351324,
-            "acc_norm_stderr": 0.004615472210316043
-        },
-        "rte": {
-            "acc": 0.48736462093862815,
-            "acc_stderr": 0.030086851767188564
-        },
-        "winogrande": {
-            "acc": 0.6077348066298343,
-            "acc_stderr": 0.013722400462000888
-        },
-        "storycloze_2016": {
-            "acc": 0.7509353287012293,
-            "acc_stderr": 0.010000841162740146
-        },
-        "boolq": {
-            "acc": 0.636085626911315,
-            "acc_stderr": 0.00841491890912884
-        },
-        "arc_easy": {
-            "acc": 0.6632996632996633,
-            "acc_stderr": 0.009697166595752475,
-            "acc_norm": 0.6447811447811448,
-            "acc_norm_stderr": 0.009820245899287124
-        },
-        "arc_challenge": {
-            "acc": 0.3199658703071672,
-            "acc_stderr": 0.013631345807016196,
-            "acc_norm": 0.3447098976109215,
-            "acc_norm_stderr": 0.01388881628678211
-        },
-        "sciq": {
-            "acc": 0.927,
-            "acc_stderr": 0.008230354715244062,
-            "acc_norm": 0.908,
-            "acc_norm_stderr": 0.009144376393151108
-        },
-        "piqa": {
-            "acc": 0.7665941240478781,
-            "acc_stderr": 0.009869247889521007,
-            "acc_norm": 0.7682263329706203,
-            "acc_norm_stderr": 0.00984514377279404
-        }
-    },
-    "versions": {
-        "anli_r1": 0,
-        "anli_r2": 0,
-        "anli_r3": 0,
-        "cb": 1,
-        "copa": 0,
-        "hellaswag": 0,
-        "rte": 0,
-        "winogrande": 0,
-        "storycloze_2016": 0,
-        "boolq": 1,
-        "arc_easy": 0,
-        "arc_challenge": 0,
-        "sciq": 0,
-        "piqa": 0
-    }
-}

8b7178b13b/evaluation/rankeval/8b7178b13b_5.csv

@@ -0,0 +1,21 @@
+task,metric,value,err,version
+anli_r1,acc,0.34,0.014987482264363935,0
+anli_r2,acc,0.337,0.014955087918653605,0
+anli_r3,acc,0.3325,0.013605417345710526,0
+arc_challenge,acc,0.3310580204778157,0.013752062419817834,0
+arc_challenge,acc_norm,0.3447098976109215,0.013888816286782112,0
+arc_easy,acc,0.6536195286195287,0.009763542075695738,0
+arc_easy,acc_norm,0.6401515151515151,0.009848484848484836,0
+boolq,acc,0.6409785932721712,0.008390241754319908,1
+cb,acc,0.23214285714285715,0.0569293902400011,1
+cb,f1,0.223351041141572,,1
+copa,acc,0.76,0.042923469599092816,0
+hellaswag,acc,0.5209121688906593,0.004985415250690905,0
+hellaswag,acc_norm,0.689205337582155,0.004618730353217064,0
+piqa,acc,0.7606093579978237,0.009955884250291688,0
+piqa,acc_norm,0.7747551686615887,0.009746643471032136,0
+rte,acc,0.5054151624548736,0.030094698123239966,0
+sciq,acc,0.923,0.008434580140240648,0
+sciq,acc_norm,0.912,0.008963053962592074,0
+storycloze_2016,acc,0.7498663816141101,0.010015143382536456,0
+winogrande,acc,0.5935280189423836,0.01380444869775337,0

8b7178b13b/evaluation/rankeval/8b7178b13b_5_lm-eval_global_step84877_2023-05-15-10-06-46_5shots_backup.json

@@ -1,87 +0,0 @@
-{
-    "results": {
-        "anli_r1": {
-            "acc": 0.34,
-            "acc_stderr": 0.014987482264363935
-        },
-        "anli_r2": {
-            "acc": 0.337,
-            "acc_stderr": 0.014955087918653605
-        },
-        "anli_r3": {
-            "acc": 0.3325,
-            "acc_stderr": 0.013605417345710526
-        },
-        "cb": {
-            "acc": 0.23214285714285715,
-            "acc_stderr": 0.0569293902400011,
-            "f1": 0.223351041141572
-        },
-        "copa": {
-            "acc": 0.76,
-            "acc_stderr": 0.042923469599092816
-        },
-        "hellaswag": {
-            "acc": 0.5209121688906593,
-            "acc_stderr": 0.004985415250690905,
-            "acc_norm": 0.689205337582155,
-            "acc_norm_stderr": 0.004618730353217064
-        },
-        "rte": {
-            "acc": 0.5054151624548736,
-            "acc_stderr": 0.030094698123239966
-        },
-        "winogrande": {
-            "acc": 0.5935280189423836,
-            "acc_stderr": 0.01380444869775337
-        },
-        "storycloze_2016": {
-            "acc": 0.7498663816141101,
-            "acc_stderr": 0.010015143382536456
-        },
-        "boolq": {
-            "acc": 0.6409785932721712,
-            "acc_stderr": 0.008390241754319908
-        },
-        "arc_easy": {
-            "acc": 0.6536195286195287,
-            "acc_stderr": 0.009763542075695738,
-            "acc_norm": 0.6401515151515151,
-            "acc_norm_stderr": 0.009848484848484836
-        },
-        "arc_challenge": {
-            "acc": 0.3310580204778157,
-            "acc_stderr": 0.013752062419817834,
-            "acc_norm": 0.3447098976109215,
-            "acc_norm_stderr": 0.013888816286782112
-        },
-        "sciq": {
-            "acc": 0.923,
-            "acc_stderr": 0.008434580140240648,
-            "acc_norm": 0.912,
-            "acc_norm_stderr": 0.008963053962592074
-        },
-        "piqa": {
-            "acc": 0.7606093579978237,
-            "acc_stderr": 0.009955884250291688,
-            "acc_norm": 0.7747551686615887,
-            "acc_norm_stderr": 0.009746643471032136
-        }
-    },
-    "versions": {
-        "anli_r1": 0,
-        "anli_r2": 0,
-        "anli_r3": 0,
-        "cb": 1,
-        "copa": 0,
-        "hellaswag": 0,
-        "rte": 0,
-        "winogrande": 0,
-        "storycloze_2016": 0,
-        "boolq": 1,
-        "arc_easy": 0,
-        "arc_challenge": 0,
-        "sciq": 0,
-        "piqa": 0
-    }
-}

8b7178b13b/transformers/vocab.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:196139668be63f3b5d6574427317ae82f612a97c5d1cdaf36ed2256dbf636783
+size 1042301

8b7178b178b/evaluation/8b7178b178b_0_babi.json

@@ -0,0 +1,22 @@
+{
+  "results": {
+    "babi": {
+      "em": 0.0,
+      "em_stderr": 0.0
+    }
+  },
+  "versions": {
+    "babi": 0
+  },
+  "config": {
+    "model": "gpt2",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b178b/transformers",
+    "num_fewshot": 0,
+    "batch_size": null,
+    "device": null,
+    "no_cache": true,
+    "limit": 3000,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

8b7178b178b/evaluation/8b7178b178b_1_babi.json

@@ -0,0 +1,22 @@
+{
+  "results": {
+    "babi": {
+      "em": 0.0,
+      "em_stderr": 0.0
+    }
+  },
+  "versions": {
+    "babi": 0
+  },
+  "config": {
+    "model": "gpt2",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b178b/transformers",
+    "num_fewshot": 1,
+    "batch_size": null,
+    "device": null,
+    "no_cache": true,
+    "limit": 3000,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

8b7178b178b/evaluation/8b7178b178b_2_babi.json

@@ -0,0 +1,22 @@
+{
+  "results": {
+    "babi": {
+      "em": 0.0,
+      "em_stderr": 0.0
+    }
+  },
+  "versions": {
+    "babi": 0
+  },
+  "config": {
+    "model": "gpt2",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b178b/transformers",
+    "num_fewshot": 2,
+    "batch_size": null,
+    "device": null,
+    "no_cache": true,
+    "limit": 3000,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

8b7178b178b/evaluation/8b7178b178b_3_babi.json

@@ -0,0 +1,22 @@
+{
+  "results": {
+    "babi": {
+      "em": 0.0,
+      "em_stderr": 0.0
+    }
+  },
+  "versions": {
+    "babi": 0
+  },
+  "config": {
+    "model": "gpt2",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b178b/transformers",
+    "num_fewshot": 3,
+    "batch_size": null,
+    "device": null,
+    "no_cache": true,
+    "limit": 3000,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

8b7178b178b/evaluation/8b7178b178b_4_babi.json

@@ -0,0 +1,22 @@
+{
+  "results": {
+    "babi": {
+      "em": 0.0,
+      "em_stderr": 0.0
+    }
+  },
+  "versions": {
+    "babi": 0
+  },
+  "config": {
+    "model": "gpt2",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b178b/transformers",
+    "num_fewshot": 4,
+    "batch_size": null,
+    "device": null,
+    "no_cache": true,
+    "limit": 3000,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

8b7178b178b/evaluation/8b7178b178b_5_babi.json

@@ -0,0 +1,22 @@
+{
+  "results": {
+    "babi": {
+      "em": 0.0,
+      "em_stderr": 0.0
+    }
+  },
+  "versions": {
+    "babi": 0
+  },
+  "config": {
+    "model": "gpt2",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b178b/transformers",
+    "num_fewshot": 5,
+    "batch_size": null,
+    "device": null,
+    "no_cache": true,
+    "limit": 3000,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}