Upload eval_results/Qwen/Qwen1.5-0.5B-Chat/main/mini_math/results_2024-04-23T12-52-50.054036.json with huggingface_hub

eval_results/Qwen/Qwen1.5-0.5B-Chat/main/mini_math/results_2024-04-23T12-52-50.054036.json

@@ -0,0 +1,291 @@
+{
+  "config_general": {
+    "lighteval_sha": "?",
+    "num_fewshot_seeds": 1,
+    "override_batch_size": 4,
+    "max_samples": null,
+    "job_id": "",
+    "start_time": 6286403.535148175,
+    "end_time": 6287087.710961109,
+    "total_evaluation_time_secondes": "684.175812933594",
+    "model_name": "Qwen/Qwen1.5-0.5B-Chat",
+    "model_sha": "f82bd3692de0283f4a4b31e06d164dd8467fb52e",
+    "model_dtype": "torch.bfloat16",
+    "model_size": "1.05 GB",
+    "config": null
+  },
+  "results": {
+    "custom|mini_math:level_1|0": {
+      "qem": 0.0,
+      "qem_stderr": 0.0
+    },
+    "custom|mini_math:level_2|0": {
+      "qem": 0.0,
+      "qem_stderr": 0.0
+    },
+    "custom|mini_math:level_3|0": {
+      "qem": 0.0,
+      "qem_stderr": 0.0
+    },
+    "custom|mini_math:level_4|0": {
+      "qem": 0.0,
+      "qem_stderr": 0.0
+    },
+    "custom|mini_math:level_5|0": {
+      "qem": 0.0,
+      "qem_stderr": 0.0
+    },
+    "custom|mini_math:_average|0": {
+      "qem": 0.0,
+      "qem_stderr": 0.0
+    },
+    "all": {
+      "qem": 0.0,
+      "qem_stderr": 0.0
+    }
+  },
+  "versions": {
+    "custom|mini_math:level_1|0": 0,
+    "custom|mini_math:level_2|0": 0,
+    "custom|mini_math:level_3|0": 0,
+    "custom|mini_math:level_4|0": 0,
+    "custom|mini_math:level_5|0": 0
+  },
+  "config_tasks": {
+    "custom|mini_math:level_1": {
+      "name": "mini_math:level_1",
+      "prompt_function": "math",
+      "hf_repo": "HuggingFaceH4/lighteval-mini-math",
+      "hf_subset": "Level 1",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom",
+        "mini_math"
+      ],
+      "original_num_docs": 38,
+      "effective_num_docs": 38,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|mini_math:level_2": {
+      "name": "mini_math:level_2",
+      "prompt_function": "math",
+      "hf_repo": "HuggingFaceH4/lighteval-mini-math",
+      "hf_subset": "Level 2",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom",
+        "mini_math"
+      ],
+      "original_num_docs": 71,
+      "effective_num_docs": 71,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|mini_math:level_3": {
+      "name": "mini_math:level_3",
+      "prompt_function": "math",
+      "hf_repo": "HuggingFaceH4/lighteval-mini-math",
+      "hf_subset": "Level 3",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom",
+        "mini_math"
+      ],
+      "original_num_docs": 78,
+      "effective_num_docs": 78,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|mini_math:level_4": {
+      "name": "mini_math:level_4",
+      "prompt_function": "math",
+      "hf_repo": "HuggingFaceH4/lighteval-mini-math",
+      "hf_subset": "Level 4",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom",
+        "mini_math"
+      ],
+      "original_num_docs": 109,
+      "effective_num_docs": 109,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|mini_math:level_5": {
+      "name": "mini_math:level_5",
+      "prompt_function": "math",
+      "hf_repo": "HuggingFaceH4/lighteval-mini-math",
+      "hf_subset": "Level 5",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom",
+        "mini_math"
+      ],
+      "original_num_docs": 107,
+      "effective_num_docs": 107,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    }
+  },
+  "summary_tasks": {
+    "custom|mini_math:level_1|0": {
+      "hashes": {
+        "hash_examples": "781a25d9a2f29e87",
+        "hash_full_prompts": "09ef7f6df57f47ef",
+        "hash_input_tokens": "a52e14904d82f4f0",
+        "hash_cont_tokens": "1378bece65ca5077"
+      },
+      "truncated": 38,
+      "non_truncated": 0,
+      "padded": 27,
+      "non_padded": 11,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|mini_math:level_2|0": {
+      "hashes": {
+        "hash_examples": "471a4a8568c70994",
+        "hash_full_prompts": "71779330375b1791",
+        "hash_input_tokens": "3882edc6f5609345",
+        "hash_cont_tokens": "ff9806e2946dc746"
+      },
+      "truncated": 71,
+      "non_truncated": 0,
+      "padded": 24,
+      "non_padded": 47,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|mini_math:level_3|0": {
+      "hashes": {
+        "hash_examples": "aa6979a90c9a7776",
+        "hash_full_prompts": "a9999d3932558466",
+        "hash_input_tokens": "01ceca3580b9eb5e",
+        "hash_cont_tokens": "fbf0bffb8a2612ef"
+      },
+      "truncated": 78,
+      "non_truncated": 0,
+      "padded": 36,
+      "non_padded": 42,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|mini_math:level_4|0": {
+      "hashes": {
+        "hash_examples": "2e9e3be067de6ec6",
+        "hash_full_prompts": "30f669c1c4439175",
+        "hash_input_tokens": "4a8097cf379ba41c",
+        "hash_cont_tokens": "88e54955e7213376"
+      },
+      "truncated": 109,
+      "non_truncated": 0,
+      "padded": 38,
+      "non_padded": 71,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|mini_math:level_5|0": {
+      "hashes": {
+        "hash_examples": "18b402b510f9a746",
+        "hash_full_prompts": "0c8f48ff502c1190",
+        "hash_input_tokens": "b6ecf04a8c2e48b4",
+        "hash_cont_tokens": "c603094defd72220"
+      },
+      "truncated": 106,
+      "non_truncated": 1,
+      "padded": 38,
+      "non_padded": 69,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    }
+  },
+  "summary_general": {
+    "hashes": {
+      "hash_examples": "470ce01cab0383c9",
+      "hash_full_prompts": "36a8a8499eb2fd53",
+      "hash_input_tokens": "a651497058968421",
+      "hash_cont_tokens": "16318eec6b11890e"
+    },
+    "truncated": 402,
+    "non_truncated": 1,
+    "padded": 163,
+    "non_padded": 240,
+    "num_truncated_few_shots": 0
+  }
+}