open-r1-eval-leaderboard
/
eval_results
/AI-MO
/mistral-7b-sft
/aimo_v03.00
/math
/results_2024-04-23T18-58-14.675111.json
edbeeching
HF staff
Upload eval_results/AI-MO/mistral-7b-sft/aimo_v03.00/math/results_2024-04-23T18-58-14.675111.json with huggingface_hub
a8b096d
verified
| { | |
| "config_general": { | |
| "lighteval_sha": "?", | |
| "num_fewshot_seeds": 1, | |
| "override_batch_size": 4, | |
| "max_samples": null, | |
| "job_id": "", | |
| "start_time": 1024730.024178654, | |
| "end_time": 1035012.272490175, | |
| "total_evaluation_time_secondes": "10282.24831152102", | |
| "model_name": "AI-MO/mistral-7b-sft", | |
| "model_sha": "650855a96ddbc599a32d1cb91fd65a4739755d02", | |
| "model_dtype": "torch.bfloat16", | |
| "model_size": "13.99 GB", | |
| "config": null | |
| }, | |
| "results": { | |
| "lighteval|math:algebra|0": { | |
| "qem": 0.002527379949452401, | |
| "qem_stderr": 0.0014579526348155309 | |
| }, | |
| "lighteval|math:counting_and_probability|0": { | |
| "qem": 0.0, | |
| "qem_stderr": 0.0 | |
| }, | |
| "lighteval|math:geometry|0": { | |
| "qem": 0.0, | |
| "qem_stderr": 0.0 | |
| }, | |
| "lighteval|math:intermediate_algebra|0": { | |
| "qem": 0.005537098560354375, | |
| "qem_stderr": 0.0024707690436948657 | |
| }, | |
| "lighteval|math:number_theory|0": { | |
| "qem": 0.001851851851851852, | |
| "qem_stderr": 0.0018518518518518454 | |
| }, | |
| "lighteval|math:prealgebra|0": { | |
| "qem": 0.002296211251435132, | |
| "qem_stderr": 0.0016227331369346285 | |
| }, | |
| "lighteval|math:precalculus|0": { | |
| "qem": 0.005494505494505495, | |
| "qem_stderr": 0.0031664282264934705 | |
| }, | |
| "lighteval|math:_average|0": { | |
| "qem": 0.0025295781582284653, | |
| "qem_stderr": 0.0015099621276843344 | |
| }, | |
| "all": { | |
| "qem": 0.0025295781582284653, | |
| "qem_stderr": 0.0015099621276843344 | |
| } | |
| }, | |
| "versions": { | |
| "lighteval|math:algebra|0": 0, | |
| "lighteval|math:counting_and_probability|0": 0, | |
| "lighteval|math:geometry|0": 0, | |
| "lighteval|math:intermediate_algebra|0": 0, | |
| "lighteval|math:number_theory|0": 0, | |
| "lighteval|math:prealgebra|0": 0, | |
| "lighteval|math:precalculus|0": 0 | |
| }, | |
| "config_tasks": { | |
| "lighteval|math:algebra": { | |
| "name": "math:algebra", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "algebra", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 1187, | |
| "effective_num_docs": 1187, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:counting_and_probability": { | |
| "name": "math:counting_and_probability", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "counting_and_probability", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 474, | |
| "effective_num_docs": 474, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:geometry": { | |
| "name": "math:geometry", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "geometry", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 479, | |
| "effective_num_docs": 479, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:intermediate_algebra": { | |
| "name": "math:intermediate_algebra", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "intermediate_algebra", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 903, | |
| "effective_num_docs": 903, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:number_theory": { | |
| "name": "math:number_theory", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "number_theory", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 540, | |
| "effective_num_docs": 540, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:prealgebra": { | |
| "name": "math:prealgebra", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "prealgebra", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 871, | |
| "effective_num_docs": 871, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| }, | |
| "lighteval|math:precalculus": { | |
| "name": "math:precalculus", | |
| "prompt_function": "math", | |
| "hf_repo": "lighteval/MATH", | |
| "hf_subset": "precalculus", | |
| "metric": [ | |
| "quasi_exact_match_math" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "test", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": 2048, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "lighteval", | |
| "math" | |
| ], | |
| "original_num_docs": 546, | |
| "effective_num_docs": 546, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| } | |
| }, | |
| "summary_tasks": { | |
| "lighteval|math:algebra|0": { | |
| "hashes": { | |
| "hash_examples": "37a2fd2f076d2e49", | |
| "hash_full_prompts": "461f2ab197cdfd01", | |
| "hash_input_tokens": "6ccb9cd0578997f2", | |
| "hash_cont_tokens": "641c6c94ae22bd38" | |
| }, | |
| "truncated": 1187, | |
| "non_truncated": 0, | |
| "padded": 186, | |
| "non_padded": 1001, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:counting_and_probability|0": { | |
| "hashes": { | |
| "hash_examples": "97b4892e28bc078b", | |
| "hash_full_prompts": "e2cc9eb28ca159b1", | |
| "hash_input_tokens": "d5d5c2e1da593390", | |
| "hash_cont_tokens": "1173c6543475fc89" | |
| }, | |
| "truncated": 474, | |
| "non_truncated": 0, | |
| "padded": 42, | |
| "non_padded": 432, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:geometry|0": { | |
| "hashes": { | |
| "hash_examples": "9e9f0228b8b3d093", | |
| "hash_full_prompts": "e581f42fefeed8c9", | |
| "hash_input_tokens": "812493196cf41f70", | |
| "hash_cont_tokens": "7882030a027e5163" | |
| }, | |
| "truncated": 479, | |
| "non_truncated": 0, | |
| "padded": 129, | |
| "non_padded": 350, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:intermediate_algebra|0": { | |
| "hashes": { | |
| "hash_examples": "cfe73a8e28ae94de", | |
| "hash_full_prompts": "7557d30ba2cc15ef", | |
| "hash_input_tokens": "403cd4ed2608ce4b", | |
| "hash_cont_tokens": "caf017889824a1c3" | |
| }, | |
| "truncated": 900, | |
| "non_truncated": 3, | |
| "padded": 54, | |
| "non_padded": 849, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:number_theory|0": { | |
| "hashes": { | |
| "hash_examples": "4ee5237cf144afac", | |
| "hash_full_prompts": "7b983a6fe1e3d0a8", | |
| "hash_input_tokens": "3a054306f1625c42", | |
| "hash_cont_tokens": "ab833704d33b839d" | |
| }, | |
| "truncated": 540, | |
| "non_truncated": 0, | |
| "padded": 7, | |
| "non_padded": 533, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:prealgebra|0": { | |
| "hashes": { | |
| "hash_examples": "3fb3afeb885f73d8", | |
| "hash_full_prompts": "736a03165dbf8f49", | |
| "hash_input_tokens": "5ebd9e6ab29d3d72", | |
| "hash_cont_tokens": "5879a73271ce9172" | |
| }, | |
| "truncated": 871, | |
| "non_truncated": 0, | |
| "padded": 55, | |
| "non_padded": 816, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "lighteval|math:precalculus|0": { | |
| "hashes": { | |
| "hash_examples": "753e25ab9ec4b46c", | |
| "hash_full_prompts": "632ca5aa79f53b59", | |
| "hash_input_tokens": "9004be14b5ded67f", | |
| "hash_cont_tokens": "16941501f23981f2" | |
| }, | |
| "truncated": 546, | |
| "non_truncated": 0, | |
| "padded": 24, | |
| "non_padded": 522, | |
| "effective_few_shots": 0.0, | |
| "num_truncated_few_shots": 0 | |
| } | |
| }, | |
| "summary_general": { | |
| "hashes": { | |
| "hash_examples": "b76099aa9092a203", | |
| "hash_full_prompts": "2d2e105e1a0eeab9", | |
| "hash_input_tokens": "d95739dc5b6c6daf", | |
| "hash_cont_tokens": "d6c0e193b8193c23" | |
| }, | |
| "truncated": 4997, | |
| "non_truncated": 3, | |
| "padded": 497, | |
| "non_padded": 4503, | |
| "num_truncated_few_shots": 0 | |
| } | |
| } |