|
--- |
|
base_model: unsloth/Llama-3.2-3B-Instruct-bnb-4bit |
|
datasets: |
|
- microsoft/orca-agentinstruct-1M-v1 |
|
pipeline_tag: text-generation |
|
library_name: transformers |
|
license: llama3.2 |
|
tags: |
|
- unsloth |
|
- transformers |
|
--- |
|
|
|
 |
|
|
|
### eval |
|
|
|
|
|
|
|
| Tasks |Version|Filter|n-shot| Metric | |Value | |Stderr| |
|
|----------------------------------------------------------|-------|------|-----:|-----------------------|---|-----:|---|------| |
|
|hellaswag | 1|none | 0|acc |↑ |0.5141|± |0.0050| |
|
| | |none | 0|acc_norm |↑ |0.6793|± |0.0047| |
|
|leaderboard_bbh | N/A| | | | | | | | |
|
| - leaderboard_bbh_boolean_expressions | 1|none | 3|acc_norm |↑ |0.6040|± |0.0310| |
|
| - leaderboard_bbh_causal_judgement | 1|none | 3|acc_norm |↑ |0.5668|± |0.0363| |
|
| - leaderboard_bbh_date_understanding | 1|none | 3|acc_norm |↑ |0.4880|± |0.0317| |
|
| - leaderboard_bbh_disambiguation_qa | 1|none | 3|acc_norm |↑ |0.3760|± |0.0307| |
|
| - leaderboard_bbh_formal_fallacies | 1|none | 3|acc_norm |↑ |0.5400|± |0.0316| |
|
| - leaderboard_bbh_geometric_shapes | 1|none | 3|acc_norm |↑ |0.2200|± |0.0263| |
|
| - leaderboard_bbh_hyperbaton | 1|none | 3|acc_norm |↑ |0.5640|± |0.0314| |
|
| - leaderboard_bbh_logical_deduction_five_objects | 1|none | 3|acc_norm |↑ |0.4560|± |0.0316| |
|
| - leaderboard_bbh_logical_deduction_seven_objects | 1|none | 3|acc_norm |↑ |0.4360|± |0.0314| |
|
| - leaderboard_bbh_logical_deduction_three_objects | 1|none | 3|acc_norm |↑ |0.4880|± |0.0317| |
|
| - leaderboard_bbh_movie_recommendation | 1|none | 3|acc_norm |↑ |0.6360|± |0.0305| |
|
| - leaderboard_bbh_navigate | 1|none | 3|acc_norm |↑ |0.6200|± |0.0308| |
|
| - leaderboard_bbh_object_counting | 1|none | 3|acc_norm |↑ |0.4120|± |0.0312| |
|
| - leaderboard_bbh_penguins_in_a_table | 1|none | 3|acc_norm |↑ |0.3219|± |0.0388| |
|
| - leaderboard_bbh_reasoning_about_colored_objects | 1|none | 3|acc_norm |↑ |0.3440|± |0.0301| |
|
| - leaderboard_bbh_ruin_names | 1|none | 3|acc_norm |↑ |0.3240|± |0.0297| |
|
| - leaderboard_bbh_salient_translation_error_detection | 1|none | 3|acc_norm |↑ |0.3120|± |0.0294| |
|
| - leaderboard_bbh_snarks | 1|none | 3|acc_norm |↑ |0.4494|± |0.0374| |
|
| - leaderboard_bbh_sports_understanding | 1|none | 3|acc_norm |↑ |0.6040|± |0.0310| |
|
| - leaderboard_bbh_temporal_sequences | 1|none | 3|acc_norm |↑ |0.1000|± |0.0190| |
|
| - leaderboard_bbh_tracking_shuffled_objects_five_objects | 1|none | 3|acc_norm |↑ |0.1600|± |0.0232| |
|
| - leaderboard_bbh_tracking_shuffled_objects_seven_objects| 1|none | 3|acc_norm |↑ |0.1200|± |0.0206| |
|
| - leaderboard_bbh_tracking_shuffled_objects_three_objects| 1|none | 3|acc_norm |↑ |0.3440|± |0.0301| |
|
| - leaderboard_bbh_web_of_lies | 1|none | 3|acc_norm |↑ |0.5160|± |0.0317| |
|
|leaderboard_gpqa | N/A| | | | | | | | |
|
| - leaderboard_gpqa_diamond | 1|none | 0|acc_norm |↑ |0.2727|± |0.0317| |
|
| - leaderboard_gpqa_extended | 1|none | 0|acc_norm |↑ |0.2802|± |0.0192| |
|
| - leaderboard_gpqa_main | 1|none | 0|acc_norm |↑ |0.2545|± |0.0206| |
|
|leaderboard_ifeval | 3|none | 0|inst_level_loose_acc |↑ |0.5252|± | N/A| |
|
| | |none | 0|inst_level_strict_acc |↑ |0.4748|± | N/A| |
|
| | |none | 0|prompt_level_loose_acc |↑ |0.3919|± |0.0210| |
|
| | |none | 0|prompt_level_strict_acc|↑ |0.3420|± |0.0204| |
|
|leaderboard_math_hard | N/A| | | | | | | | |
|
| - leaderboard_math_algebra_hard | 2|none | 4|exact_match |↑ |0.2150|± |0.0235| |
|
| - leaderboard_math_counting_and_prob_hard | 2|none | 4|exact_match |↑ |0.0244|± |0.0140| |
|
| - leaderboard_math_geometry_hard | 2|none | 4|exact_match |↑ |0.0606|± |0.0208| |
|
| - leaderboard_math_intermediate_algebra_hard | 2|none | 4|exact_match |↑ |0.0143|± |0.0071| |
|
| - leaderboard_math_num_theory_hard | 2|none | 4|exact_match |↑ |0.0649|± |0.0199| |
|
| - leaderboard_math_prealgebra_hard | 2|none | 4|exact_match |↑ |0.1762|± |0.0275| |
|
| - leaderboard_math_precalculus_hard | 2|none | 4|exact_match |↑ |0.0519|± |0.0192| |
|
|leaderboard_mmlu_pro | 0.1|none | 5|acc |↑ |0.2822|± |0.0041| |
|
|leaderboard_musr | N/A| | | | | | | | |
|
| - leaderboard_musr_murder_mysteries | 1|none | 0|acc_norm |↑ |0.5400|± |0.0316| |
|
| - leaderboard_musr_object_placements | 1|none | 0|acc_norm |↑ |0.2344|± |0.0265| |
|
| - leaderboard_musr_team_allocation | 1|none | 0|acc_norm |↑ |0.3200|± |0.0296| |
|
|
|
|
|
|
|
### Framework versions |
|
|
|
- unsloth 2024.11.5 |
|
- trl 0.12.0 |
|
|
|
### Training HW |
|
- V100 |
|
|
|
{ |
|
"results": { |
|
"leaderboard_musr": { |
|
" ": " ", |
|
"alias": "leaderboard_musr" |
|
}, |
|
"leaderboard_musr_murder_mysteries": { |
|
"alias": " - leaderboard_musr_murder_mysteries", |
|
"acc_norm,none": 0.54, |
|
"acc_norm_stderr,none": 0.03158465389149902 |
|
}, |
|
"leaderboard_musr_object_placements": { |
|
"alias": " - leaderboard_musr_object_placements", |
|
"acc_norm,none": 0.234375, |
|
"acc_norm_stderr,none": 0.02652733398834892 |
|
}, |
|
"leaderboard_musr_team_allocation": { |
|
"alias": " - leaderboard_musr_team_allocation", |
|
"acc_norm,none": 0.32, |
|
"acc_norm_stderr,none": 0.029561724955241033 |
|
} |
|
}, |
|
"group_subtasks": { |
|
"leaderboard_musr": [ |
|
"leaderboard_musr_murder_mysteries", |
|
"leaderboard_musr_object_placements", |
|
"leaderboard_musr_team_allocation" |
|
] |
|
}, |
|
"configs": { |
|
"leaderboard_musr_murder_mysteries": { |
|
"task": "leaderboard_musr_murder_mysteries", |
|
"dataset_path": "TAUR-Lab/MuSR", |
|
"test_split": "murder_mysteries", |
|
"doc_to_text": "def doc_to_text(doc):\n \"\"\"\n Convert a doc to text.\n \"\"\"\n choices = \"\"\n for i, choice in enumerate(ast.literal_eval(doc[\"choices\"])):\n choices += f\"{i+1} - {choice}\\n\"\n\n text = DOC_TO_TEXT.format(\n narrative=doc[\"narrative\"], question=doc[\"question\"], choices=choices\n )\n\n return text\n", |
|
"doc_to_target": "{{answer_choice}}", |
|
"doc_to_choice": "{{choices}}", |
|
"description": "", |
|
"target_delimiter": " ", |
|
"fewshot_delimiter": "\n\n", |
|
"num_fewshot": 0, |
|
"metric_list": [ |
|
{ |
|
"metric": "acc_norm", |
|
"aggregation": "mean", |
|
"higher_is_better": true |
|
} |
|
], |
|
"output_type": "multiple_choice", |
|
"repeats": 1, |
|
"should_decontaminate": false, |
|
"metadata": { |
|
"version": 1.0 |
|
} |
|
}, |
|
"leaderboard_musr_object_placements": { |
|
"task": "leaderboard_musr_object_placements", |
|
"dataset_path": "TAUR-Lab/MuSR", |
|
"test_split": "object_placements", |
|
"doc_to_text": "def doc_to_text(doc):\n \"\"\"\n Convert a doc to text.\n \"\"\"\n choices = \"\"\n for i, choice in enumerate(ast.literal_eval(doc[\"choices\"])):\n choices += f\"{i+1} - {choice}\\n\"\n\n text = DOC_TO_TEXT.format(\n narrative=doc[\"narrative\"], question=doc[\"question\"], choices=choices\n )\n\n return text\n", |
|
"doc_to_target": "{{answer_choice}}", |
|
"doc_to_choice": "{{choices}}", |
|
"description": "", |
|
"target_delimiter": " ", |
|
"fewshot_delimiter": "\n\n", |
|
"num_fewshot": 0, |
|
"metric_list": [ |
|
{ |
|
"metric": "acc_norm", |
|
"aggregation": "mean", |
|
"higher_is_better": true |
|
} |
|
], |
|
"output_type": "multiple_choice", |
|
"repeats": 1, |
|
"should_decontaminate": false, |
|
"metadata": { |
|
"version": 1.0 |
|
} |
|
}, |
|
"leaderboard_musr_team_allocation": { |
|
"task": "leaderboard_musr_team_allocation", |
|
"dataset_path": "TAUR-Lab/MuSR", |
|
"test_split": "team_allocation", |
|
"doc_to_text": "def doc_to_text(doc):\n \"\"\"\n Convert a doc to text.\n \"\"\"\n choices = \"\"\n for i, choice in enumerate(ast.literal_eval(doc[\"choices\"])):\n choices += f\"{i+1} - {choice}\\n\"\n\n text = DOC_TO_TEXT.format(\n narrative=doc[\"narrative\"], question=doc[\"question\"], choices=choices\n )\n\n return text\n", |
|
"doc_to_target": "{{answer_choice}}", |
|
"doc_to_choice": "{{choices}}", |
|
"description": "", |
|
"target_delimiter": " ", |
|
"fewshot_delimiter": "\n\n", |
|
"num_fewshot": 0, |
|
"metric_list": [ |
|
{ |
|
"metric": "acc_norm", |
|
"aggregation": "mean", |
|
"higher_is_better": true |
|
} |
|
], |
|
"output_type": "multiple_choice", |
|
"repeats": 1, |
|
"should_decontaminate": false, |
|
"metadata": { |
|
"version": 1.0 |
|
} |
|
} |
|
}, |
|
"versions": { |
|
"leaderboard_musr_murder_mysteries": 1.0, |
|
"leaderboard_musr_object_placements": 1.0, |
|
"leaderboard_musr_team_allocation": 1.0 |
|
}, |
|
"n-shot": { |
|
"leaderboard_musr_murder_mysteries": 0, |
|
"leaderboard_musr_object_placements": 0, |
|
"leaderboard_musr_team_allocation": 0 |
|
}, |
|
"higher_is_better": { |
|
"leaderboard_musr": { |
|
"acc_norm": true |
|
}, |
|
"leaderboard_musr_murder_mysteries": { |
|
"acc_norm": true |
|
}, |
|
"leaderboard_musr_object_placements": { |
|
"acc_norm": true |
|
}, |
|
"leaderboard_musr_team_allocation": { |
|
"acc_norm": true |
|
} |
|
}, |
|
"n-samples": { |
|
"leaderboard_musr_murder_mysteries": { |
|
"original": 250, |
|
"effective": 250 |
|
}, |
|
"leaderboard_musr_object_placements": { |
|
"original": 256, |
|
"effective": 256 |
|
}, |
|
"leaderboard_musr_team_allocation": { |
|
"original": 250, |
|
"effective": 250 |
|
} |
|
}, |
|
"config": { |
|
"model": "hf", |
|
"model_args": "pretrained=DevQuasar/analytical_reasoning_r16a32_unsloth-Llama-3.2-3B-Instruct-bnb-4bit", |
|
"batch_size": "auto:4", |
|
"batch_sizes": [ |
|
16, |
|
16, |
|
16, |
|
32 |
|
], |
|
"device": null, |
|
"use_cache": "eval_cache", |
|
"limit": null, |
|
"bootstrap_iters": 100000, |
|
"gen_kwargs": null, |
|
"random_seed": 0, |
|
"numpy_seed": 1234, |
|
"torch_seed": 1234, |
|
"fewshot_seed": 1234 |
|
}, |
|
"git_hash": "0230356", |
|
"date": 1732986471.4917576, |
|
"pretty_env_info": "PyTorch version: 2.5.1+cu124\nIs debug build: False\nCUDA used to build PyTorch: 12.4\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.36\n\nPython version: 3.11.10 (main, Oct 3 2024, 07:29:13) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-6.1.0-26-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA GeForce GTX 1050 Ti\nGPU 1: Tesla P40\nGPU 2: Tesla V100-PCIE-32GB\nGPU 3: Tesla V100-PCIE-32GB\n\nNvidia driver version: 535.183.01\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 43 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 32\nOn-line CPU(s) list: 0-31\nVendor ID: AuthenticAMD\nModel name: AMD Ryzen Threadripper 1950X 16-Core Processor\nCPU family: 23\nModel: 1\nThread(s) per core: 2\nCore(s) per socket: 16\nSocket(s): 1\nStepping: 1\nFrequency boost: enabled\nCPU(s) scaling MHz: 66%\nCPU max MHz: 3400.0000\nCPU min MHz: 2200.0000\nBogoMIPS: 6786.43\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid extd_apicid amd_dcm aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb hw_pstate ssbd ibpb vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt sha_ni xsaveopt xsavec xgetbv1 clzero irperf xsaveerptr arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif overflow_recov succor smca sev\nVirtualization: AMD-V\nL1d cache: 512 KiB (16 instances)\nL1i cache: 1 MiB (16 instances)\nL2 cache: 8 MiB (16 instances)\nL3 cache: 32 MiB (4 instances)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-31\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT vulnerable\nVulnerability Spec rstack overflow: Mitigation; safe RET\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; IBPB conditional; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.1.3\n[pip3] torch==2.5.1\n[pip3] triton==3.1.0\n[conda] numpy 2.1.3 pypi_0 pypi\n[conda] torch 2.5.1 pypi_0 pypi\n[conda] triton 3.1.0 pypi_0 pypi", |
|
"transformers_version": "4.46.3", |
|
"upper_git_hash": null, |
|
"tokenizer_pad_token": [ |
|
"<|finetune_right_pad_id|>", |
|
"128004" |
|
], |
|
"tokenizer_eos_token": [ |
|
"<|eot_id|>", |
|
"128009" |
|
], |
|
"tokenizer_bos_token": [ |
|
"<|begin_of_text|>", |
|
"128000" |
|
], |
|
"eot_token_id": 128009, |
|
"max_length": 131072, |
|
"task_hashes": { |
|
"leaderboard_musr_murder_mysteries": "a696259562ea5c5c09a2613e30526fae1de29f55da9e28e8d7e8a53027e6d330", |
|
"leaderboard_musr_object_placements": "3aa8c5e5bc59cd6ba2326269b9f0bf3cee8cba1b4e9e1d1330cf5f1f59ea0dce", |
|
"leaderboard_musr_team_allocation": "5a75f135c145ee861a1cf31b63346709ef41b9d542be6a61c5818c210a3797a5" |
|
}, |
|
"model_source": "hf", |
|
"model_name": "DevQuasar/analytical_reasoning_r16a32_unsloth-Llama-3.2-3B-Instruct-bnb-4bit", |
|
"model_name_sanitized": "DevQuasar__analytical_reasoning_r16a32_unsloth-Llama-3.2-3B-Instruct-bnb-4bit", |
|
"system_instruction": null, |
|
"system_instruction_sha": null, |
|
"fewshot_as_multiturn": false, |
|
"chat_template": null, |
|
"chat_template_sha": null, |
|
"start_time": 52195.45405349, |
|
"end_time": 52407.302247922, |
|
"total_evaluation_time_seconds": "211.84819443200104" |
|
} |
