| base_model: meta-llama/Llama-3.1-8B | |
| dtype: bfloat16 | |
| merge_method: ties | |
| parameters: | |
| int8_mask: 1.0 | |
| normalize: 1.0 | |
| slices: | |
| - sources: | |
| - layer_range: [0, 32] | |
| model: meta-llama/Llama-3.1-8B | |
| - layer_range: [0, 32] | |
| model: MathGenie/MathCoder2-Llama-3-8B | |
| parameters: | |
| density: | |
| - value: 1.0 | |
| weight: | |
| - filter: model.layers.0.self_attn.q_proj | |
| value: 0.34567118839365213 | |
| - filter: model.layers.1.self_attn.q_proj | |
| value: 0.3194050175456974 | |
| - filter: model.layers.2.self_attn.q_proj | |
| value: 0.35258471691195725 | |
| - filter: model.layers.3.self_attn.q_proj | |
| value: 0.3345938301995496 | |
| - filter: model.layers.4.self_attn.q_proj | |
| value: 0.30193788299376734 | |
| - filter: model.layers.5.self_attn.q_proj | |
| value: 0.3173885717278584 | |
| - filter: model.layers.6.self_attn.q_proj | |
| value: 0.3099251034410517 | |
| - filter: model.layers.7.self_attn.q_proj | |
| value: 0.32521866652699943 | |
| - filter: model.layers.8.self_attn.q_proj | |
| value: 0.3688734091028125 | |
| - filter: model.layers.9.self_attn.q_proj | |
| value: 0.3703137275441261 | |
| - filter: model.layers.10.self_attn.q_proj | |
| value: 0.31995495731419893 | |
| - filter: model.layers.11.self_attn.q_proj | |
| value: 0.3247211019745456 | |
| - filter: model.layers.12.self_attn.q_proj | |
| value: 0.31401037029277745 | |
| - filter: model.layers.13.self_attn.q_proj | |
| value: 0.3123343633792489 | |
| - filter: model.layers.14.self_attn.q_proj | |
| value: 0.30020950086419107 | |
| - filter: model.layers.15.self_attn.q_proj | |
| value: 0.3220499659561096 | |
| - filter: model.layers.16.self_attn.q_proj | |
| value: 0.3007856282407165 | |
| - filter: model.layers.17.self_attn.q_proj | |
| value: 0.3021473838579584 | |
| - filter: model.layers.18.self_attn.q_proj | |
| value: 0.3078562824071649 | |
| - filter: model.layers.19.self_attn.q_proj | |
| value: 0.3117844236107474 | |
| - filter: model.layers.20.self_attn.q_proj | |
| value: 0.3057088985492065 | |
| - filter: model.layers.21.self_attn.q_proj | |
| value: 0.3555962918347038 | |
| - filter: model.layers.22.self_attn.q_proj | |
| value: 0.3584245535012832 | |
| - filter: model.layers.23.self_attn.q_proj | |
| value: 0.4073430052898968 | |
| - filter: model.layers.24.self_attn.q_proj | |
| value: 0.451652438066307 | |
| - filter: model.layers.25.self_attn.q_proj | |
| value: 0.47265489970146124 | |
| - filter: model.layers.26.self_attn.q_proj | |
| value: 0.5172262085581103 | |
| - filter: model.layers.27.self_attn.q_proj | |
| value: 0.5207091604252867 | |
| - filter: model.layers.28.self_attn.q_proj | |
| value: 0.5678730424763002 | |
| - filter: model.layers.29.self_attn.q_proj | |
| value: 0.5689467344052794 | |
| - filter: model.layers.30.self_attn.q_proj | |
| value: 0.7031844131357041 | |
| - filter: model.layers.31.self_attn.q_proj | |
| value: 0.5159692033729639 | |
| - filter: model.layers.0.self_attn.k_proj | |
| value: 0.32548054260723824 | |
| - filter: model.layers.1.self_attn.k_proj | |
| value: 0.32346409678939925 | |
| - filter: model.layers.2.self_attn.k_proj | |
| value: 0.3201120829623422 | |
| - filter: model.layers.3.self_attn.k_proj | |
| value: 0.30683496569423346 | |
| - filter: model.layers.4.self_attn.k_proj | |
| value: 0.31762426020007334 | |
| - filter: model.layers.5.self_attn.k_proj | |
| value: 0.30327345100298536 | |
| - filter: model.layers.6.self_attn.k_proj | |
| value: 0.3263185460640025 | |
| - filter: model.layers.7.self_attn.k_proj | |
| value: 0.3229665322369455 | |
| - filter: model.layers.8.self_attn.k_proj | |
| value: 0.3647619546430629 | |
| - filter: model.layers.9.self_attn.k_proj | |
| value: 0.3679830304300005 | |
| - filter: model.layers.10.self_attn.k_proj | |
| value: 0.34794951029172994 | |
| - filter: model.layers.11.self_attn.k_proj | |
| value: 0.3474519457392762 | |
| - filter: model.layers.12.self_attn.k_proj | |
| value: 0.3480018855077777 | |
| - filter: model.layers.13.self_attn.k_proj | |
| value: 0.38031739380924945 | |
| - filter: model.layers.14.self_attn.k_proj | |
| value: 0.35365840884093647 | |
| - filter: model.layers.15.self_attn.k_proj | |
| value: 0.35551772901063217 | |
| - filter: model.layers.16.self_attn.k_proj | |
| value: 0.3715707327292725 | |
| - filter: model.layers.17.self_attn.k_proj | |
| value: 0.3642382024825852 | |
| - filter: model.layers.18.self_attn.k_proj | |
| value: 0.3526109045199811 | |
| - filter: model.layers.19.self_attn.k_proj | |
| value: 0.3439428062640758 | |
| - filter: model.layers.20.self_attn.k_proj | |
| value: 0.3556224794427277 | |
| - filter: model.layers.21.self_attn.k_proj | |
| value: 0.390713874194731 | |
| - filter: model.layers.22.self_attn.k_proj | |
| value: 0.38280521657151834 | |
| - filter: model.layers.23.self_attn.k_proj | |
| value: 0.3970774629445346 | |
| - filter: model.layers.24.self_attn.k_proj | |
| value: 0.4266170847954748 | |
| - filter: model.layers.25.self_attn.k_proj | |
| value: 0.4180275493636411 | |
| - filter: model.layers.26.self_attn.k_proj | |
| value: 0.4680196930812339 | |
| - filter: model.layers.27.self_attn.k_proj | |
| value: 0.4603205363222123 | |
| - filter: model.layers.28.self_attn.k_proj | |
| value: 0.49577855758655004 | |
| - filter: model.layers.29.self_attn.k_proj | |
| value: 0.49716650081181585 | |
| - filter: model.layers.30.self_attn.k_proj | |
| value: 0.610715969203373 | |
| - filter: model.layers.31.self_attn.k_proj | |
| value: 0.46008484784999737 | |
| - filter: model.layers.0.self_attn.v_proj | |
| value: 0.3039805164196302 | |
| - filter: model.layers.1.self_attn.v_proj | |
| value: 0.31008222908919497 | |
| - filter: model.layers.2.self_attn.v_proj | |
| value: 0.3261614204158592 | |
| - filter: model.layers.3.self_attn.v_proj | |
| value: 0.34339286649557427 | |
| - filter: model.layers.4.self_attn.v_proj | |
| value: 0.33574608495260044 | |
| - filter: model.layers.5.self_attn.v_proj | |
| value: 0.3314251296286597 | |
| - filter: model.layers.6.self_attn.v_proj | |
| value: 0.3357722725606243 | |
| - filter: model.layers.7.self_attn.v_proj | |
| value: 0.3345938301995496 | |
| - filter: model.layers.8.self_attn.v_proj | |
| value: 0.33404389043104804 | |
| - filter: model.layers.9.self_attn.v_proj | |
| value: 0.36114806473576705 | |
| - filter: model.layers.10.self_attn.v_proj | |
| value: 0.3461163777300581 | |
| - filter: model.layers.11.self_attn.v_proj | |
| value: 0.34881370135651807 | |
| - filter: model.layers.12.self_attn.v_proj | |
| value: 0.35465353794584403 | |
| - filter: model.layers.13.self_attn.v_proj | |
| value: 0.3599958099827162 | |
| - filter: model.layers.14.self_attn.v_proj | |
| value: 0.36486670507515845 | |
| - filter: model.layers.15.self_attn.v_proj | |
| value: 0.3886450531608443 | |
| - filter: model.layers.16.self_attn.v_proj | |
| value: 0.3802650185932017 | |
| - filter: model.layers.17.self_attn.v_proj | |
| value: 0.3948253286544807 | |
| - filter: model.layers.18.self_attn.v_proj | |
| value: 0.4028911119258367 | |
| - filter: model.layers.19.self_attn.v_proj | |
| value: 0.41336615513538993 | |
| - filter: model.layers.20.self_attn.v_proj | |
| value: 0.4098832032682135 | |
| - filter: model.layers.21.self_attn.v_proj | |
| value: 0.39906772115434974 | |
| - filter: model.layers.22.self_attn.v_proj | |
| value: 0.4269051484837375 | |
| - filter: model.layers.23.self_attn.v_proj | |
| value: 0.4284502173571466 | |
| - filter: model.layers.24.self_attn.v_proj | |
| value: 0.43452574241868747 | |
| - filter: model.layers.25.self_attn.v_proj | |
| value: 0.4286073430052899 | |
| - filter: model.layers.26.self_attn.v_proj | |
| value: 0.421196249934531 | |
| - filter: model.layers.27.self_attn.v_proj | |
| value: 0.43709212800502806 | |
| - filter: model.layers.28.self_attn.v_proj | |
| value: 0.4468339181899125 | |
| - filter: model.layers.29.self_attn.v_proj | |
| value: 0.4439009060912376 | |
| - filter: model.layers.30.self_attn.v_proj | |
| value: 0.4961451840988844 | |
| - filter: model.layers.31.self_attn.v_proj | |
| value: 0.36374063793013145 | |
| - filter: model.layers.0.self_attn.o_proj | |
| value: 0.301335568009218 | |
| - filter: model.layers.1.self_attn.o_proj | |
| value: 0.30623265070968414 | |
| - filter: model.layers.2.self_attn.o_proj | |
| value: 0.3042685801078929 | |
| - filter: model.layers.3.self_attn.o_proj | |
| value: 0.3 | |
| - filter: model.layers.4.self_attn.o_proj | |
| value: 0.32671136018436076 | |
| - filter: model.layers.5.self_attn.o_proj | |
| value: 0.3351961451840989 | |
| - filter: model.layers.6.self_attn.o_proj | |
| value: 0.30518514638872885 | |
| - filter: model.layers.7.self_attn.o_proj | |
| value: 0.3552820405384172 | |
| - filter: model.layers.8.self_attn.o_proj | |
| value: 0.358241240245116 | |
| - filter: model.layers.9.self_attn.o_proj | |
| value: 0.35758655004451895 | |
| - filter: model.layers.10.self_attn.o_proj | |
| value: 0.3841669721887603 | |
| - filter: model.layers.11.self_attn.o_proj | |
| value: 0.3434714293196459 | |
| - filter: model.layers.12.self_attn.o_proj | |
| value: 0.38160058660241974 | |
| - filter: model.layers.13.self_attn.o_proj | |
| value: 0.3708374797046038 | |
| - filter: model.layers.14.self_attn.o_proj | |
| value: 0.4145707851044885 | |
| - filter: model.layers.15.self_attn.o_proj | |
| value: 0.39864871942596763 | |
| - filter: model.layers.16.self_attn.o_proj | |
| value: 0.41519928769706177 | |
| - filter: model.layers.17.self_attn.o_proj | |
| value: 0.31864557691300477 | |
| - filter: model.layers.18.self_attn.o_proj | |
| value: 0.3101084166972189 | |
| - filter: model.layers.19.self_attn.o_proj | |
| value: 0.3196145184098884 | |
| - filter: model.layers.20.self_attn.o_proj | |
| value: 0.4667365002880637 | |
| - filter: model.layers.21.self_attn.o_proj | |
| value: 0.3742418687477086 | |
| - filter: model.layers.22.self_attn.o_proj | |
| value: 0.3403812915728277 | |
| - filter: model.layers.23.self_attn.o_proj | |
| value: 0.3232284083171843 | |
| - filter: model.layers.24.self_attn.o_proj | |
| value: 0.3742418687477086 | |
| - filter: model.layers.25.self_attn.o_proj | |
| value: 0.6244644634159116 | |
| - filter: model.layers.26.self_attn.o_proj | |
| value: 0.5438066307023517 | |
| - filter: model.layers.27.self_attn.o_proj | |
| value: 0.542366312261038 | |
| - filter: model.layers.28.self_attn.o_proj | |
| value: 0.4544021369088147 | |
| - filter: model.layers.29.self_attn.o_proj | |
| value: 0.4355732467396428 | |
| - filter: model.layers.30.self_attn.o_proj | |
| value: 0.4489027392237993 | |
| - filter: model.layers.31.self_attn.o_proj | |
| value: 0.3717016707693919 | |
| - filter: model.layers.0.mlp.gate_proj | |
| value: 0.321735714659823 | |
| - filter: model.layers.1.mlp.gate_proj | |
| value: 0.32713036191274286 | |
| - filter: model.layers.2.mlp.gate_proj | |
| value: 0.3402765411407322 | |
| - filter: model.layers.3.mlp.gate_proj | |
| value: 0.34344524171162205 | |
| - filter: model.layers.4.mlp.gate_proj | |
| value: 0.34572356360969986 | |
| - filter: model.layers.5.mlp.gate_proj | |
| value: 0.34881370135651807 | |
| - filter: model.layers.6.mlp.gate_proj | |
| value: 0.3491017650447808 | |
| - filter: model.layers.7.mlp.gate_proj | |
| value: 0.34637825381029697 | |
| - filter: model.layers.8.mlp.gate_proj | |
| value: 0.351301524118787 | |
| - filter: model.layers.9.mlp.gate_proj | |
| value: 0.35083014717435707 | |
| - filter: model.layers.10.mlp.gate_proj | |
| value: 0.35062064631016604 | |
| - filter: model.layers.11.mlp.gate_proj | |
| value: 0.35083014717435707 | |
| - filter: model.layers.12.mlp.gate_proj | |
| value: 0.35724611114020843 | |
| - filter: model.layers.13.mlp.gate_proj | |
| value: 0.36078143822343267 | |
| - filter: model.layers.14.mlp.gate_proj | |
| value: 0.3717278583774158 | |
| - filter: model.layers.15.mlp.gate_proj | |
| value: 0.3819410255067302 | |
| - filter: model.layers.16.mlp.gate_proj | |
| value: 0.3969203372963913 | |
| - filter: model.layers.17.mlp.gate_proj | |
| value: 0.4064526266170848 | |
| - filter: model.layers.18.mlp.gate_proj | |
| value: 0.4028387367097889 | |
| - filter: model.layers.19.mlp.gate_proj | |
| value: 0.40865238569109097 | |
| - filter: model.layers.20.mlp.gate_proj | |
| value: 0.4113758969255748 | |
| - filter: model.layers.21.mlp.gate_proj | |
| value: 0.4152778505211334 | |
| - filter: model.layers.22.mlp.gate_proj | |
| value: 0.41237102603048237 | |
| - filter: model.layers.23.mlp.gate_proj | |
| value: 0.4021316712931441 | |
| - filter: model.layers.24.mlp.gate_proj | |
| value: 0.3956895197192688 | |
| - filter: model.layers.25.mlp.gate_proj | |
| value: 0.407290630073849 | |
| - filter: model.layers.26.mlp.gate_proj | |
| value: 0.41842036348399936 | |
| - filter: model.layers.27.mlp.gate_proj | |
| value: 0.43950138794322524 | |
| - filter: model.layers.28.mlp.gate_proj | |
| value: 0.4232912585764416 | |
| - filter: model.layers.29.mlp.gate_proj | |
| value: 0.43353061331377996 | |
| - filter: model.layers.30.mlp.gate_proj | |
| value: 0.4429843398104017 | |
| - filter: model.layers.31.mlp.gate_proj | |
| value: 0.39241606871628343 | |
| - filter: model.layers.0.mlp.up_proj | |
| value: 0.31846226365683755 | |
| - filter: model.layers.1.mlp.up_proj | |
| value: 0.3282826166657937 | |
| - filter: model.layers.2.mlp.up_proj | |
| value: 0.34148117110983084 | |
| - filter: model.layers.3.mlp.up_proj | |
| value: 0.34344524171162205 | |
| - filter: model.layers.4.mlp.up_proj | |
| value: 0.3439951814801236 | |
| - filter: model.layers.5.mlp.up_proj | |
| value: 0.3482899491960404 | |
| - filter: model.layers.6.mlp.up_proj | |
| value: 0.34933745351699574 | |
| - filter: model.layers.7.mlp.up_proj | |
| value: 0.34700675640287015 | |
| - filter: model.layers.8.mlp.up_proj | |
| value: 0.35025401979783166 | |
| - filter: model.layers.9.mlp.up_proj | |
| value: 0.34996595610956893 | |
| - filter: model.layers.10.mlp.up_proj | |
| value: 0.3525585293039334 | |
| - filter: model.layers.11.mlp.up_proj | |
| value: 0.3536060336248887 | |
| - filter: model.layers.12.mlp.up_proj | |
| value: 0.358686429581522 | |
| - filter: model.layers.13.mlp.up_proj | |
| value: 0.3634525742418687 | |
| - filter: model.layers.14.mlp.up_proj | |
| value: 0.3718849840255591 | |
| - filter: model.layers.15.mlp.up_proj | |
| value: 0.38311946786780493 | |
| - filter: model.layers.16.mlp.up_proj | |
| value: 0.3967108364322003 | |
| - filter: model.layers.17.mlp.up_proj | |
| value: 0.4041481171109831 | |
| - filter: model.layers.18.mlp.up_proj | |
| value: 0.4013460430524276 | |
| - filter: model.layers.19.mlp.up_proj | |
| value: 0.40493374535169957 | |
| - filter: model.layers.20.mlp.up_proj | |
| value: 0.408861886555282 | |
| - filter: model.layers.21.mlp.up_proj | |
| value: 0.41176871104593304 | |
| - filter: model.layers.22.mlp.up_proj | |
| value: 0.4101974545645001 | |
| - filter: model.layers.23.mlp.up_proj | |
| value: 0.4023411721573351 | |
| - filter: model.layers.24.mlp.up_proj | |
| value: 0.39652752317603307 | |
| - filter: model.layers.25.mlp.up_proj | |
| value: 0.40742156811396846 | |
| - filter: model.layers.26.mlp.up_proj | |
| value: 0.4191274289006442 | |
| - filter: model.layers.27.mlp.up_proj | |
| value: 0.4394490127271775 | |
| - filter: model.layers.28.mlp.up_proj | |
| value: 0.42931440842193475 | |
| - filter: model.layers.29.mlp.up_proj | |
| value: 0.44222489917770913 | |
| - filter: model.layers.30.mlp.up_proj | |
| value: 0.4493741161682292 | |
| - filter: model.layers.31.mlp.up_proj | |
| value: 0.3794532027444613 | |
| - filter: model.layers.0.mlp.down_proj | |
| value: 0.3215785890116797 | |
| - filter: model.layers.1.mlp.down_proj | |
| value: 0.34407374430419524 | |
| - filter: model.layers.2.mlp.down_proj | |
| value: 0.3128319279317027 | |
| - filter: model.layers.3.mlp.down_proj | |
| value: 0.32330697114125595 | |
| - filter: model.layers.4.mlp.down_proj | |
| value: 0.3263185460640025 | |
| - filter: model.layers.5.mlp.down_proj | |
| value: 0.3317393809249463 | |
| - filter: model.layers.6.mlp.down_proj | |
| value: 0.3353008956161944 | |
| - filter: model.layers.7.mlp.down_proj | |
| value: 0.3378934688105588 | |
| - filter: model.layers.8.mlp.down_proj | |
| value: 0.3421358613104279 | |
| - filter: model.layers.9.mlp.down_proj | |
| value: 0.3419263604462368 | |
| - filter: model.layers.10.mlp.down_proj | |
| value: 0.3389933483475619 | |
| - filter: model.layers.11.mlp.down_proj | |
| value: 0.33818153249882155 | |
| - filter: model.layers.12.mlp.down_proj | |
| value: 0.3378934688105588 | |
| - filter: model.layers.13.mlp.down_proj | |
| value: 0.35009689414968836 | |
| - filter: model.layers.14.mlp.down_proj | |
| value: 0.3351437699680511 | |
| - filter: model.layers.15.mlp.down_proj | |
| value: 0.3680354056460483 | |
| - filter: model.layers.16.mlp.down_proj | |
| value: 0.3725920494422039 | |
| - filter: model.layers.17.mlp.down_proj | |
| value: 0.36947572408736185 | |
| - filter: model.layers.18.mlp.down_proj | |
| value: 0.3621693814486985 | |
| - filter: model.layers.19.mlp.down_proj | |
| value: 0.3649452678992301 | |
| - filter: model.layers.20.mlp.down_proj | |
| value: 0.36468339181899123 | |
| - filter: model.layers.21.mlp.down_proj | |
| value: 0.3756036243649505 | |
| - filter: model.layers.22.mlp.down_proj | |
| value: 0.35449641229770074 | |
| - filter: model.layers.23.mlp.down_proj | |
| value: 0.34637825381029697 | |
| - filter: model.layers.24.mlp.down_proj | |
| value: 0.3522442780076468 | |
| - filter: model.layers.25.mlp.down_proj | |
| value: 0.3727491750903472 | |
| - filter: model.layers.26.mlp.down_proj | |
| value: 0.4206986853820772 | |
| - filter: model.layers.27.mlp.down_proj | |
| value: 0.4077620070182789 | |
| - filter: model.layers.28.mlp.down_proj | |
| value: 0.42363169748075213 | |
| - filter: model.layers.29.mlp.down_proj | |
| value: 0.534091028125491 | |
| - filter: model.layers.30.mlp.down_proj | |
| value: 0.6001361755617243 | |
| - filter: model.layers.31.mlp.down_proj | |
| value: 0.8 | |
| - value: 1.0 | |
| tokenizer_source: union |