|
--- |
|
base_model: |
|
- meta-llama/Meta-Llama-3-8B |
|
- meta-llama/Meta-Llama-3-8B-Instruct |
|
library_name: transformers |
|
tags: |
|
- mergekit |
|
- merge |
|
|
|
--- |
|
# Untitled Model (1) |
|
|
|
This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit). |
|
|
|
## Merge Details |
|
### Merge Method |
|
|
|
This model was merged using the [TIES](https://arxiv.org/abs/2306.01708) merge method using [meta-llama/Meta-Llama-3-8B](https://huggingface.co/meta-llama/Meta-Llama-3-8B) as a base. |
|
|
|
### Models Merged |
|
|
|
The following models were included in the merge: |
|
* [meta-llama/Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) |
|
|
|
### Configuration |
|
|
|
The following YAML configuration was used to produce this model: |
|
|
|
```yaml |
|
base_model: meta-llama/Meta-Llama-3-8B |
|
dtype: bfloat16 |
|
merge_method: ties |
|
parameters: |
|
int8_mask: 1.0 |
|
normalize: 1.0 |
|
slices: |
|
- sources: |
|
- layer_range: [0, 32] |
|
model: meta-llama/Meta-Llama-3-8B |
|
- layer_range: [0, 32] |
|
model: meta-llama/Meta-Llama-3-8B-Instruct |
|
parameters: |
|
density: |
|
- filter: model.layers.0.self_attn.q_proj |
|
value: 1.0 |
|
- filter: model.layers.1.self_attn.q_proj |
|
value: 0.5915012245323329 |
|
- filter: model.layers.2.self_attn.q_proj |
|
value: 0.533635558334636 |
|
- filter: model.layers.3.self_attn.q_proj |
|
value: 0.5581262049919233 |
|
- filter: model.layers.4.self_attn.q_proj |
|
value: 0.5773018602469908 |
|
- filter: model.layers.5.self_attn.q_proj |
|
value: 0.5594288989630556 |
|
- filter: model.layers.6.self_attn.q_proj |
|
value: 0.5264707414934083 |
|
- filter: model.layers.7.self_attn.q_proj |
|
value: 0.5074253556354541 |
|
- filter: model.layers.8.self_attn.q_proj |
|
value: 0.5123234849669116 |
|
- filter: model.layers.9.self_attn.q_proj |
|
value: 0.549606586420718 |
|
- filter: model.layers.10.self_attn.q_proj |
|
value: 0.5104736595279037 |
|
- filter: model.layers.11.self_attn.q_proj |
|
value: 0.521937366473868 |
|
- filter: model.layers.12.self_attn.q_proj |
|
value: 0.5211036423323433 |
|
- filter: model.layers.13.self_attn.q_proj |
|
value: 0.5566932416236777 |
|
- filter: model.layers.14.self_attn.q_proj |
|
value: 0.5287113751237559 |
|
- filter: model.layers.15.self_attn.q_proj |
|
value: 0.5587254442186441 |
|
- filter: model.layers.16.self_attn.q_proj |
|
value: 0.5647699442446981 |
|
- filter: model.layers.17.self_attn.q_proj |
|
value: 0.5868636339951019 |
|
- filter: model.layers.18.self_attn.q_proj |
|
value: 0.590172476681778 |
|
- filter: model.layers.19.self_attn.q_proj |
|
value: 0.5772236986087228 |
|
- filter: model.layers.20.self_attn.q_proj |
|
value: 0.5837371684643844 |
|
- filter: model.layers.21.self_attn.q_proj |
|
value: 0.5988484185295191 |
|
- filter: model.layers.22.self_attn.q_proj |
|
value: 0.5956177374811109 |
|
- filter: model.layers.23.self_attn.q_proj |
|
value: 0.6109113647022042 |
|
- filter: model.layers.24.self_attn.q_proj |
|
value: 0.607680683653796 |
|
- filter: model.layers.25.self_attn.q_proj |
|
value: 0.612396435829295 |
|
- filter: model.layers.26.self_attn.q_proj |
|
value: 0.655307175238393 |
|
- filter: model.layers.27.self_attn.q_proj |
|
value: 0.6167734875722996 |
|
- filter: model.layers.28.self_attn.q_proj |
|
value: 0.6221405867333646 |
|
- filter: model.layers.29.self_attn.q_proj |
|
value: 0.5649783752800792 |
|
- filter: model.layers.30.self_attn.q_proj |
|
value: 0.5978323172320359 |
|
- filter: model.layers.31.self_attn.q_proj |
|
value: 0.5 |
|
- filter: model.layers.0.self_attn.k_proj |
|
value: 0.8480537752071284 |
|
- filter: model.layers.1.self_attn.k_proj |
|
value: 0.5569537804179042 |
|
- filter: model.layers.2.self_attn.k_proj |
|
value: 0.5319160022927414 |
|
- filter: model.layers.3.self_attn.k_proj |
|
value: 0.5251680475222761 |
|
- filter: model.layers.4.self_attn.k_proj |
|
value: 0.5493460476264915 |
|
- filter: model.layers.5.self_attn.k_proj |
|
value: 0.5465843364076911 |
|
- filter: model.layers.6.self_attn.k_proj |
|
value: 0.5242301078630608 |
|
- filter: model.layers.7.self_attn.k_proj |
|
value: 0.5145901724766818 |
|
- filter: model.layers.8.self_attn.k_proj |
|
value: 0.5039862435516649 |
|
- filter: model.layers.9.self_attn.k_proj |
|
value: 0.544526079933302 |
|
- filter: model.layers.10.self_attn.k_proj |
|
value: 0.5103694440102131 |
|
- filter: model.layers.11.self_attn.k_proj |
|
value: 0.5106560366838623 |
|
- filter: model.layers.12.self_attn.k_proj |
|
value: 0.5145641185972591 |
|
- filter: model.layers.13.self_attn.k_proj |
|
value: 0.5530717523839299 |
|
- filter: model.layers.14.self_attn.k_proj |
|
value: 0.5418685842321922 |
|
- filter: model.layers.15.self_attn.k_proj |
|
value: 0.5876713042572039 |
|
- filter: model.layers.16.self_attn.k_proj |
|
value: 0.6299046428013131 |
|
- filter: model.layers.17.self_attn.k_proj |
|
value: 0.6432963368245532 |
|
- filter: model.layers.18.self_attn.k_proj |
|
value: 0.6294877807305508 |
|
- filter: model.layers.19.self_attn.k_proj |
|
value: 0.604293679328852 |
|
- filter: model.layers.20.self_attn.k_proj |
|
value: 0.6142723151477255 |
|
- filter: model.layers.21.self_attn.k_proj |
|
value: 0.6172424574019072 |
|
- filter: model.layers.22.self_attn.k_proj |
|
value: 0.627559793653275 |
|
- filter: model.layers.23.self_attn.k_proj |
|
value: 0.6269345005471315 |
|
- filter: model.layers.24.self_attn.k_proj |
|
value: 0.6253973216611954 |
|
- filter: model.layers.25.self_attn.k_proj |
|
value: 0.6097128862487624 |
|
- filter: model.layers.26.self_attn.k_proj |
|
value: 0.654499504976291 |
|
- filter: model.layers.27.self_attn.k_proj |
|
value: 0.6062737741649732 |
|
- filter: model.layers.28.self_attn.k_proj |
|
value: 0.6051534573497994 |
|
- filter: model.layers.29.self_attn.k_proj |
|
value: 0.5576832890417384 |
|
- filter: model.layers.30.self_attn.k_proj |
|
value: 0.5832942525141994 |
|
- filter: model.layers.31.self_attn.k_proj |
|
value: 0.5001302693971132 |
|
- filter: model.layers.0.self_attn.v_proj |
|
value: 0.6780001042155177 |
|
- filter: model.layers.1.self_attn.v_proj |
|
value: 0.5999687353446929 |
|
- filter: model.layers.2.self_attn.v_proj |
|
value: 0.5700067740086499 |
|
- filter: model.layers.3.self_attn.v_proj |
|
value: 0.633447970402793 |
|
- filter: model.layers.4.self_attn.v_proj |
|
value: 0.6278203324475015 |
|
- filter: model.layers.5.self_attn.v_proj |
|
value: 0.5794382783596478 |
|
- filter: model.layers.6.self_attn.v_proj |
|
value: 0.5442655411390755 |
|
- filter: model.layers.7.self_attn.v_proj |
|
value: 0.5154499504976291 |
|
- filter: model.layers.8.self_attn.v_proj |
|
value: 0.5289719139179824 |
|
- filter: model.layers.9.self_attn.v_proj |
|
value: 0.522302120785785 |
|
- filter: model.layers.10.self_attn.v_proj |
|
value: 0.5297795841800844 |
|
- filter: model.layers.11.self_attn.v_proj |
|
value: 0.5429367932885206 |
|
- filter: model.layers.12.self_attn.v_proj |
|
value: 0.5740451253191601 |
|
- filter: model.layers.13.self_attn.v_proj |
|
value: 0.5515866812568392 |
|
- filter: model.layers.14.self_attn.v_proj |
|
value: 0.5319420561721641 |
|
- filter: model.layers.15.self_attn.v_proj |
|
value: 0.6064040435620864 |
|
- filter: model.layers.16.self_attn.v_proj |
|
value: 0.6014538064717837 |
|
- filter: model.layers.17.self_attn.v_proj |
|
value: 0.6136730759210046 |
|
- filter: model.layers.18.self_attn.v_proj |
|
value: 0.6215152936272211 |
|
- filter: model.layers.19.self_attn.v_proj |
|
value: 0.6304517742691886 |
|
- filter: model.layers.20.self_attn.v_proj |
|
value: 0.6145849617007972 |
|
- filter: model.layers.21.self_attn.v_proj |
|
value: 0.6035120629461727 |
|
- filter: model.layers.22.self_attn.v_proj |
|
value: 0.624537543640248 |
|
- filter: model.layers.23.self_attn.v_proj |
|
value: 0.6480641967588974 |
|
- filter: model.layers.24.self_attn.v_proj |
|
value: 0.6352456880829556 |
|
- filter: model.layers.25.self_attn.v_proj |
|
value: 0.5832681986347767 |
|
- filter: model.layers.26.self_attn.v_proj |
|
value: 0.6047887030378823 |
|
- filter: model.layers.27.self_attn.v_proj |
|
value: 0.5266010108905216 |
|
- filter: model.layers.28.self_attn.v_proj |
|
value: 0.5026835495805325 |
|
- filter: model.layers.29.self_attn.v_proj |
|
value: 0.5128966703142098 |
|
- filter: model.layers.30.self_attn.v_proj |
|
value: 0.559038090771716 |
|
- filter: model.layers.31.self_attn.v_proj |
|
value: 0.580011463706946 |
|
- filter: model.layers.0.self_attn.o_proj |
|
value: 0.5566150799854098 |
|
- filter: model.layers.1.self_attn.o_proj |
|
value: 0.516335782397999 |
|
- filter: model.layers.2.self_attn.o_proj |
|
value: 0.5357980303267157 |
|
- filter: model.layers.3.self_attn.o_proj |
|
value: 0.5400187587931843 |
|
- filter: model.layers.4.self_attn.o_proj |
|
value: 0.6263613151998333 |
|
- filter: model.layers.5.self_attn.o_proj |
|
value: 0.5280339742587671 |
|
- filter: model.layers.6.self_attn.o_proj |
|
value: 0.5044812672606951 |
|
- filter: model.layers.7.self_attn.o_proj |
|
value: 0.5143296336824553 |
|
- filter: model.layers.8.self_attn.o_proj |
|
value: 0.5011984784534417 |
|
- filter: model.layers.9.self_attn.o_proj |
|
value: 0.5095878276275337 |
|
- filter: model.layers.10.self_attn.o_proj |
|
value: 0.558595174821531 |
|
- filter: model.layers.11.self_attn.o_proj |
|
value: 0.5168047522276067 |
|
- filter: model.layers.12.self_attn.o_proj |
|
value: 0.586264394768381 |
|
- filter: model.layers.13.self_attn.o_proj |
|
value: 0.5044812672606951 |
|
- filter: model.layers.14.self_attn.o_proj |
|
value: 0.5224844979417436 |
|
- filter: model.layers.15.self_attn.o_proj |
|
value: 0.512584023761138 |
|
- filter: model.layers.16.self_attn.o_proj |
|
value: 0.5946016361836277 |
|
- filter: model.layers.17.self_attn.o_proj |
|
value: 0.5662289614923662 |
|
- filter: model.layers.18.self_attn.o_proj |
|
value: 0.5052368297639519 |
|
- filter: model.layers.19.self_attn.o_proj |
|
value: 0.5294669376270127 |
|
- filter: model.layers.20.self_attn.o_proj |
|
value: 0.6374602678338804 |
|
- filter: model.layers.21.self_attn.o_proj |
|
value: 0.583763222343807 |
|
- filter: model.layers.22.self_attn.o_proj |
|
value: 0.694752748684279 |
|
- filter: model.layers.23.self_attn.o_proj |
|
value: 0.5834505757907352 |
|
- filter: model.layers.24.self_attn.o_proj |
|
value: 0.57493095721953 |
|
- filter: model.layers.25.self_attn.o_proj |
|
value: 0.9771507477463395 |
|
- filter: model.layers.26.self_attn.o_proj |
|
value: 0.5683132718461779 |
|
- filter: model.layers.27.self_attn.o_proj |
|
value: 0.8806992861237039 |
|
- filter: model.layers.28.self_attn.o_proj |
|
value: 0.585013808556094 |
|
- filter: model.layers.29.self_attn.o_proj |
|
value: 0.7172111927466 |
|
- filter: model.layers.30.self_attn.o_proj |
|
value: 0.539158980772237 |
|
- filter: model.layers.31.self_attn.o_proj |
|
value: 0.5273565733937783 |
|
- filter: model.layers.0.mlp.gate_proj |
|
value: 0.7398259600854568 |
|
- filter: model.layers.1.mlp.gate_proj |
|
value: 0.6623156688030848 |
|
- filter: model.layers.2.mlp.gate_proj |
|
value: 0.6218800479391382 |
|
- filter: model.layers.3.mlp.gate_proj |
|
value: 0.6496795372831015 |
|
- filter: model.layers.4.mlp.gate_proj |
|
value: 0.6073419832213016 |
|
- filter: model.layers.5.mlp.gate_proj |
|
value: 0.5901203689229326 |
|
- filter: model.layers.6.mlp.gate_proj |
|
value: 0.5831639831170862 |
|
- filter: model.layers.7.mlp.gate_proj |
|
value: 0.5327757803136887 |
|
- filter: model.layers.8.mlp.gate_proj |
|
value: 0.5048460215726122 |
|
- filter: model.layers.9.mlp.gate_proj |
|
value: 0.517299775936637 |
|
- filter: model.layers.10.mlp.gate_proj |
|
value: 0.5256109634724611 |
|
- filter: model.layers.11.mlp.gate_proj |
|
value: 0.5363191079151686 |
|
- filter: model.layers.12.mlp.gate_proj |
|
value: 0.5258454483872649 |
|
- filter: model.layers.13.mlp.gate_proj |
|
value: 0.5099004741806055 |
|
- filter: model.layers.14.mlp.gate_proj |
|
value: 0.544161325621385 |
|
- filter: model.layers.15.mlp.gate_proj |
|
value: 0.5567974571413683 |
|
- filter: model.layers.16.mlp.gate_proj |
|
value: 0.5865249335626075 |
|
- filter: model.layers.17.mlp.gate_proj |
|
value: 0.6023917461309989 |
|
- filter: model.layers.18.mlp.gate_proj |
|
value: 0.6177114272315147 |
|
- filter: model.layers.19.mlp.gate_proj |
|
value: 0.614011776353499 |
|
- filter: model.layers.20.mlp.gate_proj |
|
value: 0.6049189724349956 |
|
- filter: model.layers.21.mlp.gate_proj |
|
value: 0.5991871189620135 |
|
- filter: model.layers.22.mlp.gate_proj |
|
value: 0.6197957375853265 |
|
- filter: model.layers.23.mlp.gate_proj |
|
value: 0.6167734875722996 |
|
- filter: model.layers.24.mlp.gate_proj |
|
value: 0.5977281017143453 |
|
- filter: model.layers.25.mlp.gate_proj |
|
value: 0.6520764941899849 |
|
- filter: model.layers.26.mlp.gate_proj |
|
value: 0.5683653796050232 |
|
- filter: model.layers.27.mlp.gate_proj |
|
value: 0.5564587567088739 |
|
- filter: model.layers.28.mlp.gate_proj |
|
value: 0.5045333750195404 |
|
- filter: model.layers.29.mlp.gate_proj |
|
value: 0.5245427544161325 |
|
- filter: model.layers.30.mlp.gate_proj |
|
value: 0.5658902610598718 |
|
- filter: model.layers.31.mlp.gate_proj |
|
value: 0.5598978687926632 |
|
- filter: model.layers.0.mlp.up_proj |
|
value: 0.7446198738992236 |
|
- filter: model.layers.1.mlp.up_proj |
|
value: 0.6871971236517117 |
|
- filter: model.layers.2.mlp.up_proj |
|
value: 0.6360012505862123 |
|
- filter: model.layers.3.mlp.up_proj |
|
value: 0.6525194101401699 |
|
- filter: model.layers.4.mlp.up_proj |
|
value: 0.5973112396435829 |
|
- filter: model.layers.5.mlp.up_proj |
|
value: 0.5818873430253765 |
|
- filter: model.layers.6.mlp.up_proj |
|
value: 0.5714397373768955 |
|
- filter: model.layers.7.mlp.up_proj |
|
value: 0.5230576832890418 |
|
- filter: model.layers.8.mlp.up_proj |
|
value: 0.5084935646917826 |
|
- filter: model.layers.9.mlp.up_proj |
|
value: 0.5306393622010317 |
|
- filter: model.layers.10.mlp.up_proj |
|
value: 0.5488249700380387 |
|
- filter: model.layers.11.mlp.up_proj |
|
value: 0.5646657287270075 |
|
- filter: model.layers.12.mlp.up_proj |
|
value: 0.557266426970976 |
|
- filter: model.layers.13.mlp.up_proj |
|
value: 0.5164399979156896 |
|
- filter: model.layers.14.mlp.up_proj |
|
value: 0.5224584440623209 |
|
- filter: model.layers.15.mlp.up_proj |
|
value: 0.5427283622531395 |
|
- filter: model.layers.16.mlp.up_proj |
|
value: 0.5844927309676411 |
|
- filter: model.layers.17.mlp.up_proj |
|
value: 0.5983012870616435 |
|
- filter: model.layers.18.mlp.up_proj |
|
value: 0.6132301599708196 |
|
- filter: model.layers.19.mlp.up_proj |
|
value: 0.6065603668386222 |
|
- filter: model.layers.20.mlp.up_proj |
|
value: 0.598040748267417 |
|
- filter: model.layers.21.mlp.up_proj |
|
value: 0.5880360585691209 |
|
- filter: model.layers.22.mlp.up_proj |
|
value: 0.6064561513209317 |
|
- filter: model.layers.23.mlp.up_proj |
|
value: 0.6022093689750404 |
|
- filter: model.layers.24.mlp.up_proj |
|
value: 0.580376218018863 |
|
- filter: model.layers.25.mlp.up_proj |
|
value: 0.6365223281746653 |
|
- filter: model.layers.26.mlp.up_proj |
|
value: 0.5579698817153874 |
|
- filter: model.layers.27.mlp.up_proj |
|
value: 0.5470272523578761 |
|
- filter: model.layers.28.mlp.up_proj |
|
value: 0.5102131207336772 |
|
- filter: model.layers.29.mlp.up_proj |
|
value: 0.5205044031056224 |
|
- filter: model.layers.30.mlp.up_proj |
|
value: 0.555651086446772 |
|
- filter: model.layers.31.mlp.up_proj |
|
value: 0.6076025220155281 |
|
- filter: model.layers.0.mlp.down_proj |
|
value: 0.5857954249387733 |
|
- filter: model.layers.1.mlp.down_proj |
|
value: 0.5853264551091657 |
|
- filter: model.layers.2.mlp.down_proj |
|
value: 0.5342087436819343 |
|
- filter: model.layers.3.mlp.down_proj |
|
value: 0.5606534312959199 |
|
- filter: model.layers.4.mlp.down_proj |
|
value: 0.566541608045438 |
|
- filter: model.layers.5.mlp.down_proj |
|
value: 0.5318899484133187 |
|
- filter: model.layers.6.mlp.down_proj |
|
value: 0.538351310510135 |
|
- filter: model.layers.7.mlp.down_proj |
|
value: 0.5209994268146527 |
|
- filter: model.layers.8.mlp.down_proj |
|
value: 0.5305351466833411 |
|
- filter: model.layers.9.mlp.down_proj |
|
value: 0.503673596998593 |
|
- filter: model.layers.10.mlp.down_proj |
|
value: 0.5296232609035485 |
|
- filter: model.layers.11.mlp.down_proj |
|
value: 0.5282163514147257 |
|
- filter: model.layers.12.mlp.down_proj |
|
value: 0.549606586420718 |
|
- filter: model.layers.13.mlp.down_proj |
|
value: 0.5241519462247929 |
|
- filter: model.layers.14.mlp.down_proj |
|
value: 0.5465582825282684 |
|
- filter: model.layers.15.mlp.down_proj |
|
value: 0.5839977072586108 |
|
- filter: model.layers.16.mlp.down_proj |
|
value: 0.5891824292637173 |
|
- filter: model.layers.17.mlp.down_proj |
|
value: 0.570215205044031 |
|
- filter: model.layers.18.mlp.down_proj |
|
value: 0.5918920327236725 |
|
- filter: model.layers.19.mlp.down_proj |
|
value: 0.6033557396696368 |
|
- filter: model.layers.20.mlp.down_proj |
|
value: 0.5831900369965088 |
|
- filter: model.layers.21.mlp.down_proj |
|
value: 0.6060392892501694 |
|
- filter: model.layers.22.mlp.down_proj |
|
value: 0.5592465218070971 |
|
- filter: model.layers.23.mlp.down_proj |
|
value: 0.5628159032879996 |
|
- filter: model.layers.24.mlp.down_proj |
|
value: 0.5611484550049503 |
|
- filter: model.layers.25.mlp.down_proj |
|
value: 0.6383460997342505 |
|
- filter: model.layers.26.mlp.down_proj |
|
value: 0.5379865561982179 |
|
- filter: model.layers.27.mlp.down_proj |
|
value: 0.547183575634412 |
|
- filter: model.layers.28.mlp.down_proj |
|
value: 0.5039862435516649 |
|
- filter: model.layers.29.mlp.down_proj |
|
value: 0.8178312750768589 |
|
- filter: model.layers.30.mlp.down_proj |
|
value: 0.7657756239904121 |
|
- filter: model.layers.31.mlp.down_proj |
|
value: 0.8403939346568705 |
|
- value: 1.0 |
|
weight: |
|
- value: 1.0 |
|
tokenizer_source: union |
|
``` |
|
|
