Fredithefish
commited on
Commit
•
6485ec6
1
Parent(s):
ebfbbe1
Upload folder using huggingface_hub
Browse filesThis view is limited to 50 files because it contains too many changes.
See raw diff
- config.json +30 -0
- mergekit_config.yml +17 -0
- model-00001-of-00048.safetensors +3 -0
- model-00002-of-00048.safetensors +3 -0
- model-00003-of-00048.safetensors +3 -0
- model-00004-of-00048.safetensors +3 -0
- model-00005-of-00048.safetensors +3 -0
- model-00006-of-00048.safetensors +3 -0
- model-00007-of-00048.safetensors +3 -0
- model-00008-of-00048.safetensors +3 -0
- model-00009-of-00048.safetensors +3 -0
- model-00010-of-00048.safetensors +3 -0
- model-00011-of-00048.safetensors +3 -0
- model-00012-of-00048.safetensors +3 -0
- model-00013-of-00048.safetensors +3 -0
- model-00014-of-00048.safetensors +3 -0
- model-00015-of-00048.safetensors +3 -0
- model-00016-of-00048.safetensors +3 -0
- model-00017-of-00048.safetensors +3 -0
- model-00018-of-00048.safetensors +3 -0
- model-00019-of-00048.safetensors +3 -0
- model-00020-of-00048.safetensors +3 -0
- model-00021-of-00048.safetensors +3 -0
- model-00022-of-00048.safetensors +3 -0
- model-00023-of-00048.safetensors +3 -0
- model-00024-of-00048.safetensors +3 -0
- model-00025-of-00048.safetensors +3 -0
- model-00026-of-00048.safetensors +3 -0
- model-00027-of-00048.safetensors +3 -0
- model-00028-of-00048.safetensors +3 -0
- model-00029-of-00048.safetensors +3 -0
- model-00030-of-00048.safetensors +3 -0
- model-00031-of-00048.safetensors +3 -0
- model-00032-of-00048.safetensors +3 -0
- model-00033-of-00048.safetensors +3 -0
- model-00034-of-00048.safetensors +3 -0
- model-00035-of-00048.safetensors +3 -0
- model-00036-of-00048.safetensors +3 -0
- model-00037-of-00048.safetensors +3 -0
- model-00038-of-00048.safetensors +3 -0
- model-00039-of-00048.safetensors +3 -0
- model-00040-of-00048.safetensors +3 -0
- model-00041-of-00048.safetensors +3 -0
- model-00042-of-00048.safetensors +3 -0
- model-00043-of-00048.safetensors +3 -0
- model-00044-of-00048.safetensors +3 -0
- model-00045-of-00048.safetensors +3 -0
- model-00046-of-00048.safetensors +3 -0
- model-00047-of-00048.safetensors +3 -0
- model-00048-of-00048.safetensors +3 -0
config.json
ADDED
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"_name_or_path": "mistralai/Mixtral-8x7B-Instruct-v0.1",
|
3 |
+
"architectures": [
|
4 |
+
"MixtralForCausalLM"
|
5 |
+
],
|
6 |
+
"attention_dropout": 0.0,
|
7 |
+
"bos_token_id": 1,
|
8 |
+
"eos_token_id": 2,
|
9 |
+
"hidden_act": "silu",
|
10 |
+
"hidden_size": 4096,
|
11 |
+
"initializer_range": 0.02,
|
12 |
+
"intermediate_size": 14336,
|
13 |
+
"max_position_embeddings": 32768,
|
14 |
+
"model_type": "mixtral",
|
15 |
+
"num_attention_heads": 32,
|
16 |
+
"num_experts_per_tok": 2,
|
17 |
+
"num_hidden_layers": 32,
|
18 |
+
"num_key_value_heads": 8,
|
19 |
+
"num_local_experts": 8,
|
20 |
+
"output_router_logits": false,
|
21 |
+
"rms_norm_eps": 1e-05,
|
22 |
+
"rope_theta": 1000000.0,
|
23 |
+
"router_aux_loss_coef": 0.02,
|
24 |
+
"sliding_window": null,
|
25 |
+
"tie_word_embeddings": false,
|
26 |
+
"torch_dtype": "bfloat16",
|
27 |
+
"transformers_version": "4.39.3",
|
28 |
+
"use_cache": true,
|
29 |
+
"vocab_size": 32000
|
30 |
+
}
|
mergekit_config.yml
ADDED
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
slices:
|
2 |
+
- sources:
|
3 |
+
- model: Envoid/Fish-8x7B
|
4 |
+
layer_range: [0, 32]
|
5 |
+
- model: mistralai/Mixtral-8x7B-Instruct-v0.1
|
6 |
+
layer_range: [0, 32]
|
7 |
+
|
8 |
+
merge_method: slerp
|
9 |
+
base_model: mistralai/Mixtral-8x7B-Instruct-v0.1
|
10 |
+
parameters:
|
11 |
+
t:
|
12 |
+
- filter: self_attn
|
13 |
+
value: [0, 0.5, 0.3, 0.7, 1]
|
14 |
+
- filter: mlp
|
15 |
+
value: [1, 0.5, 0.7, 0.3, 0]
|
16 |
+
- value: 0.5 # fallback for rest of tensors
|
17 |
+
dtype: bfloat16
|
model-00001-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:4436e0fdb5a6af55b124cdde904320c9e9061d213eeba7fe80edfebb08bb6370
|
3 |
+
size 1929530672
|
model-00002-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:1d258c5a1a5d2e376f04e83b543312c2cb6b0c556d3bf597ff5143dabe8d5b75
|
3 |
+
size 1996490952
|
model-00003-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:af287e8352d8fc5bf5b7dfb0d5b4896a3ab35fe67b59f03ae00fed2e15eb750c
|
3 |
+
size 1956793496
|
model-00004-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:cfc676a4932418c916c6b4c581d1fbb965fa948fec462ffce35a0e10e7da7f57
|
3 |
+
size 1996490952
|
model-00005-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:93e2bd623cfbe4c165b972fb2dc7335a35fd9c85631edf7748d5a62d595c9022
|
3 |
+
size 1996507568
|
model-00006-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:f5d81a8b91e5e36e8f29879cda164fd442c6a0d03582ebf200d257f7fdf91d12
|
3 |
+
size 1929530688
|
model-00007-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:ec810cca61dd022e4dd8b5491be8fda0eb96089318616c06866cf2ce4990be5f
|
3 |
+
size 1996507560
|
model-00008-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:749877eb89d47f10808ae54e05ec502087b2f34025161595177ec579008af1b7
|
3 |
+
size 1963002512
|
model-00009-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:44baf3b9b8cb2a321581c17b3e3ed8793189bb00e104756903fc2becaaf29453
|
3 |
+
size 1996507560
|
model-00010-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:f0ee8e2041a4a9c1f7c5d71cebde1430dc3476f47be247dfac53827838f02b70
|
3 |
+
size 1996490952
|
model-00011-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:23917bcb9de849d767e1f603e0d117ae5311520449f60204bf1e6caef64f730b
|
3 |
+
size 1929530672
|
model-00012-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:129b5f49ada9e8f4d8c7875224ff2e3f3854aa6353b0cb48507a37d93616694f
|
3 |
+
size 1996490952
|
model-00013-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:6b0ba8d01d41d0f4f42fa32c4873aee9dda7824d1cb95cf37061d8395b638a44
|
3 |
+
size 1929547296
|
model-00014-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:a5a31e18324477c504877c40cb88323fbc7cc09ff4cc770a36884e3f9e7ea920
|
3 |
+
size 1996490952
|
model-00015-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:508159d3aa2895f0954a5e8c7106951eca0d36dbf2e517278233073d0626a4ea
|
3 |
+
size 1996507568
|
model-00016-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:e43c4d04d7a088bd4785f9a102b0425ba8dd43ba1800104b0a29615a4b6a4a5d
|
3 |
+
size 1963002528
|
model-00017-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:03107bed00a41b5f3659e3a7a47063ae2f89deaba989915a8fac7169757793f8
|
3 |
+
size 1996507584
|
model-00018-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:abf5c3de9d50bb2d472842bc32061b971e91bd7ce5a821d31e90be5536db3874
|
3 |
+
size 1963002528
|
model-00019-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:4b2efdf8ea5b18fa8face0c947c6af7c9b268e90f875855f2c7f921947ec866a
|
3 |
+
size 1963019144
|
model-00020-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:84bf3c7839c67f787a1f1634c60728ccd0bc161f4424acb640382efeab837542
|
3 |
+
size 1996507576
|
model-00021-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:3c6f3f0f5a2a60e314abf08a78cd8d648359c8f55fb7f3f04d8a62da89709f42
|
3 |
+
size 1929530712
|
model-00022-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:da5a4f42bad279dbdb416c4ca8d6e4aad6781b7f67cda06a27ba8885775af97d
|
3 |
+
size 1996507576
|
model-00023-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:4778bf776360bea3cf283d2d2b54506d864e468b8cb230fd429110cdd23311cc
|
3 |
+
size 1963002528
|
model-00024-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:54bd0fc42918cd8d2692051855fddc1f70b99203b4625f22bd1fbd5ab7be06d7
|
3 |
+
size 1996507560
|
model-00025-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:b1590818cdfe974d46e331013d4dad14a511283e897b2d8ffb071abdca842274
|
3 |
+
size 1996490968
|
model-00026-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:7f50fcd720e350fa2055b37585a234d19c3167197923781429d7484d7d46d6e4
|
3 |
+
size 1963002528
|
model-00027-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:0716986eec38043c75d32f8796f44ee1f95131dd61169e533103f01626f82df0
|
3 |
+
size 1963019144
|
model-00028-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:aa645bfc190d8d48053c4d96ef625ecf6df414749321d26a447952c31bbb531a
|
3 |
+
size 1963019144
|
model-00029-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:f2125546b770653887b9eff1d22f0c073f2d0f405967d4ff3c0f08ec7cce6874
|
3 |
+
size 1963019144
|
model-00030-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:1210b1f4024e28aaea62c8532b00a16e4ad83ae2b3c5499314d3347f9e01893f
|
3 |
+
size 1996507576
|
model-00031-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:5dafcfcb36e945fd42779dea2cd4ba28624e9f40bf8336c08f4d4312634868b1
|
3 |
+
size 1929530712
|
model-00032-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:0ec8d1e6df4b8b73441b0e1a31f3c99afdbc4fba42b7d518abb0a4901311aa07
|
3 |
+
size 1996507576
|
model-00033-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:91bb33bd4388a99bb1a9314260a90aa629b26ac6f16236e328761482b23379e4
|
3 |
+
size 1963002528
|
model-00034-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:2804f09baf0cc406f12e36dd05c7756ee5c5e95945db0845bc6c959f7612d6cb
|
3 |
+
size 1996507576
|
model-00035-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:d2d5fbb30a0c4534941f9440ee050c0a86e6854d88a121b788925031d4b557c4
|
3 |
+
size 1996490968
|
model-00036-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:2e0399fbd90f80c311347fee74d5836d2ba6cabc29e1295b7f05570ba751c5ed
|
3 |
+
size 1929530696
|
model-00037-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:51a23e07fb60f9c150cecd88203bc531e1e6509aca1e83c8f353057d95243efb
|
3 |
+
size 1996490968
|
model-00038-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:c47b3d25896986b7c4dc625e7d8186fddaace0c8a864d2663503b6d9232263c0
|
3 |
+
size 1996582008
|
model-00039-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:85a3e8c222e50d639b06b458f231f649fcd3671f344dd9ab10f03079628c35bc
|
3 |
+
size 1929456288
|
model-00040-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:68cc8aa2e5872d305fde09e35496e6334d976ceb63afe8ecfdd0c3e31709d84c
|
3 |
+
size 1996507584
|
model-00041-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:bfd28585247f1698914d54ccfb3d10929801a032157ab24dc9a1bcb4f49a036f
|
3 |
+
size 1963002528
|
model-00042-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:be9e0d3c717bb51fcf8d0345f3f4bb437c10e6d9fb3b31be9f7f1d5a71e3542d
|
3 |
+
size 1996507576
|
model-00043-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:c8b3494945063d41955719c78f49af8d714dc1e880f545014457121b6b304803
|
3 |
+
size 1963002528
|
model-00044-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:5aef5c6eb207ddddbe1b8d47d3433e23a858610f442e30aa2f51d8c2c1795f79
|
3 |
+
size 1963019136
|
model-00045-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:e5c9ba0f1bf61de697b8e1ad13e862185fdff9daea44ba646abb7f0f8b0be9e1
|
3 |
+
size 1996490968
|
model-00046-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:baaa5f752fbbecd4e677ff30de3b1a2720b6c20e107055dc97d648efcc0e51ef
|
3 |
+
size 1990306848
|
model-00047-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:f1aee54121474a1ff7ff43b8f872f76885d879a3d482e3b4d700b9f302297287
|
3 |
+
size 1996507576
|
model-00048-of-00048.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:f8e79b22efe9a4a3e9baec0483a0d615599d22fcedd014047f69c894a0ff3d9d
|
3 |
+
size 587203248
|