Upload MixtralForCausalLM

config.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "./multimaster3",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mixtral",
+  "num_attention_heads": 32,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "num_local_experts": 6,
+  "output_router_logits": false,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "router_aux_loss_coef": 0.001,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.37.2",
+  "use_cache": true,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.37.2"
+}

model-00001-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8782bf699f8af6f82f0a2406da06267a5a07c867e90d0779b459d8b784a8bcc
+size 4987161928

model-00002-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06972335669b689f03dec2561ae5865e6d9978cf0558c067a8ba95b443064e40
+size 4966289216

model-00003-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba48776a90ad2874b0bb12a19024e6ffc120a6b25ca6caaf03ff8fe89fe90b87
+size 4865527152

model-00004-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34d54abb56f2233a01498f2ade45ede8572c8cb7bdb8a003697fbe4ec0460521
+size 4865527152

model-00005-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6dfc8d02fe7e0eb65a5c596fd22028ba8cb6fecefdf5dfbdee928a7c44ab3158
+size 4865527152

model-00006-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09d6bfed40a23eec5f79abc62fdcf7ab5d2dcfd4302372a65bf115a856143c4f
+size 4865527152

model-00007-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1236462de560662b7521a2efbb9588d484f5daec7d006463db15ced85eb22eeb
+size 4865527152

model-00008-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6527ed3e08ee62c07d77ec91f612719ddf39b475226b3728762e82fbb8dc85e
+size 4865527152

model-00009-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70823fe8eab41d353cb00ec505d106b8f0e511ce8828a24b1f20e5c22fced9cf
+size 4865527144

model-00010-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3c987163d1756ea3c23e91c1d3c52a694b352d5f43293beba70aa6ebefc0d27
+size 4966223816

model-00011-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50c950a1fe511d2471220b4970d517f44252a919cfb654d4dc18dd66269244df
+size 4932734584

model-00012-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e72e8673a980c273468792a7bd22518b748ba76144dd6ccfd19eac7673a856a
+size 4865527176

model-00013-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39dde508aedc9c918d1177b3b5887e270ceb8d548e84eeb07f583cdf3a45c1f7
+size 4865527176

model-00014-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa49a57137200d51e61d46ef846cfeb1dc7f481dde61c89774699f0c947037e6
+size 4865527176

model-00015-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88b42cea77bce44a6f8ba788b625e3c5c8397f6559178fc4da4b3ec6dda680f4
+size 4865527176

model-00016-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66c5e9205e4257bcbeacfa21f00cde3f9c7694b37e31dfbf46bfa045deee549f
+size 4865527176

model-00017-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6eee311d09301513cc4200fde9cd25470f8f2c3a3e5538a0912153118b9d0acc
+size 4865527176

model-00018-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b00883cf9afb4c9699b83d837f2325f1efa41a4d6d64fff5f10304a503c032c9
+size 4865527176

model-00019-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98c5d150662b8a5c8f28859a6a5f9518572b56b5149117d1fcd13e1c9c797729
+size 4966223816

model-00020-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd6408d5d74dc67172aa4e9da3dd2361e2f0327f3cffbb673db6f7e777647d90
+size 4932734584

model-00021-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20c4b1bca77911c0393f93a94cbe584cce0188a563b9a0f0427dbd32dcd3c6e6
+size 4865527176

model-00022-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae259b942ac001230bbcee8cf6e4e459f5d4465aae11bd5813ece456a0a7889d
+size 4865527176

model-00023-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eceac8a41edd7e0d317c7c742804bf6b137f15f74e5f67b9c9d078051eca8b6c
+size 4865527176

model-00024-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:835b394b7b2099f4481d137dfb538e1938d9b1f2a891792e3b237d60f34bfeb4
+size 4865527176

model-00025-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4028cd825b433df63efdf5d5d25e8622e47ddcf1ff6c79f9626e29808c8f7eaa
+size 4865527176

model-00026-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b83e8dc31182588118d6a15069bf63886d5e3c0121e7b190b61a2853669b237d
+size 4865527176

model-00027-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef8d149236cf2d50591a3161fdd1c268449c729edd757fabec32def070d8d14d
+size 4865527176

model-00028-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9962e8469e893c87ab56abd1e23d238f377c8e6076adc062f9b5b1d2562510d1
+size 4966223816

model-00029-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:772f10c00ab2e4bc650bdd0b8058b6e6333ae42f7712989f408c82011f05478e
+size 4819405792

model.safetensors.index.json

@@ -0,0 +1,810 @@
+{
+  "metadata": {
+    "total_size": 141712965632
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00029-of-00029.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00029.safetensors",
+    "model.layers.0.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00029.safetensors",
+    "model.layers.0.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00029.safetensors",
+    "model.layers.0.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00029.safetensors",
+    "model.layers.0.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00029.safetensors",
+    "model.layers.0.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00029.safetensors",
+    "model.layers.0.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00029.safetensors",
+    "model.layers.0.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00029.safetensors",
+    "model.layers.0.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00029.safetensors",
+    "model.layers.0.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00029.safetensors",
+    "model.layers.0.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00029.safetensors",
+    "model.layers.0.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00029.safetensors",
+    "model.layers.0.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00029.safetensors",
+    "model.layers.0.block_sparse_moe.experts.4.w1.weight": "model-00001-of-00029.safetensors",
+    "model.layers.0.block_sparse_moe.experts.4.w2.weight": "model-00001-of-00029.safetensors",
+    "model.layers.0.block_sparse_moe.experts.4.w3.weight": "model-00001-of-00029.safetensors",
+    "model.layers.0.block_sparse_moe.experts.5.w1.weight": "model-00001-of-00029.safetensors",
+    "model.layers.0.block_sparse_moe.experts.5.w2.weight": "model-00001-of-00029.safetensors",
+    "model.layers.0.block_sparse_moe.experts.5.w3.weight": "model-00001-of-00029.safetensors",
+    "model.layers.0.block_sparse_moe.gate.weight": "model-00001-of-00029.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00029.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00029.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00029.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00029.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00029.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00029.safetensors",
+    "model.layers.1.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00029.safetensors",
+    "model.layers.1.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00029.safetensors",
+    "model.layers.1.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00029.safetensors",
+    "model.layers.1.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00029.safetensors",
+    "model.layers.1.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00029.safetensors",
+    "model.layers.1.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00029.safetensors",
+    "model.layers.1.block_sparse_moe.experts.2.w1.weight": "model-00002-of-00029.safetensors",
+    "model.layers.1.block_sparse_moe.experts.2.w2.weight": "model-00002-of-00029.safetensors",
+    "model.layers.1.block_sparse_moe.experts.2.w3.weight": "model-00002-of-00029.safetensors",
+    "model.layers.1.block_sparse_moe.experts.3.w1.weight": "model-00002-of-00029.safetensors",
+    "model.layers.1.block_sparse_moe.experts.3.w2.weight": "model-00002-of-00029.safetensors",
+    "model.layers.1.block_sparse_moe.experts.3.w3.weight": "model-00002-of-00029.safetensors",
+    "model.layers.1.block_sparse_moe.experts.4.w1.weight": "model-00002-of-00029.safetensors",
+    "model.layers.1.block_sparse_moe.experts.4.w2.weight": "model-00002-of-00029.safetensors",
+    "model.layers.1.block_sparse_moe.experts.4.w3.weight": "model-00002-of-00029.safetensors",
+    "model.layers.1.block_sparse_moe.experts.5.w1.weight": "model-00002-of-00029.safetensors",
+    "model.layers.1.block_sparse_moe.experts.5.w2.weight": "model-00002-of-00029.safetensors",
+    "model.layers.1.block_sparse_moe.experts.5.w3.weight": "model-00002-of-00029.safetensors",
+    "model.layers.1.block_sparse_moe.gate.weight": "model-00002-of-00029.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00002-of-00029.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00002-of-00029.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00002-of-00029.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00002-of-00029.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00029.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00002-of-00029.safetensors",
+    "model.layers.10.block_sparse_moe.experts.0.w1.weight": "model-00010-of-00029.safetensors",
+    "model.layers.10.block_sparse_moe.experts.0.w2.weight": "model-00010-of-00029.safetensors",
+    "model.layers.10.block_sparse_moe.experts.0.w3.weight": "model-00010-of-00029.safetensors",
+    "model.layers.10.block_sparse_moe.experts.1.w1.weight": "model-00010-of-00029.safetensors",
+    "model.layers.10.block_sparse_moe.experts.1.w2.weight": "model-00010-of-00029.safetensors",
+    "model.layers.10.block_sparse_moe.experts.1.w3.weight": "model-00010-of-00029.safetensors",
+    "model.layers.10.block_sparse_moe.experts.2.w1.weight": "model-00010-of-00029.safetensors",
+    "model.layers.10.block_sparse_moe.experts.2.w2.weight": "model-00010-of-00029.safetensors",
+    "model.layers.10.block_sparse_moe.experts.2.w3.weight": "model-00010-of-00029.safetensors",
+    "model.layers.10.block_sparse_moe.experts.3.w1.weight": "model-00010-of-00029.safetensors",
+    "model.layers.10.block_sparse_moe.experts.3.w2.weight": "model-00010-of-00029.safetensors",
+    "model.layers.10.block_sparse_moe.experts.3.w3.weight": "model-00010-of-00029.safetensors",
+    "model.layers.10.block_sparse_moe.experts.4.w1.weight": "model-00010-of-00029.safetensors",
+    "model.layers.10.block_sparse_moe.experts.4.w2.weight": "model-00010-of-00029.safetensors",
+    "model.layers.10.block_sparse_moe.experts.4.w3.weight": "model-00010-of-00029.safetensors",
+    "model.layers.10.block_sparse_moe.experts.5.w1.weight": "model-00010-of-00029.safetensors",
+    "model.layers.10.block_sparse_moe.experts.5.w2.weight": "model-00010-of-00029.safetensors",
+    "model.layers.10.block_sparse_moe.experts.5.w3.weight": "model-00010-of-00029.safetensors",
+    "model.layers.10.block_sparse_moe.gate.weight": "model-00010-of-00029.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00010-of-00029.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00010-of-00029.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00010-of-00029.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00010-of-00029.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00010-of-00029.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00010-of-00029.safetensors",
+    "model.layers.11.block_sparse_moe.experts.0.w1.weight": "model-00011-of-00029.safetensors",
+    "model.layers.11.block_sparse_moe.experts.0.w2.weight": "model-00011-of-00029.safetensors",
+    "model.layers.11.block_sparse_moe.experts.0.w3.weight": "model-00011-of-00029.safetensors",
+    "model.layers.11.block_sparse_moe.experts.1.w1.weight": "model-00011-of-00029.safetensors",
+    "model.layers.11.block_sparse_moe.experts.1.w2.weight": "model-00011-of-00029.safetensors",
+    "model.layers.11.block_sparse_moe.experts.1.w3.weight": "model-00011-of-00029.safetensors",
+    "model.layers.11.block_sparse_moe.experts.2.w1.weight": "model-00011-of-00029.safetensors",
+    "model.layers.11.block_sparse_moe.experts.2.w2.weight": "model-00011-of-00029.safetensors",
+    "model.layers.11.block_sparse_moe.experts.2.w3.weight": "model-00011-of-00029.safetensors",
+    "model.layers.11.block_sparse_moe.experts.3.w1.weight": "model-00011-of-00029.safetensors",
+    "model.layers.11.block_sparse_moe.experts.3.w2.weight": "model-00011-of-00029.safetensors",
+    "model.layers.11.block_sparse_moe.experts.3.w3.weight": "model-00011-of-00029.safetensors",
+    "model.layers.11.block_sparse_moe.experts.4.w1.weight": "model-00011-of-00029.safetensors",
+    "model.layers.11.block_sparse_moe.experts.4.w2.weight": "model-00011-of-00029.safetensors",
+    "model.layers.11.block_sparse_moe.experts.4.w3.weight": "model-00011-of-00029.safetensors",
+    "model.layers.11.block_sparse_moe.experts.5.w1.weight": "model-00011-of-00029.safetensors",
+    "model.layers.11.block_sparse_moe.experts.5.w2.weight": "model-00011-of-00029.safetensors",
+    "model.layers.11.block_sparse_moe.experts.5.w3.weight": "model-00011-of-00029.safetensors",
+    "model.layers.11.block_sparse_moe.gate.weight": "model-00011-of-00029.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00011-of-00029.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00011-of-00029.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00010-of-00029.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00011-of-00029.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00010-of-00029.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00010-of-00029.safetensors",
+    "model.layers.12.block_sparse_moe.experts.0.w1.weight": "model-00011-of-00029.safetensors",
+    "model.layers.12.block_sparse_moe.experts.0.w2.weight": "model-00011-of-00029.safetensors",
+    "model.layers.12.block_sparse_moe.experts.0.w3.weight": "model-00012-of-00029.safetensors",
+    "model.layers.12.block_sparse_moe.experts.1.w1.weight": "model-00012-of-00029.safetensors",
+    "model.layers.12.block_sparse_moe.experts.1.w2.weight": "model-00012-of-00029.safetensors",
+    "model.layers.12.block_sparse_moe.experts.1.w3.weight": "model-00012-of-00029.safetensors",
+    "model.layers.12.block_sparse_moe.experts.2.w1.weight": "model-00012-of-00029.safetensors",
+    "model.layers.12.block_sparse_moe.experts.2.w2.weight": "model-00012-of-00029.safetensors",
+    "model.layers.12.block_sparse_moe.experts.2.w3.weight": "model-00012-of-00029.safetensors",
+    "model.layers.12.block_sparse_moe.experts.3.w1.weight": "model-00012-of-00029.safetensors",
+    "model.layers.12.block_sparse_moe.experts.3.w2.weight": "model-00012-of-00029.safetensors",
+    "model.layers.12.block_sparse_moe.experts.3.w3.weight": "model-00012-of-00029.safetensors",
+    "model.layers.12.block_sparse_moe.experts.4.w1.weight": "model-00012-of-00029.safetensors",
+    "model.layers.12.block_sparse_moe.experts.4.w2.weight": "model-00012-of-00029.safetensors",
+    "model.layers.12.block_sparse_moe.experts.4.w3.weight": "model-00012-of-00029.safetensors",
+    "model.layers.12.block_sparse_moe.experts.5.w1.weight": "model-00012-of-00029.safetensors",
+    "model.layers.12.block_sparse_moe.experts.5.w2.weight": "model-00012-of-00029.safetensors",
+    "model.layers.12.block_sparse_moe.experts.5.w3.weight": "model-00012-of-00029.safetensors",
+    "model.layers.12.block_sparse_moe.gate.weight": "model-00011-of-00029.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00012-of-00029.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00012-of-00029.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00011-of-00029.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00011-of-00029.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00011-of-00029.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00011-of-00029.safetensors",
+    "model.layers.13.block_sparse_moe.experts.0.w1.weight": "model-00012-of-00029.safetensors",
+    "model.layers.13.block_sparse_moe.experts.0.w2.weight": "model-00012-of-00029.safetensors",
+    "model.layers.13.block_sparse_moe.experts.0.w3.weight": "model-00012-of-00029.safetensors",
+    "model.layers.13.block_sparse_moe.experts.1.w1.weight": "model-00012-of-00029.safetensors",
+    "model.layers.13.block_sparse_moe.experts.1.w2.weight": "model-00013-of-00029.safetensors",
+    "model.layers.13.block_sparse_moe.experts.1.w3.weight": "model-00013-of-00029.safetensors",
+    "model.layers.13.block_sparse_moe.experts.2.w1.weight": "model-00013-of-00029.safetensors",
+    "model.layers.13.block_sparse_moe.experts.2.w2.weight": "model-00013-of-00029.safetensors",
+    "model.layers.13.block_sparse_moe.experts.2.w3.weight": "model-00013-of-00029.safetensors",
+    "model.layers.13.block_sparse_moe.experts.3.w1.weight": "model-00013-of-00029.safetensors",
+    "model.layers.13.block_sparse_moe.experts.3.w2.weight": "model-00013-of-00029.safetensors",
+    "model.layers.13.block_sparse_moe.experts.3.w3.weight": "model-00013-of-00029.safetensors",
+    "model.layers.13.block_sparse_moe.experts.4.w1.weight": "model-00013-of-00029.safetensors",
+    "model.layers.13.block_sparse_moe.experts.4.w2.weight": "model-00013-of-00029.safetensors",
+    "model.layers.13.block_sparse_moe.experts.4.w3.weight": "model-00013-of-00029.safetensors",
+    "model.layers.13.block_sparse_moe.experts.5.w1.weight": "model-00013-of-00029.safetensors",
+    "model.layers.13.block_sparse_moe.experts.5.w2.weight": "model-00013-of-00029.safetensors",
+    "model.layers.13.block_sparse_moe.experts.5.w3.weight": "model-00013-of-00029.safetensors",
+    "model.layers.13.block_sparse_moe.gate.weight": "model-00012-of-00029.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00013-of-00029.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00013-of-00029.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00012-of-00029.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00012-of-00029.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00012-of-00029.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00012-of-00029.safetensors",
+    "model.layers.14.block_sparse_moe.experts.0.w1.weight": "model-00013-of-00029.safetensors",
+    "model.layers.14.block_sparse_moe.experts.0.w2.weight": "model-00013-of-00029.safetensors",
+    "model.layers.14.block_sparse_moe.experts.0.w3.weight": "model-00013-of-00029.safetensors",
+    "model.layers.14.block_sparse_moe.experts.1.w1.weight": "model-00013-of-00029.safetensors",
+    "model.layers.14.block_sparse_moe.experts.1.w2.weight": "model-00013-of-00029.safetensors",
+    "model.layers.14.block_sparse_moe.experts.1.w3.weight": "model-00013-of-00029.safetensors",
+    "model.layers.14.block_sparse_moe.experts.2.w1.weight": "model-00014-of-00029.safetensors",
+    "model.layers.14.block_sparse_moe.experts.2.w2.weight": "model-00014-of-00029.safetensors",
+    "model.layers.14.block_sparse_moe.experts.2.w3.weight": "model-00014-of-00029.safetensors",
+    "model.layers.14.block_sparse_moe.experts.3.w1.weight": "model-00014-of-00029.safetensors",
+    "model.layers.14.block_sparse_moe.experts.3.w2.weight": "model-00014-of-00029.safetensors",
+    "model.layers.14.block_sparse_moe.experts.3.w3.weight": "model-00014-of-00029.safetensors",
+    "model.layers.14.block_sparse_moe.experts.4.w1.weight": "model-00014-of-00029.safetensors",
+    "model.layers.14.block_sparse_moe.experts.4.w2.weight": "model-00014-of-00029.safetensors",
+    "model.layers.14.block_sparse_moe.experts.4.w3.weight": "model-00014-of-00029.safetensors",
+    "model.layers.14.block_sparse_moe.experts.5.w1.weight": "model-00014-of-00029.safetensors",
+    "model.layers.14.block_sparse_moe.experts.5.w2.weight": "model-00014-of-00029.safetensors",
+    "model.layers.14.block_sparse_moe.experts.5.w3.weight": "model-00014-of-00029.safetensors",
+    "model.layers.14.block_sparse_moe.gate.weight": "model-00013-of-00029.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00014-of-00029.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00014-of-00029.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00013-of-00029.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00013-of-00029.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00013-of-00029.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00013-of-00029.safetensors",
+    "model.layers.15.block_sparse_moe.experts.0.w1.weight": "model-00014-of-00029.safetensors",
+    "model.layers.15.block_sparse_moe.experts.0.w2.weight": "model-00014-of-00029.safetensors",
+    "model.layers.15.block_sparse_moe.experts.0.w3.weight": "model-00014-of-00029.safetensors",
+    "model.layers.15.block_sparse_moe.experts.1.w1.weight": "model-00014-of-00029.safetensors",
+    "model.layers.15.block_sparse_moe.experts.1.w2.weight": "model-00014-of-00029.safetensors",
+    "model.layers.15.block_sparse_moe.experts.1.w3.weight": "model-00014-of-00029.safetensors",
+    "model.layers.15.block_sparse_moe.experts.2.w1.weight": "model-00014-of-00029.safetensors",
+    "model.layers.15.block_sparse_moe.experts.2.w2.weight": "model-00014-of-00029.safetensors",
+    "model.layers.15.block_sparse_moe.experts.2.w3.weight": "model-00015-of-00029.safetensors",
+    "model.layers.15.block_sparse_moe.experts.3.w1.weight": "model-00015-of-00029.safetensors",
+    "model.layers.15.block_sparse_moe.experts.3.w2.weight": "model-00015-of-00029.safetensors",
+    "model.layers.15.block_sparse_moe.experts.3.w3.weight": "model-00015-of-00029.safetensors",
+    "model.layers.15.block_sparse_moe.experts.4.w1.weight": "model-00015-of-00029.safetensors",
+    "model.layers.15.block_sparse_moe.experts.4.w2.weight": "model-00015-of-00029.safetensors",
+    "model.layers.15.block_sparse_moe.experts.4.w3.weight": "model-00015-of-00029.safetensors",
+    "model.layers.15.block_sparse_moe.experts.5.w1.weight": "model-00015-of-00029.safetensors",
+    "model.layers.15.block_sparse_moe.experts.5.w2.weight": "model-00015-of-00029.safetensors",
+    "model.layers.15.block_sparse_moe.experts.5.w3.weight": "model-00015-of-00029.safetensors",
+    "model.layers.15.block_sparse_moe.gate.weight": "model-00014-of-00029.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00015-of-00029.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00015-of-00029.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00014-of-00029.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00014-of-00029.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00014-of-00029.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00014-of-00029.safetensors",
+    "model.layers.16.block_sparse_moe.experts.0.w1.weight": "model-00015-of-00029.safetensors",
+    "model.layers.16.block_sparse_moe.experts.0.w2.weight": "model-00015-of-00029.safetensors",
+    "model.layers.16.block_sparse_moe.experts.0.w3.weight": "model-00015-of-00029.safetensors",
+    "model.layers.16.block_sparse_moe.experts.1.w1.weight": "model-00015-of-00029.safetensors",
+    "model.layers.16.block_sparse_moe.experts.1.w2.weight": "model-00015-of-00029.safetensors",
+    "model.layers.16.block_sparse_moe.experts.1.w3.weight": "model-00015-of-00029.safetensors",
+    "model.layers.16.block_sparse_moe.experts.2.w1.weight": "model-00015-of-00029.safetensors",
+    "model.layers.16.block_sparse_moe.experts.2.w2.weight": "model-00015-of-00029.safetensors",
+    "model.layers.16.block_sparse_moe.experts.2.w3.weight": "model-00015-of-00029.safetensors",
+    "model.layers.16.block_sparse_moe.experts.3.w1.weight": "model-00015-of-00029.safetensors",
+    "model.layers.16.block_sparse_moe.experts.3.w2.weight": "model-00016-of-00029.safetensors",
+    "model.layers.16.block_sparse_moe.experts.3.w3.weight": "model-00016-of-00029.safetensors",
+    "model.layers.16.block_sparse_moe.experts.4.w1.weight": "model-00016-of-00029.safetensors",
+    "model.layers.16.block_sparse_moe.experts.4.w2.weight": "model-00016-of-00029.safetensors",
+    "model.layers.16.block_sparse_moe.experts.4.w3.weight": "model-00016-of-00029.safetensors",
+    "model.layers.16.block_sparse_moe.experts.5.w1.weight": "model-00016-of-00029.safetensors",
+    "model.layers.16.block_sparse_moe.experts.5.w2.weight": "model-00016-of-00029.safetensors",
+    "model.layers.16.block_sparse_moe.experts.5.w3.weight": "model-00016-of-00029.safetensors",
+    "model.layers.16.block_sparse_moe.gate.weight": "model-00015-of-00029.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00016-of-00029.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00016-of-00029.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00015-of-00029.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00015-of-00029.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00015-of-00029.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00015-of-00029.safetensors",
+    "model.layers.17.block_sparse_moe.experts.0.w1.weight": "model-00016-of-00029.safetensors",
+    "model.layers.17.block_sparse_moe.experts.0.w2.weight": "model-00016-of-00029.safetensors",
+    "model.layers.17.block_sparse_moe.experts.0.w3.weight": "model-00016-of-00029.safetensors",
+    "model.layers.17.block_sparse_moe.experts.1.w1.weight": "model-00016-of-00029.safetensors",
+    "model.layers.17.block_sparse_moe.experts.1.w2.weight": "model-00016-of-00029.safetensors",
+    "model.layers.17.block_sparse_moe.experts.1.w3.weight": "model-00016-of-00029.safetensors",
+    "model.layers.17.block_sparse_moe.experts.2.w1.weight": "model-00016-of-00029.safetensors",
+    "model.layers.17.block_sparse_moe.experts.2.w2.weight": "model-00016-of-00029.safetensors",
+    "model.layers.17.block_sparse_moe.experts.2.w3.weight": "model-00016-of-00029.safetensors",
+    "model.layers.17.block_sparse_moe.experts.3.w1.weight": "model-00016-of-00029.safetensors",
+    "model.layers.17.block_sparse_moe.experts.3.w2.weight": "model-00016-of-00029.safetensors",
+    "model.layers.17.block_sparse_moe.experts.3.w3.weight": "model-00016-of-00029.safetensors",
+    "model.layers.17.block_sparse_moe.experts.4.w1.weight": "model-00017-of-00029.safetensors",
+    "model.layers.17.block_sparse_moe.experts.4.w2.weight": "model-00017-of-00029.safetensors",
+    "model.layers.17.block_sparse_moe.experts.4.w3.weight": "model-00017-of-00029.safetensors",
+    "model.layers.17.block_sparse_moe.experts.5.w1.weight": "model-00017-of-00029.safetensors",
+    "model.layers.17.block_sparse_moe.experts.5.w2.weight": "model-00017-of-00029.safetensors",
+    "model.layers.17.block_sparse_moe.experts.5.w3.weight": "model-00017-of-00029.safetensors",
+    "model.layers.17.block_sparse_moe.gate.weight": "model-00016-of-00029.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00017-of-00029.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00017-of-00029.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00016-of-00029.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00016-of-00029.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00016-of-00029.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00016-of-00029.safetensors",
+    "model.layers.18.block_sparse_moe.experts.0.w1.weight": "model-00017-of-00029.safetensors",
+    "model.layers.18.block_sparse_moe.experts.0.w2.weight": "model-00017-of-00029.safetensors",
+    "model.layers.18.block_sparse_moe.experts.0.w3.weight": "model-00017-of-00029.safetensors",
+    "model.layers.18.block_sparse_moe.experts.1.w1.weight": "model-00017-of-00029.safetensors",
+    "model.layers.18.block_sparse_moe.experts.1.w2.weight": "model-00017-of-00029.safetensors",
+    "model.layers.18.block_sparse_moe.experts.1.w3.weight": "model-00017-of-00029.safetensors",
+    "model.layers.18.block_sparse_moe.experts.2.w1.weight": "model-00017-of-00029.safetensors",
+    "model.layers.18.block_sparse_moe.experts.2.w2.weight": "model-00017-of-00029.safetensors",
+    "model.layers.18.block_sparse_moe.experts.2.w3.weight": "model-00017-of-00029.safetensors",
+    "model.layers.18.block_sparse_moe.experts.3.w1.weight": "model-00017-of-00029.safetensors",
+    "model.layers.18.block_sparse_moe.experts.3.w2.weight": "model-00017-of-00029.safetensors",
+    "model.layers.18.block_sparse_moe.experts.3.w3.weight": "model-00017-of-00029.safetensors",
+    "model.layers.18.block_sparse_moe.experts.4.w1.weight": "model-00017-of-00029.safetensors",
+    "model.layers.18.block_sparse_moe.experts.4.w2.weight": "model-00017-of-00029.safetensors",
+    "model.layers.18.block_sparse_moe.experts.4.w3.weight": "model-00018-of-00029.safetensors",
+    "model.layers.18.block_sparse_moe.experts.5.w1.weight": "model-00018-of-00029.safetensors",
+    "model.layers.18.block_sparse_moe.experts.5.w2.weight": "model-00018-of-00029.safetensors",
+    "model.layers.18.block_sparse_moe.experts.5.w3.weight": "model-00018-of-00029.safetensors",
+    "model.layers.18.block_sparse_moe.gate.weight": "model-00017-of-00029.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00018-of-00029.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00018-of-00029.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00017-of-00029.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00017-of-00029.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00017-of-00029.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00017-of-00029.safetensors",
+    "model.layers.19.block_sparse_moe.experts.0.w1.weight": "model-00018-of-00029.safetensors",
+    "model.layers.19.block_sparse_moe.experts.0.w2.weight": "model-00018-of-00029.safetensors",
+    "model.layers.19.block_sparse_moe.experts.0.w3.weight": "model-00018-of-00029.safetensors",
+    "model.layers.19.block_sparse_moe.experts.1.w1.weight": "model-00018-of-00029.safetensors",
+    "model.layers.19.block_sparse_moe.experts.1.w2.weight": "model-00018-of-00029.safetensors",
+    "model.layers.19.block_sparse_moe.experts.1.w3.weight": "model-00018-of-00029.safetensors",
+    "model.layers.19.block_sparse_moe.experts.2.w1.weight": "model-00018-of-00029.safetensors",
+    "model.layers.19.block_sparse_moe.experts.2.w2.weight": "model-00018-of-00029.safetensors",
+    "model.layers.19.block_sparse_moe.experts.2.w3.weight": "model-00018-of-00029.safetensors",
+    "model.layers.19.block_sparse_moe.experts.3.w1.weight": "model-00018-of-00029.safetensors",
+    "model.layers.19.block_sparse_moe.experts.3.w2.weight": "model-00018-of-00029.safetensors",
+    "model.layers.19.block_sparse_moe.experts.3.w3.weight": "model-00018-of-00029.safetensors",
+    "model.layers.19.block_sparse_moe.experts.4.w1.weight": "model-00018-of-00029.safetensors",
+    "model.layers.19.block_sparse_moe.experts.4.w2.weight": "model-00018-of-00029.safetensors",
+    "model.layers.19.block_sparse_moe.experts.4.w3.weight": "model-00018-of-00029.safetensors",
+    "model.layers.19.block_sparse_moe.experts.5.w1.weight": "model-00018-of-00029.safetensors",
+    "model.layers.19.block_sparse_moe.experts.5.w2.weight": "model-00019-of-00029.safetensors",
+    "model.layers.19.block_sparse_moe.experts.5.w3.weight": "model-00019-of-00029.safetensors",
+    "model.layers.19.block_sparse_moe.gate.weight": "model-00018-of-00029.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00019-of-00029.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00019-of-00029.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00018-of-00029.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00018-of-00029.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00018-of-00029.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00018-of-00029.safetensors",
+    "model.layers.2.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00029.safetensors",
+    "model.layers.2.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00029.safetensors",
+    "model.layers.2.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00029.safetensors",
+    "model.layers.2.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00029.safetensors",
+    "model.layers.2.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00029.safetensors",
+    "model.layers.2.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00029.safetensors",
+    "model.layers.2.block_sparse_moe.experts.2.w1.weight": "model-00003-of-00029.safetensors",
+    "model.layers.2.block_sparse_moe.experts.2.w2.weight": "model-00003-of-00029.safetensors",
+    "model.layers.2.block_sparse_moe.experts.2.w3.weight": "model-00003-of-00029.safetensors",
+    "model.layers.2.block_sparse_moe.experts.3.w1.weight": "model-00003-of-00029.safetensors",
+    "model.layers.2.block_sparse_moe.experts.3.w2.weight": "model-00003-of-00029.safetensors",
+    "model.layers.2.block_sparse_moe.experts.3.w3.weight": "model-00003-of-00029.safetensors",
+    "model.layers.2.block_sparse_moe.experts.4.w1.weight": "model-00003-of-00029.safetensors",
+    "model.layers.2.block_sparse_moe.experts.4.w2.weight": "model-00003-of-00029.safetensors",
+    "model.layers.2.block_sparse_moe.experts.4.w3.weight": "model-00003-of-00029.safetensors",
+    "model.layers.2.block_sparse_moe.experts.5.w1.weight": "model-00003-of-00029.safetensors",
+    "model.layers.2.block_sparse_moe.experts.5.w2.weight": "model-00003-of-00029.safetensors",
+    "model.layers.2.block_sparse_moe.experts.5.w3.weight": "model-00003-of-00029.safetensors",
+    "model.layers.2.block_sparse_moe.gate.weight": "model-00002-of-00029.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00003-of-00029.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00003-of-00029.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00002-of-00029.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00002-of-00029.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00002-of-00029.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00002-of-00029.safetensors",
+    "model.layers.20.block_sparse_moe.experts.0.w1.weight": "model-00019-of-00029.safetensors",
+    "model.layers.20.block_sparse_moe.experts.0.w2.weight": "model-00019-of-00029.safetensors",
+    "model.layers.20.block_sparse_moe.experts.0.w3.weight": "model-00019-of-00029.safetensors",
+    "model.layers.20.block_sparse_moe.experts.1.w1.weight": "model-00019-of-00029.safetensors",
+    "model.layers.20.block_sparse_moe.experts.1.w2.weight": "model-00019-of-00029.safetensors",
+    "model.layers.20.block_sparse_moe.experts.1.w3.weight": "model-00019-of-00029.safetensors",
+    "model.layers.20.block_sparse_moe.experts.2.w1.weight": "model-00019-of-00029.safetensors",
+    "model.layers.20.block_sparse_moe.experts.2.w2.weight": "model-00019-of-00029.safetensors",
+    "model.layers.20.block_sparse_moe.experts.2.w3.weight": "model-00019-of-00029.safetensors",
+    "model.layers.20.block_sparse_moe.experts.3.w1.weight": "model-00019-of-00029.safetensors",
+    "model.layers.20.block_sparse_moe.experts.3.w2.weight": "model-00019-of-00029.safetensors",
+    "model.layers.20.block_sparse_moe.experts.3.w3.weight": "model-00019-of-00029.safetensors",
+    "model.layers.20.block_sparse_moe.experts.4.w1.weight": "model-00019-of-00029.safetensors",
+    "model.layers.20.block_sparse_moe.experts.4.w2.weight": "model-00019-of-00029.safetensors",
+    "model.layers.20.block_sparse_moe.experts.4.w3.weight": "model-00019-of-00029.safetensors",
+    "model.layers.20.block_sparse_moe.experts.5.w1.weight": "model-00019-of-00029.safetensors",
+    "model.layers.20.block_sparse_moe.experts.5.w2.weight": "model-00019-of-00029.safetensors",
+    "model.layers.20.block_sparse_moe.experts.5.w3.weight": "model-00019-of-00029.safetensors",
+    "model.layers.20.block_sparse_moe.gate.weight": "model-00019-of-00029.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00019-of-00029.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00019-of-00029.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00019-of-00029.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00019-of-00029.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00019-of-00029.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00019-of-00029.safetensors",
+    "model.layers.21.block_sparse_moe.experts.0.w1.weight": "model-00020-of-00029.safetensors",
+    "model.layers.21.block_sparse_moe.experts.0.w2.weight": "model-00020-of-00029.safetensors",
+    "model.layers.21.block_sparse_moe.experts.0.w3.weight": "model-00020-of-00029.safetensors",
+    "model.layers.21.block_sparse_moe.experts.1.w1.weight": "model-00020-of-00029.safetensors",
+    "model.layers.21.block_sparse_moe.experts.1.w2.weight": "model-00020-of-00029.safetensors",
+    "model.layers.21.block_sparse_moe.experts.1.w3.weight": "model-00020-of-00029.safetensors",
+    "model.layers.21.block_sparse_moe.experts.2.w1.weight": "model-00020-of-00029.safetensors",
+    "model.layers.21.block_sparse_moe.experts.2.w2.weight": "model-00020-of-00029.safetensors",
+    "model.layers.21.block_sparse_moe.experts.2.w3.weight": "model-00020-of-00029.safetensors",
+    "model.layers.21.block_sparse_moe.experts.3.w1.weight": "model-00020-of-00029.safetensors",
+    "model.layers.21.block_sparse_moe.experts.3.w2.weight": "model-00020-of-00029.safetensors",
+    "model.layers.21.block_sparse_moe.experts.3.w3.weight": "model-00020-of-00029.safetensors",
+    "model.layers.21.block_sparse_moe.experts.4.w1.weight": "model-00020-of-00029.safetensors",
+    "model.layers.21.block_sparse_moe.experts.4.w2.weight": "model-00020-of-00029.safetensors",
+    "model.layers.21.block_sparse_moe.experts.4.w3.weight": "model-00020-of-00029.safetensors",
+    "model.layers.21.block_sparse_moe.experts.5.w1.weight": "model-00020-of-00029.safetensors",
+    "model.layers.21.block_sparse_moe.experts.5.w2.weight": "model-00020-of-00029.safetensors",
+    "model.layers.21.block_sparse_moe.experts.5.w3.weight": "model-00020-of-00029.safetensors",
+    "model.layers.21.block_sparse_moe.gate.weight": "model-00020-of-00029.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00020-of-00029.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00020-of-00029.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00019-of-00029.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00020-of-00029.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00019-of-00029.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00019-of-00029.safetensors",
+    "model.layers.22.block_sparse_moe.experts.0.w1.weight": "model-00020-of-00029.safetensors",
+    "model.layers.22.block_sparse_moe.experts.0.w2.weight": "model-00020-of-00029.safetensors",
+    "model.layers.22.block_sparse_moe.experts.0.w3.weight": "model-00021-of-00029.safetensors",
+    "model.layers.22.block_sparse_moe.experts.1.w1.weight": "model-00021-of-00029.safetensors",
+    "model.layers.22.block_sparse_moe.experts.1.w2.weight": "model-00021-of-00029.safetensors",
+    "model.layers.22.block_sparse_moe.experts.1.w3.weight": "model-00021-of-00029.safetensors",
+    "model.layers.22.block_sparse_moe.experts.2.w1.weight": "model-00021-of-00029.safetensors",
+    "model.layers.22.block_sparse_moe.experts.2.w2.weight": "model-00021-of-00029.safetensors",
+    "model.layers.22.block_sparse_moe.experts.2.w3.weight": "model-00021-of-00029.safetensors",
+    "model.layers.22.block_sparse_moe.experts.3.w1.weight": "model-00021-of-00029.safetensors",
+    "model.layers.22.block_sparse_moe.experts.3.w2.weight": "model-00021-of-00029.safetensors",
+    "model.layers.22.block_sparse_moe.experts.3.w3.weight": "model-00021-of-00029.safetensors",
+    "model.layers.22.block_sparse_moe.experts.4.w1.weight": "model-00021-of-00029.safetensors",
+    "model.layers.22.block_sparse_moe.experts.4.w2.weight": "model-00021-of-00029.safetensors",
+    "model.layers.22.block_sparse_moe.experts.4.w3.weight": "model-00021-of-00029.safetensors",
+    "model.layers.22.block_sparse_moe.experts.5.w1.weight": "model-00021-of-00029.safetensors",
+    "model.layers.22.block_sparse_moe.experts.5.w2.weight": "model-00021-of-00029.safetensors",
+    "model.layers.22.block_sparse_moe.experts.5.w3.weight": "model-00021-of-00029.safetensors",
+    "model.layers.22.block_sparse_moe.gate.weight": "model-00020-of-00029.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00021-of-00029.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00021-of-00029.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00020-of-00029.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00020-of-00029.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00020-of-00029.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00020-of-00029.safetensors",
+    "model.layers.23.block_sparse_moe.experts.0.w1.weight": "model-00021-of-00029.safetensors",
+    "model.layers.23.block_sparse_moe.experts.0.w2.weight": "model-00021-of-00029.safetensors",
+    "model.layers.23.block_sparse_moe.experts.0.w3.weight": "model-00021-of-00029.safetensors",
+    "model.layers.23.block_sparse_moe.experts.1.w1.weight": "model-00021-of-00029.safetensors",
+    "model.layers.23.block_sparse_moe.experts.1.w2.weight": "model-00022-of-00029.safetensors",
+    "model.layers.23.block_sparse_moe.experts.1.w3.weight": "model-00022-of-00029.safetensors",
+    "model.layers.23.block_sparse_moe.experts.2.w1.weight": "model-00022-of-00029.safetensors",
+    "model.layers.23.block_sparse_moe.experts.2.w2.weight": "model-00022-of-00029.safetensors",
+    "model.layers.23.block_sparse_moe.experts.2.w3.weight": "model-00022-of-00029.safetensors",
+    "model.layers.23.block_sparse_moe.experts.3.w1.weight": "model-00022-of-00029.safetensors",
+    "model.layers.23.block_sparse_moe.experts.3.w2.weight": "model-00022-of-00029.safetensors",
+    "model.layers.23.block_sparse_moe.experts.3.w3.weight": "model-00022-of-00029.safetensors",
+    "model.layers.23.block_sparse_moe.experts.4.w1.weight": "model-00022-of-00029.safetensors",
+    "model.layers.23.block_sparse_moe.experts.4.w2.weight": "model-00022-of-00029.safetensors",
+    "model.layers.23.block_sparse_moe.experts.4.w3.weight": "model-00022-of-00029.safetensors",
+    "model.layers.23.block_sparse_moe.experts.5.w1.weight": "model-00022-of-00029.safetensors",
+    "model.layers.23.block_sparse_moe.experts.5.w2.weight": "model-00022-of-00029.safetensors",
+    "model.layers.23.block_sparse_moe.experts.5.w3.weight": "model-00022-of-00029.safetensors",
+    "model.layers.23.block_sparse_moe.gate.weight": "model-00021-of-00029.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00022-of-00029.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00022-of-00029.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00021-of-00029.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00021-of-00029.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00021-of-00029.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00021-of-00029.safetensors",
+    "model.layers.24.block_sparse_moe.experts.0.w1.weight": "model-00022-of-00029.safetensors",
+    "model.layers.24.block_sparse_moe.experts.0.w2.weight": "model-00022-of-00029.safetensors",
+    "model.layers.24.block_sparse_moe.experts.0.w3.weight": "model-00022-of-00029.safetensors",
+    "model.layers.24.block_sparse_moe.experts.1.w1.weight": "model-00022-of-00029.safetensors",
+    "model.layers.24.block_sparse_moe.experts.1.w2.weight": "model-00022-of-00029.safetensors",
+    "model.layers.24.block_sparse_moe.experts.1.w3.weight": "model-00022-of-00029.safetensors",
+    "model.layers.24.block_sparse_moe.experts.2.w1.weight": "model-00023-of-00029.safetensors",
+    "model.layers.24.block_sparse_moe.experts.2.w2.weight": "model-00023-of-00029.safetensors",
+    "model.layers.24.block_sparse_moe.experts.2.w3.weight": "model-00023-of-00029.safetensors",
+    "model.layers.24.block_sparse_moe.experts.3.w1.weight": "model-00023-of-00029.safetensors",
+    "model.layers.24.block_sparse_moe.experts.3.w2.weight": "model-00023-of-00029.safetensors",
+    "model.layers.24.block_sparse_moe.experts.3.w3.weight": "model-00023-of-00029.safetensors",
+    "model.layers.24.block_sparse_moe.experts.4.w1.weight": "model-00023-of-00029.safetensors",
+    "model.layers.24.block_sparse_moe.experts.4.w2.weight": "model-00023-of-00029.safetensors",
+    "model.layers.24.block_sparse_moe.experts.4.w3.weight": "model-00023-of-00029.safetensors",
+    "model.layers.24.block_sparse_moe.experts.5.w1.weight": "model-00023-of-00029.safetensors",
+    "model.layers.24.block_sparse_moe.experts.5.w2.weight": "model-00023-of-00029.safetensors",
+    "model.layers.24.block_sparse_moe.experts.5.w3.weight": "model-00023-of-00029.safetensors",
+    "model.layers.24.block_sparse_moe.gate.weight": "model-00022-of-00029.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00023-of-00029.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00023-of-00029.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00022-of-00029.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00022-of-00029.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00022-of-00029.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00022-of-00029.safetensors",
+    "model.layers.25.block_sparse_moe.experts.0.w1.weight": "model-00023-of-00029.safetensors",
+    "model.layers.25.block_sparse_moe.experts.0.w2.weight": "model-00023-of-00029.safetensors",
+    "model.layers.25.block_sparse_moe.experts.0.w3.weight": "model-00023-of-00029.safetensors",
+    "model.layers.25.block_sparse_moe.experts.1.w1.weight": "model-00023-of-00029.safetensors",
+    "model.layers.25.block_sparse_moe.experts.1.w2.weight": "model-00023-of-00029.safetensors",
+    "model.layers.25.block_sparse_moe.experts.1.w3.weight": "model-00023-of-00029.safetensors",
+    "model.layers.25.block_sparse_moe.experts.2.w1.weight": "model-00023-of-00029.safetensors",
+    "model.layers.25.block_sparse_moe.experts.2.w2.weight": "model-00023-of-00029.safetensors",
+    "model.layers.25.block_sparse_moe.experts.2.w3.weight": "model-00024-of-00029.safetensors",
+    "model.layers.25.block_sparse_moe.experts.3.w1.weight": "model-00024-of-00029.safetensors",
+    "model.layers.25.block_sparse_moe.experts.3.w2.weight": "model-00024-of-00029.safetensors",
+    "model.layers.25.block_sparse_moe.experts.3.w3.weight": "model-00024-of-00029.safetensors",
+    "model.layers.25.block_sparse_moe.experts.4.w1.weight": "model-00024-of-00029.safetensors",
+    "model.layers.25.block_sparse_moe.experts.4.w2.weight": "model-00024-of-00029.safetensors",
+    "model.layers.25.block_sparse_moe.experts.4.w3.weight": "model-00024-of-00029.safetensors",
+    "model.layers.25.block_sparse_moe.experts.5.w1.weight": "model-00024-of-00029.safetensors",
+    "model.layers.25.block_sparse_moe.experts.5.w2.weight": "model-00024-of-00029.safetensors",
+    "model.layers.25.block_sparse_moe.experts.5.w3.weight": "model-00024-of-00029.safetensors",
+    "model.layers.25.block_sparse_moe.gate.weight": "model-00023-of-00029.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00024-of-00029.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00024-of-00029.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00023-of-00029.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00023-of-00029.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00023-of-00029.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00023-of-00029.safetensors",
+    "model.layers.26.block_sparse_moe.experts.0.w1.weight": "model-00024-of-00029.safetensors",
+    "model.layers.26.block_sparse_moe.experts.0.w2.weight": "model-00024-of-00029.safetensors",
+    "model.layers.26.block_sparse_moe.experts.0.w3.weight": "model-00024-of-00029.safetensors",
+    "model.layers.26.block_sparse_moe.experts.1.w1.weight": "model-00024-of-00029.safetensors",
+    "model.layers.26.block_sparse_moe.experts.1.w2.weight": "model-00024-of-00029.safetensors",
+    "model.layers.26.block_sparse_moe.experts.1.w3.weight": "model-00024-of-00029.safetensors",
+    "model.layers.26.block_sparse_moe.experts.2.w1.weight": "model-00024-of-00029.safetensors",
+    "model.layers.26.block_sparse_moe.experts.2.w2.weight": "model-00024-of-00029.safetensors",
+    "model.layers.26.block_sparse_moe.experts.2.w3.weight": "model-00024-of-00029.safetensors",
+    "model.layers.26.block_sparse_moe.experts.3.w1.weight": "model-00024-of-00029.safetensors",
+    "model.layers.26.block_sparse_moe.experts.3.w2.weight": "model-00025-of-00029.safetensors",
+    "model.layers.26.block_sparse_moe.experts.3.w3.weight": "model-00025-of-00029.safetensors",
+    "model.layers.26.block_sparse_moe.experts.4.w1.weight": "model-00025-of-00029.safetensors",
+    "model.layers.26.block_sparse_moe.experts.4.w2.weight": "model-00025-of-00029.safetensors",
+    "model.layers.26.block_sparse_moe.experts.4.w3.weight": "model-00025-of-00029.safetensors",
+    "model.layers.26.block_sparse_moe.experts.5.w1.weight": "model-00025-of-00029.safetensors",
+    "model.layers.26.block_sparse_moe.experts.5.w2.weight": "model-00025-of-00029.safetensors",
+    "model.layers.26.block_sparse_moe.experts.5.w3.weight": "model-00025-of-00029.safetensors",
+    "model.layers.26.block_sparse_moe.gate.weight": "model-00024-of-00029.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00025-of-00029.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00025-of-00029.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00024-of-00029.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00024-of-00029.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00024-of-00029.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00024-of-00029.safetensors",
+    "model.layers.27.block_sparse_moe.experts.0.w1.weight": "model-00025-of-00029.safetensors",
+    "model.layers.27.block_sparse_moe.experts.0.w2.weight": "model-00025-of-00029.safetensors",
+    "model.layers.27.block_sparse_moe.experts.0.w3.weight": "model-00025-of-00029.safetensors",
+    "model.layers.27.block_sparse_moe.experts.1.w1.weight": "model-00025-of-00029.safetensors",
+    "model.layers.27.block_sparse_moe.experts.1.w2.weight": "model-00025-of-00029.safetensors",
+    "model.layers.27.block_sparse_moe.experts.1.w3.weight": "model-00025-of-00029.safetensors",
+    "model.layers.27.block_sparse_moe.experts.2.w1.weight": "model-00025-of-00029.safetensors",
+    "model.layers.27.block_sparse_moe.experts.2.w2.weight": "model-00025-of-00029.safetensors",
+    "model.layers.27.block_sparse_moe.experts.2.w3.weight": "model-00025-of-00029.safetensors",
+    "model.layers.27.block_sparse_moe.experts.3.w1.weight": "model-00025-of-00029.safetensors",
+    "model.layers.27.block_sparse_moe.experts.3.w2.weight": "model-00025-of-00029.safetensors",
+    "model.layers.27.block_sparse_moe.experts.3.w3.weight": "model-00025-of-00029.safetensors",
+    "model.layers.27.block_sparse_moe.experts.4.w1.weight": "model-00026-of-00029.safetensors",
+    "model.layers.27.block_sparse_moe.experts.4.w2.weight": "model-00026-of-00029.safetensors",
+    "model.layers.27.block_sparse_moe.experts.4.w3.weight": "model-00026-of-00029.safetensors",
+    "model.layers.27.block_sparse_moe.experts.5.w1.weight": "model-00026-of-00029.safetensors",
+    "model.layers.27.block_sparse_moe.experts.5.w2.weight": "model-00026-of-00029.safetensors",
+    "model.layers.27.block_sparse_moe.experts.5.w3.weight": "model-00026-of-00029.safetensors",
+    "model.layers.27.block_sparse_moe.gate.weight": "model-00025-of-00029.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00026-of-00029.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00026-of-00029.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00025-of-00029.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00025-of-00029.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00025-of-00029.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00025-of-00029.safetensors",
+    "model.layers.28.block_sparse_moe.experts.0.w1.weight": "model-00026-of-00029.safetensors",
+    "model.layers.28.block_sparse_moe.experts.0.w2.weight": "model-00026-of-00029.safetensors",
+    "model.layers.28.block_sparse_moe.experts.0.w3.weight": "model-00026-of-00029.safetensors",
+    "model.layers.28.block_sparse_moe.experts.1.w1.weight": "model-00026-of-00029.safetensors",
+    "model.layers.28.block_sparse_moe.experts.1.w2.weight": "model-00026-of-00029.safetensors",
+    "model.layers.28.block_sparse_moe.experts.1.w3.weight": "model-00026-of-00029.safetensors",
+    "model.layers.28.block_sparse_moe.experts.2.w1.weight": "model-00026-of-00029.safetensors",
+    "model.layers.28.block_sparse_moe.experts.2.w2.weight": "model-00026-of-00029.safetensors",
+    "model.layers.28.block_sparse_moe.experts.2.w3.weight": "model-00026-of-00029.safetensors",
+    "model.layers.28.block_sparse_moe.experts.3.w1.weight": "model-00026-of-00029.safetensors",
+    "model.layers.28.block_sparse_moe.experts.3.w2.weight": "model-00026-of-00029.safetensors",
+    "model.layers.28.block_sparse_moe.experts.3.w3.weight": "model-00026-of-00029.safetensors",
+    "model.layers.28.block_sparse_moe.experts.4.w1.weight": "model-00026-of-00029.safetensors",
+    "model.layers.28.block_sparse_moe.experts.4.w2.weight": "model-00026-of-00029.safetensors",
+    "model.layers.28.block_sparse_moe.experts.4.w3.weight": "model-00027-of-00029.safetensors",
+    "model.layers.28.block_sparse_moe.experts.5.w1.weight": "model-00027-of-00029.safetensors",
+    "model.layers.28.block_sparse_moe.experts.5.w2.weight": "model-00027-of-00029.safetensors",
+    "model.layers.28.block_sparse_moe.experts.5.w3.weight": "model-00027-of-00029.safetensors",
+    "model.layers.28.block_sparse_moe.gate.weight": "model-00026-of-00029.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00027-of-00029.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00027-of-00029.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00026-of-00029.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00026-of-00029.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00026-of-00029.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00026-of-00029.safetensors",
+    "model.layers.29.block_sparse_moe.experts.0.w1.weight": "model-00027-of-00029.safetensors",
+    "model.layers.29.block_sparse_moe.experts.0.w2.weight": "model-00027-of-00029.safetensors",
+    "model.layers.29.block_sparse_moe.experts.0.w3.weight": "model-00027-of-00029.safetensors",
+    "model.layers.29.block_sparse_moe.experts.1.w1.weight": "model-00027-of-00029.safetensors",
+    "model.layers.29.block_sparse_moe.experts.1.w2.weight": "model-00027-of-00029.safetensors",
+    "model.layers.29.block_sparse_moe.experts.1.w3.weight": "model-00027-of-00029.safetensors",
+    "model.layers.29.block_sparse_moe.experts.2.w1.weight": "model-00027-of-00029.safetensors",
+    "model.layers.29.block_sparse_moe.experts.2.w2.weight": "model-00027-of-00029.safetensors",
+    "model.layers.29.block_sparse_moe.experts.2.w3.weight": "model-00027-of-00029.safetensors",
+    "model.layers.29.block_sparse_moe.experts.3.w1.weight": "model-00027-of-00029.safetensors",
+    "model.layers.29.block_sparse_moe.experts.3.w2.weight": "model-00027-of-00029.safetensors",
+    "model.layers.29.block_sparse_moe.experts.3.w3.weight": "model-00027-of-00029.safetensors",
+    "model.layers.29.block_sparse_moe.experts.4.w1.weight": "model-00027-of-00029.safetensors",
+    "model.layers.29.block_sparse_moe.experts.4.w2.weight": "model-00027-of-00029.safetensors",
+    "model.layers.29.block_sparse_moe.experts.4.w3.weight": "model-00027-of-00029.safetensors",
+    "model.layers.29.block_sparse_moe.experts.5.w1.weight": "model-00027-of-00029.safetensors",
+    "model.layers.29.block_sparse_moe.experts.5.w2.weight": "model-00028-of-00029.safetensors",
+    "model.layers.29.block_sparse_moe.experts.5.w3.weight": "model-00028-of-00029.safetensors",
+    "model.layers.29.block_sparse_moe.gate.weight": "model-00027-of-00029.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00028-of-00029.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00028-of-00029.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00027-of-00029.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00027-of-00029.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00027-of-00029.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00027-of-00029.safetensors",
+    "model.layers.3.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00029.safetensors",
+    "model.layers.3.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00029.safetensors",
+    "model.layers.3.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00029.safetensors",
+    "model.layers.3.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00029.safetensors",
+    "model.layers.3.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00029.safetensors",
+    "model.layers.3.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00029.safetensors",
+    "model.layers.3.block_sparse_moe.experts.2.w1.weight": "model-00004-of-00029.safetensors",
+    "model.layers.3.block_sparse_moe.experts.2.w2.weight": "model-00004-of-00029.safetensors",
+    "model.layers.3.block_sparse_moe.experts.2.w3.weight": "model-00004-of-00029.safetensors",
+    "model.layers.3.block_sparse_moe.experts.3.w1.weight": "model-00004-of-00029.safetensors",
+    "model.layers.3.block_sparse_moe.experts.3.w2.weight": "model-00004-of-00029.safetensors",
+    "model.layers.3.block_sparse_moe.experts.3.w3.weight": "model-00004-of-00029.safetensors",
+    "model.layers.3.block_sparse_moe.experts.4.w1.weight": "model-00004-of-00029.safetensors",
+    "model.layers.3.block_sparse_moe.experts.4.w2.weight": "model-00004-of-00029.safetensors",
+    "model.layers.3.block_sparse_moe.experts.4.w3.weight": "model-00004-of-00029.safetensors",
+    "model.layers.3.block_sparse_moe.experts.5.w1.weight": "model-00004-of-00029.safetensors",
+    "model.layers.3.block_sparse_moe.experts.5.w2.weight": "model-00004-of-00029.safetensors",
+    "model.layers.3.block_sparse_moe.experts.5.w3.weight": "model-00004-of-00029.safetensors",
+    "model.layers.3.block_sparse_moe.gate.weight": "model-00003-of-00029.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00004-of-00029.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00004-of-00029.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00003-of-00029.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00003-of-00029.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00003-of-00029.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00003-of-00029.safetensors",
+    "model.layers.30.block_sparse_moe.experts.0.w1.weight": "model-00028-of-00029.safetensors",
+    "model.layers.30.block_sparse_moe.experts.0.w2.weight": "model-00028-of-00029.safetensors",
+    "model.layers.30.block_sparse_moe.experts.0.w3.weight": "model-00028-of-00029.safetensors",
+    "model.layers.30.block_sparse_moe.experts.1.w1.weight": "model-00028-of-00029.safetensors",
+    "model.layers.30.block_sparse_moe.experts.1.w2.weight": "model-00028-of-00029.safetensors",
+    "model.layers.30.block_sparse_moe.experts.1.w3.weight": "model-00028-of-00029.safetensors",
+    "model.layers.30.block_sparse_moe.experts.2.w1.weight": "model-00028-of-00029.safetensors",
+    "model.layers.30.block_sparse_moe.experts.2.w2.weight": "model-00028-of-00029.safetensors",
+    "model.layers.30.block_sparse_moe.experts.2.w3.weight": "model-00028-of-00029.safetensors",
+    "model.layers.30.block_sparse_moe.experts.3.w1.weight": "model-00028-of-00029.safetensors",
+    "model.layers.30.block_sparse_moe.experts.3.w2.weight": "model-00028-of-00029.safetensors",
+    "model.layers.30.block_sparse_moe.experts.3.w3.weight": "model-00028-of-00029.safetensors",
+    "model.layers.30.block_sparse_moe.experts.4.w1.weight": "model-00028-of-00029.safetensors",
+    "model.layers.30.block_sparse_moe.experts.4.w2.weight": "model-00028-of-00029.safetensors",
+    "model.layers.30.block_sparse_moe.experts.4.w3.weight": "model-00028-of-00029.safetensors",
+    "model.layers.30.block_sparse_moe.experts.5.w1.weight": "model-00028-of-00029.safetensors",
+    "model.layers.30.block_sparse_moe.experts.5.w2.weight": "model-00028-of-00029.safetensors",
+    "model.layers.30.block_sparse_moe.experts.5.w3.weight": "model-00028-of-00029.safetensors",
+    "model.layers.30.block_sparse_moe.gate.weight": "model-00028-of-00029.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00028-of-00029.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00028-of-00029.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00028-of-00029.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00028-of-00029.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00028-of-00029.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00028-of-00029.safetensors",
+    "model.layers.31.block_sparse_moe.experts.0.w1.weight": "model-00029-of-00029.safetensors",
+    "model.layers.31.block_sparse_moe.experts.0.w2.weight": "model-00029-of-00029.safetensors",
+    "model.layers.31.block_sparse_moe.experts.0.w3.weight": "model-00029-of-00029.safetensors",
+    "model.layers.31.block_sparse_moe.experts.1.w1.weight": "model-00029-of-00029.safetensors",
+    "model.layers.31.block_sparse_moe.experts.1.w2.weight": "model-00029-of-00029.safetensors",
+    "model.layers.31.block_sparse_moe.experts.1.w3.weight": "model-00029-of-00029.safetensors",
+    "model.layers.31.block_sparse_moe.experts.2.w1.weight": "model-00029-of-00029.safetensors",
+    "model.layers.31.block_sparse_moe.experts.2.w2.weight": "model-00029-of-00029.safetensors",
+    "model.layers.31.block_sparse_moe.experts.2.w3.weight": "model-00029-of-00029.safetensors",
+    "model.layers.31.block_sparse_moe.experts.3.w1.weight": "model-00029-of-00029.safetensors",
+    "model.layers.31.block_sparse_moe.experts.3.w2.weight": "model-00029-of-00029.safetensors",
+    "model.layers.31.block_sparse_moe.experts.3.w3.weight": "model-00029-of-00029.safetensors",
+    "model.layers.31.block_sparse_moe.experts.4.w1.weight": "model-00029-of-00029.safetensors",
+    "model.layers.31.block_sparse_moe.experts.4.w2.weight": "model-00029-of-00029.safetensors",
+    "model.layers.31.block_sparse_moe.experts.4.w3.weight": "model-00029-of-00029.safetensors",
+    "model.layers.31.block_sparse_moe.experts.5.w1.weight": "model-00029-of-00029.safetensors",
+    "model.layers.31.block_sparse_moe.experts.5.w2.weight": "model-00029-of-00029.safetensors",
+    "model.layers.31.block_sparse_moe.experts.5.w3.weight": "model-00029-of-00029.safetensors",
+    "model.layers.31.block_sparse_moe.gate.weight": "model-00029-of-00029.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00029-of-00029.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00029-of-00029.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00028-of-00029.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00029-of-00029.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00028-of-00029.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00028-of-00029.safetensors",
+    "model.layers.4.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00029.safetensors",
+    "model.layers.4.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00029.safetensors",
+    "model.layers.4.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00029.safetensors",
+    "model.layers.4.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00029.safetensors",
+    "model.layers.4.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00029.safetensors",
+    "model.layers.4.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00029.safetensors",
+    "model.layers.4.block_sparse_moe.experts.2.w1.weight": "model-00005-of-00029.safetensors",
+    "model.layers.4.block_sparse_moe.experts.2.w2.weight": "model-00005-of-00029.safetensors",
+    "model.layers.4.block_sparse_moe.experts.2.w3.weight": "model-00005-of-00029.safetensors",
+    "model.layers.4.block_sparse_moe.experts.3.w1.weight": "model-00005-of-00029.safetensors",
+    "model.layers.4.block_sparse_moe.experts.3.w2.weight": "model-00005-of-00029.safetensors",
+    "model.layers.4.block_sparse_moe.experts.3.w3.weight": "model-00005-of-00029.safetensors",
+    "model.layers.4.block_sparse_moe.experts.4.w1.weight": "model-00005-of-00029.safetensors",
+    "model.layers.4.block_sparse_moe.experts.4.w2.weight": "model-00005-of-00029.safetensors",
+    "model.layers.4.block_sparse_moe.experts.4.w3.weight": "model-00005-of-00029.safetensors",
+    "model.layers.4.block_sparse_moe.experts.5.w1.weight": "model-00005-of-00029.safetensors",
+    "model.layers.4.block_sparse_moe.experts.5.w2.weight": "model-00005-of-00029.safetensors",
+    "model.layers.4.block_sparse_moe.experts.5.w3.weight": "model-00005-of-00029.safetensors",
+    "model.layers.4.block_sparse_moe.gate.weight": "model-00004-of-00029.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00005-of-00029.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00005-of-00029.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00004-of-00029.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00004-of-00029.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00004-of-00029.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00004-of-00029.safetensors",
+    "model.layers.5.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00029.safetensors",
+    "model.layers.5.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00029.safetensors",
+    "model.layers.5.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00029.safetensors",
+    "model.layers.5.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00029.safetensors",
+    "model.layers.5.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00029.safetensors",
+    "model.layers.5.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00029.safetensors",
+    "model.layers.5.block_sparse_moe.experts.2.w1.weight": "model-00005-of-00029.safetensors",
+    "model.layers.5.block_sparse_moe.experts.2.w2.weight": "model-00005-of-00029.safetensors",
+    "model.layers.5.block_sparse_moe.experts.2.w3.weight": "model-00006-of-00029.safetensors",
+    "model.layers.5.block_sparse_moe.experts.3.w1.weight": "model-00006-of-00029.safetensors",
+    "model.layers.5.block_sparse_moe.experts.3.w2.weight": "model-00006-of-00029.safetensors",
+    "model.layers.5.block_sparse_moe.experts.3.w3.weight": "model-00006-of-00029.safetensors",
+    "model.layers.5.block_sparse_moe.experts.4.w1.weight": "model-00006-of-00029.safetensors",
+    "model.layers.5.block_sparse_moe.experts.4.w2.weight": "model-00006-of-00029.safetensors",
+    "model.layers.5.block_sparse_moe.experts.4.w3.weight": "model-00006-of-00029.safetensors",
+    "model.layers.5.block_sparse_moe.experts.5.w1.weight": "model-00006-of-00029.safetensors",
+    "model.layers.5.block_sparse_moe.experts.5.w2.weight": "model-00006-of-00029.safetensors",
+    "model.layers.5.block_sparse_moe.experts.5.w3.weight": "model-00006-of-00029.safetensors",
+    "model.layers.5.block_sparse_moe.gate.weight": "model-00005-of-00029.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00006-of-00029.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00006-of-00029.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00005-of-00029.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00005-of-00029.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00005-of-00029.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00005-of-00029.safetensors",
+    "model.layers.6.block_sparse_moe.experts.0.w1.weight": "model-00006-of-00029.safetensors",
+    "model.layers.6.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00029.safetensors",
+    "model.layers.6.block_sparse_moe.experts.0.w3.weight": "model-00006-of-00029.safetensors",
+    "model.layers.6.block_sparse_moe.experts.1.w1.weight": "model-00006-of-00029.safetensors",
+    "model.layers.6.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00029.safetensors",
+    "model.layers.6.block_sparse_moe.experts.1.w3.weight": "model-00006-of-00029.safetensors",
+    "model.layers.6.block_sparse_moe.experts.2.w1.weight": "model-00006-of-00029.safetensors",
+    "model.layers.6.block_sparse_moe.experts.2.w2.weight": "model-00006-of-00029.safetensors",
+    "model.layers.6.block_sparse_moe.experts.2.w3.weight": "model-00006-of-00029.safetensors",
+    "model.layers.6.block_sparse_moe.experts.3.w1.weight": "model-00006-of-00029.safetensors",
+    "model.layers.6.block_sparse_moe.experts.3.w2.weight": "model-00007-of-00029.safetensors",
+    "model.layers.6.block_sparse_moe.experts.3.w3.weight": "model-00007-of-00029.safetensors",
+    "model.layers.6.block_sparse_moe.experts.4.w1.weight": "model-00007-of-00029.safetensors",
+    "model.layers.6.block_sparse_moe.experts.4.w2.weight": "model-00007-of-00029.safetensors",
+    "model.layers.6.block_sparse_moe.experts.4.w3.weight": "model-00007-of-00029.safetensors",
+    "model.layers.6.block_sparse_moe.experts.5.w1.weight": "model-00007-of-00029.safetensors",
+    "model.layers.6.block_sparse_moe.experts.5.w2.weight": "model-00007-of-00029.safetensors",
+    "model.layers.6.block_sparse_moe.experts.5.w3.weight": "model-00007-of-00029.safetensors",
+    "model.layers.6.block_sparse_moe.gate.weight": "model-00006-of-00029.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00007-of-00029.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00007-of-00029.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00006-of-00029.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00006-of-00029.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00006-of-00029.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00006-of-00029.safetensors",
+    "model.layers.7.block_sparse_moe.experts.0.w1.weight": "model-00007-of-00029.safetensors",
+    "model.layers.7.block_sparse_moe.experts.0.w2.weight": "model-00007-of-00029.safetensors",
+    "model.layers.7.block_sparse_moe.experts.0.w3.weight": "model-00007-of-00029.safetensors",
+    "model.layers.7.block_sparse_moe.experts.1.w1.weight": "model-00007-of-00029.safetensors",
+    "model.layers.7.block_sparse_moe.experts.1.w2.weight": "model-00007-of-00029.safetensors",
+    "model.layers.7.block_sparse_moe.experts.1.w3.weight": "model-00007-of-00029.safetensors",
+    "model.layers.7.block_sparse_moe.experts.2.w1.weight": "model-00007-of-00029.safetensors",
+    "model.layers.7.block_sparse_moe.experts.2.w2.weight": "model-00007-of-00029.safetensors",
+    "model.layers.7.block_sparse_moe.experts.2.w3.weight": "model-00007-of-00029.safetensors",
+    "model.layers.7.block_sparse_moe.experts.3.w1.weight": "model-00007-of-00029.safetensors",
+    "model.layers.7.block_sparse_moe.experts.3.w2.weight": "model-00007-of-00029.safetensors",
+    "model.layers.7.block_sparse_moe.experts.3.w3.weight": "model-00007-of-00029.safetensors",
+    "model.layers.7.block_sparse_moe.experts.4.w1.weight": "model-00008-of-00029.safetensors",
+    "model.layers.7.block_sparse_moe.experts.4.w2.weight": "model-00008-of-00029.safetensors",
+    "model.layers.7.block_sparse_moe.experts.4.w3.weight": "model-00008-of-00029.safetensors",
+    "model.layers.7.block_sparse_moe.experts.5.w1.weight": "model-00008-of-00029.safetensors",
+    "model.layers.7.block_sparse_moe.experts.5.w2.weight": "model-00008-of-00029.safetensors",
+    "model.layers.7.block_sparse_moe.experts.5.w3.weight": "model-00008-of-00029.safetensors",
+    "model.layers.7.block_sparse_moe.gate.weight": "model-00007-of-00029.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00008-of-00029.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00008-of-00029.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00007-of-00029.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00007-of-00029.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00007-of-00029.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00007-of-00029.safetensors",
+    "model.layers.8.block_sparse_moe.experts.0.w1.weight": "model-00008-of-00029.safetensors",
+    "model.layers.8.block_sparse_moe.experts.0.w2.weight": "model-00008-of-00029.safetensors",
+    "model.layers.8.block_sparse_moe.experts.0.w3.weight": "model-00008-of-00029.safetensors",
+    "model.layers.8.block_sparse_moe.experts.1.w1.weight": "model-00008-of-00029.safetensors",
+    "model.layers.8.block_sparse_moe.experts.1.w2.weight": "model-00008-of-00029.safetensors",
+    "model.layers.8.block_sparse_moe.experts.1.w3.weight": "model-00008-of-00029.safetensors",
+    "model.layers.8.block_sparse_moe.experts.2.w1.weight": "model-00008-of-00029.safetensors",
+    "model.layers.8.block_sparse_moe.experts.2.w2.weight": "model-00008-of-00029.safetensors",
+    "model.layers.8.block_sparse_moe.experts.2.w3.weight": "model-00008-of-00029.safetensors",
+    "model.layers.8.block_sparse_moe.experts.3.w1.weight": "model-00008-of-00029.safetensors",
+    "model.layers.8.block_sparse_moe.experts.3.w2.weight": "model-00008-of-00029.safetensors",
+    "model.layers.8.block_sparse_moe.experts.3.w3.weight": "model-00008-of-00029.safetensors",
+    "model.layers.8.block_sparse_moe.experts.4.w1.weight": "model-00008-of-00029.safetensors",
+    "model.layers.8.block_sparse_moe.experts.4.w2.weight": "model-00008-of-00029.safetensors",
+    "model.layers.8.block_sparse_moe.experts.4.w3.weight": "model-00009-of-00029.safetensors",
+    "model.layers.8.block_sparse_moe.experts.5.w1.weight": "model-00009-of-00029.safetensors",
+    "model.layers.8.block_sparse_moe.experts.5.w2.weight": "model-00009-of-00029.safetensors",
+    "model.layers.8.block_sparse_moe.experts.5.w3.weight": "model-00009-of-00029.safetensors",
+    "model.layers.8.block_sparse_moe.gate.weight": "model-00008-of-00029.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00009-of-00029.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00009-of-00029.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00008-of-00029.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00008-of-00029.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00008-of-00029.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00008-of-00029.safetensors",
+    "model.layers.9.block_sparse_moe.experts.0.w1.weight": "model-00009-of-00029.safetensors",
+    "model.layers.9.block_sparse_moe.experts.0.w2.weight": "model-00009-of-00029.safetensors",
+    "model.layers.9.block_sparse_moe.experts.0.w3.weight": "model-00009-of-00029.safetensors",
+    "model.layers.9.block_sparse_moe.experts.1.w1.weight": "model-00009-of-00029.safetensors",
+    "model.layers.9.block_sparse_moe.experts.1.w2.weight": "model-00009-of-00029.safetensors",
+    "model.layers.9.block_sparse_moe.experts.1.w3.weight": "model-00009-of-00029.safetensors",
+    "model.layers.9.block_sparse_moe.experts.2.w1.weight": "model-00009-of-00029.safetensors",
+    "model.layers.9.block_sparse_moe.experts.2.w2.weight": "model-00009-of-00029.safetensors",
+    "model.layers.9.block_sparse_moe.experts.2.w3.weight": "model-00009-of-00029.safetensors",
+    "model.layers.9.block_sparse_moe.experts.3.w1.weight": "model-00009-of-00029.safetensors",
+    "model.layers.9.block_sparse_moe.experts.3.w2.weight": "model-00009-of-00029.safetensors",
+    "model.layers.9.block_sparse_moe.experts.3.w3.weight": "model-00009-of-00029.safetensors",
+    "model.layers.9.block_sparse_moe.experts.4.w1.weight": "model-00009-of-00029.safetensors",
+    "model.layers.9.block_sparse_moe.experts.4.w2.weight": "model-00009-of-00029.safetensors",
+    "model.layers.9.block_sparse_moe.experts.4.w3.weight": "model-00009-of-00029.safetensors",
+    "model.layers.9.block_sparse_moe.experts.5.w1.weight": "model-00009-of-00029.safetensors",
+    "model.layers.9.block_sparse_moe.experts.5.w2.weight": "model-00010-of-00029.safetensors",
+    "model.layers.9.block_sparse_moe.experts.5.w3.weight": "model-00010-of-00029.safetensors",
+    "model.layers.9.block_sparse_moe.gate.weight": "model-00009-of-00029.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00010-of-00029.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00010-of-00029.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00009-of-00029.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00009-of-00029.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00009-of-00029.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00009-of-00029.safetensors",
+    "model.norm.weight": "model-00029-of-00029.safetensors"
+  }
+}