change stuff

config.json

@@ -1,42 +0,0 @@
-{
-    "_name_or_path": "/content/InfinityRP",
-    "architectures": [
-        "MixtralForCausalLM"
-    ],
-    "attention_dropout": 0.0,
-    "bos_token_id": 1,
-    "eos_token_id": 2,
-    "hidden_act": "silu",
-    "hidden_size": 4096,
-    "initializer_range": 0.02,
-    "intermediate_size": 14336,
-    "max_position_embeddings": 32768,
-    "model_type": "mixtral",
-    "num_attention_heads": 32,
-    "num_experts_per_tok": 2,
-    "num_hidden_layers": 32,
-    "num_key_value_heads": 8,
-    "num_local_experts": 4,
-    "output_router_logits": false,
-    "rms_norm_eps": 1e-05,
-    "rope_theta": 10000.0,
-    "router_aux_loss_coef": 0.001,
-    "router_jitter_noise": 0.0,
-    "sliding_window": null,
-    "tie_word_embeddings": false,
-    "torch_dtype": "bfloat16",
-    "transformers_version": "4.40.0.dev0",
-    "use_cache": true,
-    "vocab_size": 32000,
-    "quantization_config": {
-        "quant_method": "exl2",
-        "version": "0.0.20",
-        "bits": 3.2,
-        "head_bits": 8,
-        "calibration": {
-            "rows": 100,
-            "length": 2048,
-            "dataset": "(default)"
-        }
-    }
-}

huggingface-metadata.txt

@@ -1,10 +0,0 @@
-url: https://huggingface.co/R136a1/BeyondInfinity-4x7B
-branch: main
-download date: 2024-05-06 19:27:46
-sha256sum:
-    4d45ac7f560e95ecb3e7acb5b47c06276b98197b540e1da375945c15c15b2f12 model-00001-of-00005.safetensors
-    ac500adcaca5544723877ed9ae8d5f2184d7003ef66bfe6d01cdb45116af8277 model-00002-of-00005.safetensors
-    eff6922820f01d00ea70b964038e65a3cd4daec3152c2a959b963fb0a0eec976 model-00003-of-00005.safetensors
-    ddeb4d5d7691179f7304a5a46c1bb32d0b063a488de9eed61f5ec0aaddd2ec5a model-00004-of-00005.safetensors
-    e2dcbb31e8f209ea5ba14fba800f67d8ff859a5679859cb4becdc5d687b7baad model-00005-of-00005.safetensors
-    dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055 tokenizer.model

model.safetensors.index.json

@@ -1 +0,0 @@
-{"metadata": {"mergekit_version": "0.0.4.2", "total_size": 48307380224}, "weight_map": {"model.embed_tokens.weight": "model-00001-of-00005.safetensors", "model.norm.weight": "model-00001-of-00005.safetensors", "lm_head.weight": "model-00001-of-00005.safetensors", "model.layers.0.input_layernorm.weight": "model-00001-of-00005.safetensors", "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00005.safetensors", "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00005.safetensors", "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", "model.layers.0.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00005.safetensors", "model.layers.0.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00005.safetensors", "model.layers.0.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00005.safetensors", "model.layers.0.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00005.safetensors", "model.layers.0.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00005.safetensors", "model.layers.0.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00005.safetensors", "model.layers.0.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00005.safetensors", "model.layers.0.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00005.safetensors", "model.layers.0.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00005.safetensors", "model.layers.0.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00005.safetensors", "model.layers.0.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00005.safetensors", "model.layers.0.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00005.safetensors", "model.layers.1.input_layernorm.weight": "model-00001-of-00005.safetensors", "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00005.safetensors", "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00005.safetensors", "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", "model.layers.1.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00005.safetensors", "model.layers.1.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00005.safetensors", "model.layers.1.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00005.safetensors", "model.layers.1.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00005.safetensors", "model.layers.1.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00005.safetensors", "model.layers.1.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00005.safetensors", "model.layers.1.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00005.safetensors", "model.layers.1.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00005.safetensors", "model.layers.1.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00005.safetensors", "model.layers.1.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00005.safetensors", "model.layers.1.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00005.safetensors", "model.layers.1.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00005.safetensors", "model.layers.2.input_layernorm.weight": "model-00001-of-00005.safetensors", "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00005.safetensors", "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00005.safetensors", "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", "model.layers.2.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00005.safetensors", "model.layers.2.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00005.safetensors", "model.layers.2.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00005.safetensors", "model.layers.2.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00005.safetensors", "model.layers.2.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00005.safetensors", "model.layers.2.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00005.safetensors", "model.layers.2.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00005.safetensors", "model.layers.2.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00005.safetensors", "model.layers.2.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00005.safetensors", "model.layers.2.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00005.safetensors", "model.layers.2.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00005.safetensors", "model.layers.2.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00005.safetensors", "model.layers.3.input_layernorm.weight": "model-00001-of-00005.safetensors", "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00005.safetensors", "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00005.safetensors", "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", "model.layers.3.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00005.safetensors", "model.layers.3.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00005.safetensors", "model.layers.3.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00005.safetensors", "model.layers.3.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00005.safetensors", "model.layers.3.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00005.safetensors", "model.layers.3.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00005.safetensors", "model.layers.3.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00005.safetensors", "model.layers.3.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00005.safetensors", "model.layers.3.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00005.safetensors", "model.layers.3.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00005.safetensors", "model.layers.3.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00005.safetensors", "model.layers.3.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00005.safetensors", "model.layers.4.input_layernorm.weight": "model-00001-of-00005.safetensors", "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00005.safetensors", "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00005.safetensors", "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", "model.layers.4.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00005.safetensors", "model.layers.4.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00005.safetensors", "model.layers.4.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00005.safetensors", "model.layers.4.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00005.safetensors", "model.layers.4.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00005.safetensors", "model.layers.4.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00005.safetensors", "model.layers.4.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00005.safetensors", "model.layers.4.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00005.safetensors", "model.layers.4.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00005.safetensors", "model.layers.4.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00005.safetensors", "model.layers.4.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00005.safetensors", "model.layers.4.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00005.safetensors", "model.layers.5.input_layernorm.weight": "model-00001-of-00005.safetensors", "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00005.safetensors", "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00005.safetensors", "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", "model.layers.5.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00005.safetensors", "model.layers.5.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00005.safetensors", "model.layers.5.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00005.safetensors", "model.layers.5.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00005.safetensors", "model.layers.5.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00005.safetensors", "model.layers.5.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00005.safetensors", "model.layers.5.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00005.safetensors", "model.layers.5.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00005.safetensors", "model.layers.5.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00005.safetensors", "model.layers.5.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00005.safetensors", "model.layers.5.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00005.safetensors", "model.layers.5.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00005.safetensors", "model.layers.6.input_layernorm.weight": "model-00001-of-00005.safetensors", "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00005.safetensors", "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00005.safetensors", "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00005.safetensors", "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00005.safetensors", "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00005.safetensors", "model.layers.6.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00005.safetensors", "model.layers.6.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00005.safetensors", "model.layers.6.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00005.safetensors", "model.layers.6.block_sparse_moe.experts.3.w3.weight": "model-00002-of-00005.safetensors", "model.layers.6.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00005.safetensors", "model.layers.6.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00005.safetensors", "model.layers.6.block_sparse_moe.experts.2.w1.weight": "model-00002-of-00005.safetensors", "model.layers.6.block_sparse_moe.experts.3.w1.weight": "model-00002-of-00005.safetensors", "model.layers.6.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00005.safetensors", "model.layers.6.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00005.safetensors", "model.layers.6.block_sparse_moe.experts.2.w2.weight": "model-00002-of-00005.safetensors", "model.layers.6.block_sparse_moe.experts.3.w2.weight": "model-00002-of-00005.safetensors", "model.layers.7.input_layernorm.weight": "model-00002-of-00005.safetensors", "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", "model.layers.7.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00005.safetensors", "model.layers.7.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00005.safetensors", "model.layers.7.block_sparse_moe.experts.2.w3.weight": "model-00002-of-00005.safetensors", "model.layers.7.block_sparse_moe.experts.3.w3.weight": "model-00002-of-00005.safetensors", "model.layers.7.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00005.safetensors", "model.layers.7.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00005.safetensors", "model.layers.7.block_sparse_moe.experts.2.w1.weight": "model-00002-of-00005.safetensors", "model.layers.7.block_sparse_moe.experts.3.w1.weight": "model-00002-of-00005.safetensors", "model.layers.7.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00005.safetensors", "model.layers.7.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00005.safetensors", "model.layers.7.block_sparse_moe.experts.2.w2.weight": "model-00002-of-00005.safetensors", "model.layers.7.block_sparse_moe.experts.3.w2.weight": "model-00002-of-00005.safetensors", "model.layers.8.input_layernorm.weight": "model-00002-of-00005.safetensors", "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", "model.layers.8.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00005.safetensors", "model.layers.8.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00005.safetensors", "model.layers.8.block_sparse_moe.experts.2.w3.weight": "model-00002-of-00005.safetensors", "model.layers.8.block_sparse_moe.experts.3.w3.weight": "model-00002-of-00005.safetensors", "model.layers.8.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00005.safetensors", "model.layers.8.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00005.safetensors", "model.layers.8.block_sparse_moe.experts.2.w1.weight": "model-00002-of-00005.safetensors", "model.layers.8.block_sparse_moe.experts.3.w1.weight": "model-00002-of-00005.safetensors", "model.layers.8.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00005.safetensors", "model.layers.8.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00005.safetensors", "model.layers.8.block_sparse_moe.experts.2.w2.weight": "model-00002-of-00005.safetensors", "model.layers.8.block_sparse_moe.experts.3.w2.weight": "model-00002-of-00005.safetensors", "model.layers.9.input_layernorm.weight": "model-00002-of-00005.safetensors", "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", "model.layers.9.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00005.safetensors", "model.layers.9.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00005.safetensors", "model.layers.9.block_sparse_moe.experts.2.w3.weight": "model-00002-of-00005.safetensors", "model.layers.9.block_sparse_moe.experts.3.w3.weight": "model-00002-of-00005.safetensors", "model.layers.9.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00005.safetensors", "model.layers.9.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00005.safetensors", "model.layers.9.block_sparse_moe.experts.2.w1.weight": "model-00002-of-00005.safetensors", "model.layers.9.block_sparse_moe.experts.3.w1.weight": "model-00002-of-00005.safetensors", "model.layers.9.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00005.safetensors", "model.layers.9.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00005.safetensors", "model.layers.9.block_sparse_moe.experts.2.w2.weight": "model-00002-of-00005.safetensors", "model.layers.9.block_sparse_moe.experts.3.w2.weight": "model-00002-of-00005.safetensors", "model.layers.10.input_layernorm.weight": "model-00002-of-00005.safetensors", "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", "model.layers.10.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00005.safetensors", "model.layers.10.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00005.safetensors", "model.layers.10.block_sparse_moe.experts.2.w3.weight": "model-00002-of-00005.safetensors", "model.layers.10.block_sparse_moe.experts.3.w3.weight": "model-00002-of-00005.safetensors", "model.layers.10.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00005.safetensors", "model.layers.10.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00005.safetensors", "model.layers.10.block_sparse_moe.experts.2.w1.weight": "model-00002-of-00005.safetensors", "model.layers.10.block_sparse_moe.experts.3.w1.weight": "model-00002-of-00005.safetensors", "model.layers.10.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00005.safetensors", "model.layers.10.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00005.safetensors", "model.layers.10.block_sparse_moe.experts.2.w2.weight": "model-00002-of-00005.safetensors", "model.layers.10.block_sparse_moe.experts.3.w2.weight": "model-00002-of-00005.safetensors", "model.layers.11.input_layernorm.weight": "model-00002-of-00005.safetensors", "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", "model.layers.11.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00005.safetensors", "model.layers.11.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00005.safetensors", "model.layers.11.block_sparse_moe.experts.2.w3.weight": "model-00002-of-00005.safetensors", "model.layers.11.block_sparse_moe.experts.3.w3.weight": "model-00002-of-00005.safetensors", "model.layers.11.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00005.safetensors", "model.layers.11.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00005.safetensors", "model.layers.11.block_sparse_moe.experts.2.w1.weight": "model-00002-of-00005.safetensors", "model.layers.11.block_sparse_moe.experts.3.w1.weight": "model-00002-of-00005.safetensors", "model.layers.11.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00005.safetensors", "model.layers.11.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00005.safetensors", "model.layers.11.block_sparse_moe.experts.2.w2.weight": "model-00002-of-00005.safetensors", "model.layers.11.block_sparse_moe.experts.3.w2.weight": "model-00002-of-00005.safetensors", "model.layers.12.input_layernorm.weight": "model-00002-of-00005.safetensors", "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00005.safetensors", "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00005.safetensors", "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00005.safetensors", "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00005.safetensors", "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00005.safetensors", "model.layers.12.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00005.safetensors", "model.layers.12.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00005.safetensors", "model.layers.12.block_sparse_moe.experts.2.w3.weight": "model-00002-of-00005.safetensors", "model.layers.12.block_sparse_moe.experts.3.w3.weight": "model-00002-of-00005.safetensors", "model.layers.12.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00005.safetensors", "model.layers.12.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00005.safetensors", "model.layers.12.block_sparse_moe.experts.2.w1.weight": "model-00002-of-00005.safetensors", "model.layers.12.block_sparse_moe.experts.3.w1.weight": "model-00002-of-00005.safetensors", "model.layers.12.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00005.safetensors", "model.layers.12.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00005.safetensors", "model.layers.12.block_sparse_moe.experts.2.w2.weight": "model-00002-of-00005.safetensors", "model.layers.12.block_sparse_moe.experts.3.w2.weight": "model-00003-of-00005.safetensors", "model.layers.13.input_layernorm.weight": "model-00003-of-00005.safetensors", "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", "model.layers.13.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00005.safetensors", "model.layers.13.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00005.safetensors", "model.layers.13.block_sparse_moe.experts.2.w3.weight": "model-00003-of-00005.safetensors", "model.layers.13.block_sparse_moe.experts.3.w3.weight": "model-00003-of-00005.safetensors", "model.layers.13.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00005.safetensors", "model.layers.13.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00005.safetensors", "model.layers.13.block_sparse_moe.experts.2.w1.weight": "model-00003-of-00005.safetensors", "model.layers.13.block_sparse_moe.experts.3.w1.weight": "model-00003-of-00005.safetensors", "model.layers.13.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00005.safetensors", "model.layers.13.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00005.safetensors", "model.layers.13.block_sparse_moe.experts.2.w2.weight": "model-00003-of-00005.safetensors", "model.layers.13.block_sparse_moe.experts.3.w2.weight": "model-00003-of-00005.safetensors", "model.layers.14.input_layernorm.weight": "model-00003-of-00005.safetensors", "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", "model.layers.14.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", "model.layers.14.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00005.safetensors", "model.layers.14.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00005.safetensors", "model.layers.14.block_sparse_moe.experts.2.w3.weight": "model-00003-of-00005.safetensors", "model.layers.14.block_sparse_moe.experts.3.w3.weight": "model-00003-of-00005.safetensors", "model.layers.14.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00005.safetensors", "model.layers.14.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00005.safetensors", "model.layers.14.block_sparse_moe.experts.2.w1.weight": "model-00003-of-00005.safetensors", "model.layers.14.block_sparse_moe.experts.3.w1.weight": "model-00003-of-00005.safetensors", "model.layers.14.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00005.safetensors", "model.layers.14.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00005.safetensors", "model.layers.14.block_sparse_moe.experts.2.w2.weight": "model-00003-of-00005.safetensors", "model.layers.14.block_sparse_moe.experts.3.w2.weight": "model-00003-of-00005.safetensors", "model.layers.15.input_layernorm.weight": "model-00003-of-00005.safetensors", "model.layers.15.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", "model.layers.15.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", "model.layers.15.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", "model.layers.15.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", "model.layers.15.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00005.safetensors", "model.layers.15.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00005.safetensors", "model.layers.15.block_sparse_moe.experts.2.w3.weight": "model-00003-of-00005.safetensors", "model.layers.15.block_sparse_moe.experts.3.w3.weight": "model-00003-of-00005.safetensors", "model.layers.15.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00005.safetensors", "model.layers.15.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00005.safetensors", "model.layers.15.block_sparse_moe.experts.2.w1.weight": "model-00003-of-00005.safetensors", "model.layers.15.block_sparse_moe.experts.3.w1.weight": "model-00003-of-00005.safetensors", "model.layers.15.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00005.safetensors", "model.layers.15.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00005.safetensors", "model.layers.15.block_sparse_moe.experts.2.w2.weight": "model-00003-of-00005.safetensors", "model.layers.15.block_sparse_moe.experts.3.w2.weight": "model-00003-of-00005.safetensors", "model.layers.16.input_layernorm.weight": "model-00003-of-00005.safetensors", "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", "model.layers.16.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00005.safetensors", "model.layers.16.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00005.safetensors", "model.layers.16.block_sparse_moe.experts.2.w3.weight": "model-00003-of-00005.safetensors", "model.layers.16.block_sparse_moe.experts.3.w3.weight": "model-00003-of-00005.safetensors", "model.layers.16.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00005.safetensors", "model.layers.16.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00005.safetensors", "model.layers.16.block_sparse_moe.experts.2.w1.weight": "model-00003-of-00005.safetensors", "model.layers.16.block_sparse_moe.experts.3.w1.weight": "model-00003-of-00005.safetensors", "model.layers.16.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00005.safetensors", "model.layers.16.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00005.safetensors", "model.layers.16.block_sparse_moe.experts.2.w2.weight": "model-00003-of-00005.safetensors", "model.layers.16.block_sparse_moe.experts.3.w2.weight": "model-00003-of-00005.safetensors", "model.layers.17.input_layernorm.weight": "model-00003-of-00005.safetensors", "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", "model.layers.17.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00005.safetensors", "model.layers.17.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00005.safetensors", "model.layers.17.block_sparse_moe.experts.2.w3.weight": "model-00003-of-00005.safetensors", "model.layers.17.block_sparse_moe.experts.3.w3.weight": "model-00003-of-00005.safetensors", "model.layers.17.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00005.safetensors", "model.layers.17.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00005.safetensors", "model.layers.17.block_sparse_moe.experts.2.w1.weight": "model-00003-of-00005.safetensors", "model.layers.17.block_sparse_moe.experts.3.w1.weight": "model-00003-of-00005.safetensors", "model.layers.17.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00005.safetensors", "model.layers.17.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00005.safetensors", "model.layers.17.block_sparse_moe.experts.2.w2.weight": "model-00003-of-00005.safetensors", "model.layers.17.block_sparse_moe.experts.3.w2.weight": "model-00003-of-00005.safetensors", "model.layers.18.input_layernorm.weight": "model-00003-of-00005.safetensors", "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", "model.layers.18.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00005.safetensors", "model.layers.18.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00005.safetensors", "model.layers.18.block_sparse_moe.experts.2.w3.weight": "model-00003-of-00005.safetensors", "model.layers.18.block_sparse_moe.experts.3.w3.weight": "model-00003-of-00005.safetensors", "model.layers.18.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00005.safetensors", "model.layers.18.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00005.safetensors", "model.layers.18.block_sparse_moe.experts.2.w1.weight": "model-00003-of-00005.safetensors", "model.layers.18.block_sparse_moe.experts.3.w1.weight": "model-00003-of-00005.safetensors", "model.layers.18.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00005.safetensors", "model.layers.18.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00005.safetensors", "model.layers.18.block_sparse_moe.experts.2.w2.weight": "model-00003-of-00005.safetensors", "model.layers.18.block_sparse_moe.experts.3.w2.weight": "model-00003-of-00005.safetensors", "model.layers.19.input_layernorm.weight": "model-00003-of-00005.safetensors", "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00005.safetensors", "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00005.safetensors", "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00005.safetensors", "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00005.safetensors", "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00005.safetensors", "model.layers.19.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00005.safetensors", "model.layers.19.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00005.safetensors", "model.layers.19.block_sparse_moe.experts.2.w3.weight": "model-00003-of-00005.safetensors", "model.layers.19.block_sparse_moe.experts.3.w3.weight": "model-00003-of-00005.safetensors", "model.layers.19.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00005.safetensors", "model.layers.19.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00005.safetensors", "model.layers.19.block_sparse_moe.experts.2.w1.weight": "model-00003-of-00005.safetensors", "model.layers.19.block_sparse_moe.experts.3.w1.weight": "model-00004-of-00005.safetensors", "model.layers.19.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00005.safetensors", "model.layers.19.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00005.safetensors", "model.layers.19.block_sparse_moe.experts.2.w2.weight": "model-00004-of-00005.safetensors", "model.layers.19.block_sparse_moe.experts.3.w2.weight": "model-00004-of-00005.safetensors", "model.layers.20.input_layernorm.weight": "model-00004-of-00005.safetensors", "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", "model.layers.20.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", "model.layers.20.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00005.safetensors", "model.layers.20.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00005.safetensors", "model.layers.20.block_sparse_moe.experts.2.w3.weight": "model-00004-of-00005.safetensors", "model.layers.20.block_sparse_moe.experts.3.w3.weight": "model-00004-of-00005.safetensors", "model.layers.20.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00005.safetensors", "model.layers.20.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00005.safetensors", "model.layers.20.block_sparse_moe.experts.2.w1.weight": "model-00004-of-00005.safetensors", "model.layers.20.block_sparse_moe.experts.3.w1.weight": "model-00004-of-00005.safetensors", "model.layers.20.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00005.safetensors", "model.layers.20.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00005.safetensors", "model.layers.20.block_sparse_moe.experts.2.w2.weight": "model-00004-of-00005.safetensors", "model.layers.20.block_sparse_moe.experts.3.w2.weight": "model-00004-of-00005.safetensors", "model.layers.21.input_layernorm.weight": "model-00004-of-00005.safetensors", "model.layers.21.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", "model.layers.21.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", "model.layers.21.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", "model.layers.21.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", "model.layers.21.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", "model.layers.21.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00005.safetensors", "model.layers.21.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00005.safetensors", "model.layers.21.block_sparse_moe.experts.2.w3.weight": "model-00004-of-00005.safetensors", "model.layers.21.block_sparse_moe.experts.3.w3.weight": "model-00004-of-00005.safetensors", "model.layers.21.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00005.safetensors", "model.layers.21.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00005.safetensors", "model.layers.21.block_sparse_moe.experts.2.w1.weight": "model-00004-of-00005.safetensors", "model.layers.21.block_sparse_moe.experts.3.w1.weight": "model-00004-of-00005.safetensors", "model.layers.21.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00005.safetensors", "model.layers.21.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00005.safetensors", "model.layers.21.block_sparse_moe.experts.2.w2.weight": "model-00004-of-00005.safetensors", "model.layers.21.block_sparse_moe.experts.3.w2.weight": "model-00004-of-00005.safetensors", "model.layers.22.input_layernorm.weight": "model-00004-of-00005.safetensors", "model.layers.22.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", "model.layers.22.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", "model.layers.22.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", "model.layers.22.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", "model.layers.22.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", "model.layers.22.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00005.safetensors", "model.layers.22.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00005.safetensors", "model.layers.22.block_sparse_moe.experts.2.w3.weight": "model-00004-of-00005.safetensors", "model.layers.22.block_sparse_moe.experts.3.w3.weight": "model-00004-of-00005.safetensors", "model.layers.22.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00005.safetensors", "model.layers.22.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00005.safetensors", "model.layers.22.block_sparse_moe.experts.2.w1.weight": "model-00004-of-00005.safetensors", "model.layers.22.block_sparse_moe.experts.3.w1.weight": "model-00004-of-00005.safetensors", "model.layers.22.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00005.safetensors", "model.layers.22.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00005.safetensors", "model.layers.22.block_sparse_moe.experts.2.w2.weight": "model-00004-of-00005.safetensors", "model.layers.22.block_sparse_moe.experts.3.w2.weight": "model-00004-of-00005.safetensors", "model.layers.23.input_layernorm.weight": "model-00004-of-00005.safetensors", "model.layers.23.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", "model.layers.23.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", "model.layers.23.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", "model.layers.23.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", "model.layers.23.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", "model.layers.23.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00005.safetensors", "model.layers.23.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00005.safetensors", "model.layers.23.block_sparse_moe.experts.2.w3.weight": "model-00004-of-00005.safetensors", "model.layers.23.block_sparse_moe.experts.3.w3.weight": "model-00004-of-00005.safetensors", "model.layers.23.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00005.safetensors", "model.layers.23.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00005.safetensors", "model.layers.23.block_sparse_moe.experts.2.w1.weight": "model-00004-of-00005.safetensors", "model.layers.23.block_sparse_moe.experts.3.w1.weight": "model-00004-of-00005.safetensors", "model.layers.23.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00005.safetensors", "model.layers.23.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00005.safetensors", "model.layers.23.block_sparse_moe.experts.2.w2.weight": "model-00004-of-00005.safetensors", "model.layers.23.block_sparse_moe.experts.3.w2.weight": "model-00004-of-00005.safetensors", "model.layers.24.input_layernorm.weight": "model-00004-of-00005.safetensors", "model.layers.24.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", "model.layers.24.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", "model.layers.24.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", "model.layers.24.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", "model.layers.24.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", "model.layers.24.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00005.safetensors", "model.layers.24.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00005.safetensors", "model.layers.24.block_sparse_moe.experts.2.w3.weight": "model-00004-of-00005.safetensors", "model.layers.24.block_sparse_moe.experts.3.w3.weight": "model-00004-of-00005.safetensors", "model.layers.24.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00005.safetensors", "model.layers.24.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00005.safetensors", "model.layers.24.block_sparse_moe.experts.2.w1.weight": "model-00004-of-00005.safetensors", "model.layers.24.block_sparse_moe.experts.3.w1.weight": "model-00004-of-00005.safetensors", "model.layers.24.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00005.safetensors", "model.layers.24.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00005.safetensors", "model.layers.24.block_sparse_moe.experts.2.w2.weight": "model-00004-of-00005.safetensors", "model.layers.24.block_sparse_moe.experts.3.w2.weight": "model-00004-of-00005.safetensors", "model.layers.25.input_layernorm.weight": "model-00004-of-00005.safetensors", "model.layers.25.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", "model.layers.25.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", "model.layers.25.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", "model.layers.25.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", "model.layers.25.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", "model.layers.25.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00005.safetensors", "model.layers.25.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00005.safetensors", "model.layers.25.block_sparse_moe.experts.2.w3.weight": "model-00004-of-00005.safetensors", "model.layers.25.block_sparse_moe.experts.3.w3.weight": "model-00004-of-00005.safetensors", "model.layers.25.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00005.safetensors", "model.layers.25.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00005.safetensors", "model.layers.25.block_sparse_moe.experts.2.w1.weight": "model-00004-of-00005.safetensors", "model.layers.25.block_sparse_moe.experts.3.w1.weight": "model-00004-of-00005.safetensors", "model.layers.25.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00005.safetensors", "model.layers.25.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00005.safetensors", "model.layers.25.block_sparse_moe.experts.2.w2.weight": "model-00004-of-00005.safetensors", "model.layers.25.block_sparse_moe.experts.3.w2.weight": "model-00004-of-00005.safetensors", "model.layers.26.input_layernorm.weight": "model-00004-of-00005.safetensors", "model.layers.26.self_attn.q_proj.weight": "model-00004-of-00005.safetensors", "model.layers.26.self_attn.k_proj.weight": "model-00004-of-00005.safetensors", "model.layers.26.self_attn.v_proj.weight": "model-00004-of-00005.safetensors", "model.layers.26.self_attn.o_proj.weight": "model-00004-of-00005.safetensors", "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00005.safetensors", "model.layers.26.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00005.safetensors", "model.layers.26.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00005.safetensors", "model.layers.26.block_sparse_moe.experts.2.w3.weight": "model-00004-of-00005.safetensors", "model.layers.26.block_sparse_moe.experts.3.w3.weight": "model-00005-of-00005.safetensors", "model.layers.26.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00005.safetensors", "model.layers.26.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00005.safetensors", "model.layers.26.block_sparse_moe.experts.2.w1.weight": "model-00005-of-00005.safetensors", "model.layers.26.block_sparse_moe.experts.3.w1.weight": "model-00005-of-00005.safetensors", "model.layers.26.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00005.safetensors", "model.layers.26.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00005.safetensors", "model.layers.26.block_sparse_moe.experts.2.w2.weight": "model-00005-of-00005.safetensors", "model.layers.26.block_sparse_moe.experts.3.w2.weight": "model-00005-of-00005.safetensors", "model.layers.27.input_layernorm.weight": "model-00005-of-00005.safetensors", "model.layers.27.self_attn.q_proj.weight": "model-00005-of-00005.safetensors", "model.layers.27.self_attn.k_proj.weight": "model-00005-of-00005.safetensors", "model.layers.27.self_attn.v_proj.weight": "model-00005-of-00005.safetensors", "model.layers.27.self_attn.o_proj.weight": "model-00005-of-00005.safetensors", "model.layers.27.post_attention_layernorm.weight": "model-00005-of-00005.safetensors", "model.layers.27.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00005.safetensors", "model.layers.27.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00005.safetensors", "model.layers.27.block_sparse_moe.experts.2.w3.weight": "model-00005-of-00005.safetensors", "model.layers.27.block_sparse_moe.experts.3.w3.weight": "model-00005-of-00005.safetensors", "model.layers.27.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00005.safetensors", "model.layers.27.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00005.safetensors", "model.layers.27.block_sparse_moe.experts.2.w1.weight": "model-00005-of-00005.safetensors", "model.layers.27.block_sparse_moe.experts.3.w1.weight": "model-00005-of-00005.safetensors", "model.layers.27.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00005.safetensors", "model.layers.27.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00005.safetensors", "model.layers.27.block_sparse_moe.experts.2.w2.weight": "model-00005-of-00005.safetensors", "model.layers.27.block_sparse_moe.experts.3.w2.weight": "model-00005-of-00005.safetensors", "model.layers.28.input_layernorm.weight": "model-00005-of-00005.safetensors", "model.layers.28.self_attn.q_proj.weight": "model-00005-of-00005.safetensors", "model.layers.28.self_attn.k_proj.weight": "model-00005-of-00005.safetensors", "model.layers.28.self_attn.v_proj.weight": "model-00005-of-00005.safetensors", "model.layers.28.self_attn.o_proj.weight": "model-00005-of-00005.safetensors", "model.layers.28.post_attention_layernorm.weight": "model-00005-of-00005.safetensors", "model.layers.28.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00005.safetensors", "model.layers.28.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00005.safetensors", "model.layers.28.block_sparse_moe.experts.2.w3.weight": "model-00005-of-00005.safetensors", "model.layers.28.block_sparse_moe.experts.3.w3.weight": "model-00005-of-00005.safetensors", "model.layers.28.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00005.safetensors", "model.layers.28.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00005.safetensors", "model.layers.28.block_sparse_moe.experts.2.w1.weight": "model-00005-of-00005.safetensors", "model.layers.28.block_sparse_moe.experts.3.w1.weight": "model-00005-of-00005.safetensors", "model.layers.28.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00005.safetensors", "model.layers.28.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00005.safetensors", "model.layers.28.block_sparse_moe.experts.2.w2.weight": "model-00005-of-00005.safetensors", "model.layers.28.block_sparse_moe.experts.3.w2.weight": "model-00005-of-00005.safetensors", "model.layers.29.input_layernorm.weight": "model-00005-of-00005.safetensors", "model.layers.29.self_attn.q_proj.weight": "model-00005-of-00005.safetensors", "model.layers.29.self_attn.k_proj.weight": "model-00005-of-00005.safetensors", "model.layers.29.self_attn.v_proj.weight": "model-00005-of-00005.safetensors", "model.layers.29.self_attn.o_proj.weight": "model-00005-of-00005.safetensors", "model.layers.29.post_attention_layernorm.weight": "model-00005-of-00005.safetensors", "model.layers.29.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00005.safetensors", "model.layers.29.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00005.safetensors", "model.layers.29.block_sparse_moe.experts.2.w3.weight": "model-00005-of-00005.safetensors", "model.layers.29.block_sparse_moe.experts.3.w3.weight": "model-00005-of-00005.safetensors", "model.layers.29.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00005.safetensors", "model.layers.29.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00005.safetensors", "model.layers.29.block_sparse_moe.experts.2.w1.weight": "model-00005-of-00005.safetensors", "model.layers.29.block_sparse_moe.experts.3.w1.weight": "model-00005-of-00005.safetensors", "model.layers.29.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00005.safetensors", "model.layers.29.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00005.safetensors", "model.layers.29.block_sparse_moe.experts.2.w2.weight": "model-00005-of-00005.safetensors", "model.layers.29.block_sparse_moe.experts.3.w2.weight": "model-00005-of-00005.safetensors", "model.layers.30.input_layernorm.weight": "model-00005-of-00005.safetensors", "model.layers.30.self_attn.q_proj.weight": "model-00005-of-00005.safetensors", "model.layers.30.self_attn.k_proj.weight": "model-00005-of-00005.safetensors", "model.layers.30.self_attn.v_proj.weight": "model-00005-of-00005.safetensors", "model.layers.30.self_attn.o_proj.weight": "model-00005-of-00005.safetensors", "model.layers.30.post_attention_layernorm.weight": "model-00005-of-00005.safetensors", "model.layers.30.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00005.safetensors", "model.layers.30.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00005.safetensors", "model.layers.30.block_sparse_moe.experts.2.w3.weight": "model-00005-of-00005.safetensors", "model.layers.30.block_sparse_moe.experts.3.w3.weight": "model-00005-of-00005.safetensors", "model.layers.30.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00005.safetensors", "model.layers.30.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00005.safetensors", "model.layers.30.block_sparse_moe.experts.2.w1.weight": "model-00005-of-00005.safetensors", "model.layers.30.block_sparse_moe.experts.3.w1.weight": "model-00005-of-00005.safetensors", "model.layers.30.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00005.safetensors", "model.layers.30.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00005.safetensors", "model.layers.30.block_sparse_moe.experts.2.w2.weight": "model-00005-of-00005.safetensors", "model.layers.30.block_sparse_moe.experts.3.w2.weight": "model-00005-of-00005.safetensors", "model.layers.31.input_layernorm.weight": "model-00005-of-00005.safetensors", "model.layers.31.self_attn.q_proj.weight": "model-00005-of-00005.safetensors", "model.layers.31.self_attn.k_proj.weight": "model-00005-of-00005.safetensors", "model.layers.31.self_attn.v_proj.weight": "model-00005-of-00005.safetensors", "model.layers.31.self_attn.o_proj.weight": "model-00005-of-00005.safetensors", "model.layers.31.post_attention_layernorm.weight": "model-00005-of-00005.safetensors", "model.layers.31.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00005.safetensors", "model.layers.31.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00005.safetensors", "model.layers.31.block_sparse_moe.experts.2.w3.weight": "model-00005-of-00005.safetensors", "model.layers.31.block_sparse_moe.experts.3.w3.weight": "model-00005-of-00005.safetensors", "model.layers.31.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00005.safetensors", "model.layers.31.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00005.safetensors", "model.layers.31.block_sparse_moe.experts.2.w1.weight": "model-00005-of-00005.safetensors", "model.layers.31.block_sparse_moe.experts.3.w1.weight": "model-00005-of-00005.safetensors", "model.layers.31.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00005.safetensors", "model.layers.31.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00005.safetensors", "model.layers.31.block_sparse_moe.experts.2.w2.weight": "model-00005-of-00005.safetensors", "model.layers.31.block_sparse_moe.experts.3.w2.weight": "model-00005-of-00005.safetensors", "model.layers.0.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.1.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.2.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.3.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.4.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.5.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.6.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.7.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.8.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.9.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.10.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.11.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.12.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.13.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.14.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.15.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.16.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.17.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.18.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.19.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.20.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.21.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.22.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.23.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.24.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.25.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.26.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.27.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.28.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.29.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.30.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors", "model.layers.31.block_sparse_moe.gate.weight": "model-00005-of-00005.safetensors"}}

output-00001-of-00002.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3b150bce4ce6f85a5467bf3783ef9da592e80803dc235f93674cc19b12b77612
-size 8564246576

output-00002-of-00002.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c2f6eabc44415a7af311664c8c1cbac348782e1b773abcfbbe564c07ed3b9c3f
-size 1386303216

special_tokens_map.json

@@ -1,24 +0,0 @@
-{
-  "bos_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": "<s>",
-  "unk_token": {
-    "content": "<unk>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
-}

tokenizer.model

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
-size 493443

tokenizer_config.json

@@ -1,42 +0,0 @@
-{
-  "add_bos_token": true,
-  "add_eos_token": false,
-  "added_tokens_decoder": {
-    "0": {
-      "content": "<unk>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "1": {
-      "content": "<s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "2": {
-      "content": "</s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    }
-  },
-  "additional_special_tokens": [],
-  "bos_token": "<s>",
-  "clean_up_tokenization_spaces": false,
-  "eos_token": "</s>",
-  "legacy": true,
-  "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "<s>",
-  "sp_model_kwargs": {},
-  "spaces_between_special_tokens": false,
-  "tokenizer_class": "LlamaTokenizer",
-  "unk_token": "<unk>",
-  "use_default_system_prompt": false
-}