Upload folder using huggingface_hub

config.json

@@ -1,6 +1,7 @@
 {
   "attention_bias": false,
   "attention_dropout": 0.0,
+  "attention_value_scale": 0.707,
   "bos_token_id": null,
   "eos_token_id": 151645,
   "head_dim": 192,
@@ -47,11 +48,10 @@
     }
   },
   "routed_scaling_factor": 1.0,
-  "router_jitter_noise": 0.0,
   "sliding_window": 128,
   "tie_word_embeddings": false,
   "topk_group": 1,
-  "transformers_version": "5.6.0.dev0",
+  "transformers_version": "5.7.0.dev0",
   "use_cache": true,
   "v_head_dim": 128,
   "vocab_size": 151669

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:42af2467e4bfa98049267514a080ad74861e15b08495967418358583199cb31b
-size 1999907056
+oid sha256:1b3f5a618c41b9ba418d56f7d2532f0577aecdb30e8694c426a399a4959f1b6c
+size 1999644448

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:793de365a5fe73df01793e6467be00fae4edaef269b05ae94c5090b1448fdc0a
+oid sha256:41d75a901bebd8146f8c3682300eeb3768458b6ace41e4738a96a0e2c644a0d4
 size 1998137832

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2fb38def785a4b9001af6ef78281a8479e5ef85952f2eadedfe5a3bedb6b118a
-size 684165360
+oid sha256:532bebad37176d2276e8d860e4be8d0b040d6caac4aba9a82e24cbb923a19f15
+size 684427968

model.safetensors.index.json

@@ -18,8 +18,6 @@
     "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.1.mlp.gate.weight": "model-00001-of-00003.safetensors",
-    "model.layers.1.mlp.gate.e_score_correction_bias": "model-00001-of-00003.safetensors",
     "model.layers.1.mlp.experts.0.gate_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.1.mlp.experts.0.up_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.1.mlp.experts.1.gate_proj.weight": "model-00001-of-00003.safetensors",
@@ -212,6 +210,8 @@
     "model.layers.1.mlp.experts.61.down_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.1.mlp.experts.62.down_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.1.mlp.experts.63.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate.e_score_correction_bias": "model-00001-of-00003.safetensors",
     "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
     "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
     "model.layers.2.self_attn.attention_sink_bias": "model-00001-of-00003.safetensors",
@@ -219,8 +219,6 @@
     "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.2.mlp.gate.weight": "model-00001-of-00003.safetensors",
-    "model.layers.2.mlp.gate.e_score_correction_bias": "model-00001-of-00003.safetensors",
     "model.layers.2.mlp.experts.0.gate_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.2.mlp.experts.0.up_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.2.mlp.experts.1.gate_proj.weight": "model-00001-of-00003.safetensors",
@@ -413,6 +411,8 @@
     "model.layers.2.mlp.experts.61.down_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.2.mlp.experts.62.down_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.2.mlp.experts.63.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.mlp.gate.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.mlp.gate.e_score_correction_bias": "model-00002-of-00003.safetensors",
     "model.layers.2.input_layernorm.weight": "model-00002-of-00003.safetensors",
     "model.layers.2.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
     "model.layers.3.self_attn.attention_sink_bias": "model-00002-of-00003.safetensors",
@@ -420,8 +420,6 @@
     "model.layers.3.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.3.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.3.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.3.mlp.gate.weight": "model-00002-of-00003.safetensors",
-    "model.layers.3.mlp.gate.e_score_correction_bias": "model-00002-of-00003.safetensors",
     "model.layers.3.mlp.experts.0.gate_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.3.mlp.experts.0.up_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.3.mlp.experts.1.gate_proj.weight": "model-00002-of-00003.safetensors",
@@ -614,6 +612,8 @@
     "model.layers.3.mlp.experts.61.down_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.3.mlp.experts.62.down_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.3.mlp.experts.63.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.3.mlp.gate.weight": "model-00002-of-00003.safetensors",
+    "model.layers.3.mlp.gate.e_score_correction_bias": "model-00002-of-00003.safetensors",
     "model.layers.3.input_layernorm.weight": "model-00002-of-00003.safetensors",
     "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
     "model.layers.4.self_attn.attention_sink_bias": "model-00002-of-00003.safetensors",
@@ -621,8 +621,6 @@
     "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.4.mlp.gate.weight": "model-00002-of-00003.safetensors",
-    "model.layers.4.mlp.gate.e_score_correction_bias": "model-00002-of-00003.safetensors",
     "model.layers.4.mlp.experts.0.gate_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.4.mlp.experts.0.up_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.4.mlp.experts.1.gate_proj.weight": "model-00002-of-00003.safetensors",
@@ -815,6 +813,8 @@
     "model.layers.4.mlp.experts.61.down_proj.weight": "model-00003-of-00003.safetensors",
     "model.layers.4.mlp.experts.62.down_proj.weight": "model-00003-of-00003.safetensors",
     "model.layers.4.mlp.experts.63.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.mlp.gate.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.mlp.gate.e_score_correction_bias": "model-00003-of-00003.safetensors",
     "model.layers.4.input_layernorm.weight": "model-00003-of-00003.safetensors",
     "model.layers.4.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
     "model.norm.weight": "model-00003-of-00003.safetensors",