Upload 7 files

config.json

@@ -45,7 +45,7 @@
   "audio_token_id": 258881,
   "boa_token_id": 256000,
   "boi_token_id": 255999,
-  "dtype": "bfloat16",
+  "dtype": "float32",
   "eoa_token_id": 258883,
   "eoa_token_index": 258883,
   "eoi_token_id": 258882,
@@ -61,13 +61,13 @@
     "attention_dropout": 0.0,
     "attention_k_eq_v": false,
     "bos_token_id": 2,
-    "dtype": "bfloat16",
+    "dtype": "float32",
     "enable_moe_block": false,
     "eos_token_id": 1,
     "expert_intermediate_size": null,
     "final_logit_softcapping": 30.0,
-    "global_head_dim": 8,
-    "head_dim": 8,
+    "global_head_dim": 4,
+    "head_dim": 4,
     "hidden_activation": "gelu_pytorch_tanh",
     "hidden_size": 8,
     "hidden_size_per_layer_input": 1,
@@ -80,12 +80,13 @@
     ],
     "max_position_embeddings": 131072,
     "model_type": "gemma4_text",
+    "moe_intermediate_size": null,
     "num_attention_heads": 2,
     "num_experts": null,
     "num_global_key_value_heads": null,
     "num_hidden_layers": 3,
     "num_key_value_heads": 1,
-    "num_kv_shared_layers": 0,
+    "num_kv_shared_layers": 1,
     "pad_token_id": 0,
     "rms_norm_eps": 1e-06,
     "rope_parameters": {

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:29a6fd9be57d9af362845eb2504fc7420c168f3cb8ffd98777fcb5452d04843d
-size 6221872
+oid sha256:feaf3cfe37e063524c5a46b94a145b50aeb7ea07d6b983070dda965ed0248c86
+size 11997808