Training in progress, step 1000

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "./Bitnet-Mistral.0.2-v2-2048-6/final_model/",
+  "_name_or_path": "liminerity/Bitnet-Mistral.0.2-330m-v0.2-grokfast-v2.9",
   "architectures": [
     "MistralForCausalLM"
   ],
@@ -9,12 +9,12 @@
   "hidden_act": "silu",
   "hidden_size": 1024,
   "initializer_range": 0.02,
-  "intermediate_size": 512,
+  "intermediate_size": 2048,
   "max_position_embeddings": 8192,
   "model_type": "mistral",
-  "num_attention_heads": 8,
-  "num_hidden_layers": 36,
-  "num_key_value_heads": 8,
+  "num_attention_heads": 16,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 16,
   "rms_norm_eps": 1e-05,
   "rope_theta": 1000000.0,
   "sliding_window": 4096,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:40ee020031af1df916804df992dcb66d8a63d4c25e7cde2f912f98f5ec8f6895
-size 1092800768
+oid sha256:bc7fbe560a406a64f86c2ed447d14243f3031d84b462981d2500137398d314a0
+size 1604486104

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9ed29ea17c35b49cfca87433a00a019d89fc7fe737a713637dd67c71c965ff52
-size 5240
+oid sha256:995e15dc035f07acd643783ea09c5b4093b5936852fa7cc04d5ebf00db47e564
+size 5176