Trained with Unsloth

config.json

@@ -13,17 +13,14 @@
   "hidden_size": 3072,
   "initializer_range": 0.02,
   "intermediate_size": 24576,
-  "max_position_embeddings": 16384,
+  "max_position_embeddings": 8192,
   "model_type": "gemma",
   "num_attention_heads": 16,
   "num_hidden_layers": 28,
   "num_key_value_heads": 16,
   "pad_token_id": 0,
   "rms_norm_eps": 1e-06,
-  "rope_scaling": {
-    "factor": 2.0,
-    "type": "linear"
-  },
+  "rope_scaling": null,
   "rope_theta": 10000.0,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.40.1",

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d5df9566a7b47803175f8632bcd14a607f635f73062c4d9dff1eed8429bac9e6
+oid sha256:31bae2d4da4e18ac0ee57237851f44538fcf195ffefc6c425a7de386b7038411
 size 4995496656

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:08603609ec765e38788126340fae2c4c414b956c2e878ef8ee38a6201ef0d1d3
+oid sha256:3e9ff56600eef3bf4b57d7b64386551f6ee45fe44980ec7268b3082443be97e9
 size 4982953168

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2881faa29f5c5fde536e729f19aabdaf7e69f7a1042c4a2c3da4dacc19aaad7e
+oid sha256:36c69845323020b64a6344bc0f819ef985b3aa7e9cac87541288bf268079f1d3
 size 4982953200

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:809dcdd309927e6b6ad160ed1afe14dc6112ffc3d07de06e010da2ab11e6473f
+oid sha256:b6a5f866fc3a5469bdcb5d9488ae08ae9be0cc8389e140a204c38dcc1c122fad
 size 2113988336