Upload folder using huggingface_hub

config.json

@@ -2,17 +2,29 @@
   "architectures": [
     "Gemma2Model"
   ],
-  "dim": 4096,
-  "ffn_dim_multiplier": null,
-  "max_batch_size": 32,
-  "max_seq_len": 1024,
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": 50.0,
+  "bos_token_id": 2,
+  "eos_token_id": 1,
+  "final_logit_softcapping": 30.0,
+  "head_dim": 256,
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 24576,
+  "max_position_embeddings": 8192,
   "model_type": "gemma2",
-  "multiple_of": 256,
-  "n_heads": 32,
-  "n_kv_heads": 8,
-  "n_layers": 32,
-  "norm_eps": 1e-05,
+  "num_attention_heads": 16,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 16,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 224,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
   "torch_dtype": "float32",
   "transformers_version": "4.42.3",
+  "use_cache": true,
   "vocab_size": 50257
 }

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3ed110786d0628e50b3bea55f5e86ba6914c3c3f77cd8c18e8dcaee5a76ee298
-size 4967392824
+oid sha256:a77ca0e8ea5b3455808c2042946fdbbe9338a267a3c0fc6b74df10e569eb6623
+size 4946115520

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8786e98875fb06808e3be9ed56b2f735ea9035733ca0b9526138ba865fc4cdbe
-size 2064960496
+oid sha256:e804047dad69e495540b3cb78a86936d5ca3323b212ae5d5c85db4480447bdac
+size 617558144

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 7032340480
+    "total_size": 5563662336
   },
   "weight_map": {
     "embeddings.weight": "model-00001-of-00002.safetensors",
@@ -73,41 +73,25 @@
     "layers.23.wq.weight": "model-00001-of-00002.safetensors",
     "layers.23.wv.weight": "model-00001-of-00002.safetensors",
     "layers.24.wk.weight": "model-00001-of-00002.safetensors",
-    "layers.24.wo.weight": "model-00002-of-00002.safetensors",
+    "layers.24.wo.weight": "model-00001-of-00002.safetensors",
     "layers.24.wq.weight": "model-00001-of-00002.safetensors",
     "layers.24.wv.weight": "model-00001-of-00002.safetensors",
-    "layers.25.wk.weight": "model-00002-of-00002.safetensors",
-    "layers.25.wo.weight": "model-00002-of-00002.safetensors",
-    "layers.25.wq.weight": "model-00002-of-00002.safetensors",
-    "layers.25.wv.weight": "model-00002-of-00002.safetensors",
-    "layers.26.wk.weight": "model-00002-of-00002.safetensors",
-    "layers.26.wo.weight": "model-00002-of-00002.safetensors",
-    "layers.26.wq.weight": "model-00002-of-00002.safetensors",
-    "layers.26.wv.weight": "model-00002-of-00002.safetensors",
-    "layers.27.wk.weight": "model-00002-of-00002.safetensors",
-    "layers.27.wo.weight": "model-00002-of-00002.safetensors",
-    "layers.27.wq.weight": "model-00002-of-00002.safetensors",
-    "layers.27.wv.weight": "model-00002-of-00002.safetensors",
-    "layers.28.wk.weight": "model-00002-of-00002.safetensors",
-    "layers.28.wo.weight": "model-00002-of-00002.safetensors",
-    "layers.28.wq.weight": "model-00002-of-00002.safetensors",
-    "layers.28.wv.weight": "model-00002-of-00002.safetensors",
-    "layers.29.wk.weight": "model-00002-of-00002.safetensors",
-    "layers.29.wo.weight": "model-00002-of-00002.safetensors",
-    "layers.29.wq.weight": "model-00002-of-00002.safetensors",
-    "layers.29.wv.weight": "model-00002-of-00002.safetensors",
+    "layers.25.wk.weight": "model-00001-of-00002.safetensors",
+    "layers.25.wo.weight": "model-00001-of-00002.safetensors",
+    "layers.25.wq.weight": "model-00001-of-00002.safetensors",
+    "layers.25.wv.weight": "model-00001-of-00002.safetensors",
+    "layers.26.wk.weight": "model-00001-of-00002.safetensors",
+    "layers.26.wo.weight": "model-00001-of-00002.safetensors",
+    "layers.26.wq.weight": "model-00001-of-00002.safetensors",
+    "layers.26.wv.weight": "model-00001-of-00002.safetensors",
+    "layers.27.wk.weight": "model-00001-of-00002.safetensors",
+    "layers.27.wo.weight": "model-00001-of-00002.safetensors",
+    "layers.27.wq.weight": "model-00001-of-00002.safetensors",
+    "layers.27.wv.weight": "model-00001-of-00002.safetensors",
     "layers.3.wk.weight": "model-00001-of-00002.safetensors",
     "layers.3.wo.weight": "model-00001-of-00002.safetensors",
     "layers.3.wq.weight": "model-00001-of-00002.safetensors",
     "layers.3.wv.weight": "model-00001-of-00002.safetensors",
-    "layers.30.wk.weight": "model-00002-of-00002.safetensors",
-    "layers.30.wo.weight": "model-00002-of-00002.safetensors",
-    "layers.30.wq.weight": "model-00002-of-00002.safetensors",
-    "layers.30.wv.weight": "model-00002-of-00002.safetensors",
-    "layers.31.wk.weight": "model-00002-of-00002.safetensors",
-    "layers.31.wo.weight": "model-00002-of-00002.safetensors",
-    "layers.31.wq.weight": "model-00002-of-00002.safetensors",
-    "layers.31.wv.weight": "model-00002-of-00002.safetensors",
     "layers.4.wk.weight": "model-00001-of-00002.safetensors",
     "layers.4.wo.weight": "model-00001-of-00002.safetensors",
     "layers.4.wq.weight": "model-00001-of-00002.safetensors",
@@ -133,8 +117,8 @@
     "layers.9.wq.weight": "model-00001-of-00002.safetensors",
     "layers.9.wv.weight": "model-00001-of-00002.safetensors",
     "lm_head.weight": "model-00002-of-00002.safetensors",
-    "ln_f.bias": "model-00002-of-00002.safetensors",
-    "ln_f.weight": "model-00002-of-00002.safetensors",
+    "ln_f.bias": "model-00001-of-00002.safetensors",
+    "ln_f.weight": "model-00001-of-00002.safetensors",
     "pos_embeddings.weight": "model-00001-of-00002.safetensors"
   }
 }