update

.gitattributes

@@ -41,3 +41,10 @@ pytorch_model-00002-of-00007.bin filter=lfs diff=lfs merge=lfs -text
 pytorch_model-00003-of-00007.bin filter=lfs diff=lfs merge=lfs -text
 pytorch_model-00004-of-00007.bin filter=lfs diff=lfs merge=lfs -text
 tokenizer.model filter=lfs diff=lfs merge=lfs -text
+model-00004-of-00007.safetensors filter=lfs diff=lfs merge=lfs -text
+model-00005-of-00007.safetensors filter=lfs diff=lfs merge=lfs -text
+model-00006-of-00007.safetensors filter=lfs diff=lfs merge=lfs -text
+model-00007-of-00007.safetensors filter=lfs diff=lfs merge=lfs -text
+model-00001-of-00007.safetensors filter=lfs diff=lfs merge=lfs -text
+model-00002-of-00007.safetensors filter=lfs diff=lfs merge=lfs -text
+model-00003-of-00007.safetensors filter=lfs diff=lfs merge=lfs -text

model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1052386eac358a18add3d0f92521c85ab338979da8eeb08a6499555b857f80d
+size 1827774160

model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f88d7f69a3940711b87308eb20dd0544df22aff81134fa0d5160d3c859f5d321
+size 1968291888

model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:447d41b7c5e7b2558905c98733469aa9e132540c91e13c4cdd7bfc58b60cc650
+size 1927406936

model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:386305288e2bcc4a26fcffef433c94592d155bf36e365f054f63d3e6b55a1429
+size 1815217640

model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c304e2821774d649997d1f5ae5fa12afd4d1dc5fea2bf6e8b829405b81fa7f2
+size 1968291920

model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef45bbd33d6d540d3d3523bb169b52b91b9e265176ab7efc6838233bc3b7e95d
+size 1927406960

model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6eb545c6aa8a839c56a78b80331fcb4fa403341925f79fb6a761e2e847bf1e40
+size 1052805400

model.safetensors.index.json

@@ -0,0 +1,207 @@
+{
+    "metadata": {
+        "total_size": 12487168064
+    },
+    "weight_map": {
+        "transformer.embedding.word_embeddings.weight": "model-00001-of-00007.safetensors",
+        "transformer.encoder.final_layernorm.weight": "model-00007-of-00007.safetensors",
+        "transformer.encoder.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.0.mlp.dense_4h_to_h.weight": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.0.mlp.dense_h_to_4h.weight": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.0.self_attention.dense.weight": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.0.self_attention.query_key_value.bias": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.0.self_attention.query_key_value.weight": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.1.mlp.dense_4h_to_h.weight": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.1.mlp.dense_h_to_4h.weight": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.1.self_attention.dense.weight": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.1.self_attention.query_key_value.bias": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.1.self_attention.query_key_value.weight": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.10.mlp.dense_4h_to_h.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.10.mlp.dense_h_to_4h.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.10.self_attention.dense.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.10.self_attention.query_key_value.bias": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.10.self_attention.query_key_value.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.11.mlp.dense_4h_to_h.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.11.mlp.dense_h_to_4h.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.11.self_attention.dense.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.11.self_attention.query_key_value.bias": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.11.self_attention.query_key_value.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.12.mlp.dense_4h_to_h.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.12.mlp.dense_h_to_4h.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.12.self_attention.dense.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.12.self_attention.query_key_value.bias": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.12.self_attention.query_key_value.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.13.input_layernorm.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.13.mlp.dense_4h_to_h.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.13.mlp.dense_h_to_4h.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.13.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.13.self_attention.dense.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.13.self_attention.query_key_value.bias": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.13.self_attention.query_key_value.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.14.mlp.dense_4h_to_h.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.14.mlp.dense_h_to_4h.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.14.self_attention.dense.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.14.self_attention.query_key_value.bias": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.14.self_attention.query_key_value.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.15.mlp.dense_4h_to_h.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.15.mlp.dense_h_to_4h.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.15.self_attention.dense.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.15.self_attention.query_key_value.bias": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.15.self_attention.query_key_value.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.16.mlp.dense_4h_to_h.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.16.mlp.dense_h_to_4h.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.16.self_attention.dense.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.16.self_attention.query_key_value.bias": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.16.self_attention.query_key_value.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.17.mlp.dense_4h_to_h.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.17.mlp.dense_h_to_4h.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.17.self_attention.dense.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.17.self_attention.query_key_value.bias": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.17.self_attention.query_key_value.weight": "model-00004-of-00007.safetensors",
+        "transformer.encoder.layers.18.input_layernorm.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.18.mlp.dense_4h_to_h.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.18.mlp.dense_h_to_4h.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.18.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.18.self_attention.dense.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.18.self_attention.query_key_value.bias": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.18.self_attention.query_key_value.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.19.input_layernorm.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.19.mlp.dense_4h_to_h.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.19.mlp.dense_h_to_4h.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.19.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.19.self_attention.dense.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.19.self_attention.query_key_value.bias": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.19.self_attention.query_key_value.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.2.mlp.dense_4h_to_h.weight": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.2.mlp.dense_h_to_4h.weight": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.2.self_attention.dense.weight": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.2.self_attention.query_key_value.bias": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.2.self_attention.query_key_value.weight": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.20.mlp.dense_4h_to_h.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.20.mlp.dense_h_to_4h.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.20.self_attention.dense.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.20.self_attention.query_key_value.bias": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.20.self_attention.query_key_value.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.21.mlp.dense_4h_to_h.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.21.mlp.dense_h_to_4h.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.21.self_attention.dense.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.21.self_attention.query_key_value.bias": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.21.self_attention.query_key_value.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors",
+        "transformer.encoder.layers.22.mlp.dense_4h_to_h.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.22.mlp.dense_h_to_4h.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.22.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.22.self_attention.dense.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.22.self_attention.query_key_value.bias": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.22.self_attention.query_key_value.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.23.input_layernorm.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.23.mlp.dense_4h_to_h.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.23.mlp.dense_h_to_4h.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.23.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.23.self_attention.dense.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.23.self_attention.query_key_value.bias": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.23.self_attention.query_key_value.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.24.input_layernorm.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.24.mlp.dense_4h_to_h.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.24.mlp.dense_h_to_4h.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.24.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.24.self_attention.dense.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.24.self_attention.query_key_value.bias": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.24.self_attention.query_key_value.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.25.mlp.dense_4h_to_h.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.25.mlp.dense_h_to_4h.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.25.self_attention.dense.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.25.self_attention.query_key_value.bias": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.25.self_attention.query_key_value.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.26.mlp.dense_4h_to_h.weight": "model-00007-of-00007.safetensors",
+        "transformer.encoder.layers.26.mlp.dense_h_to_4h.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.26.self_attention.dense.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.26.self_attention.query_key_value.bias": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.26.self_attention.query_key_value.weight": "model-00006-of-00007.safetensors",
+        "transformer.encoder.layers.27.input_layernorm.weight": "model-00007-of-00007.safetensors",
+        "transformer.encoder.layers.27.mlp.dense_4h_to_h.weight": "model-00007-of-00007.safetensors",
+        "transformer.encoder.layers.27.mlp.dense_h_to_4h.weight": "model-00007-of-00007.safetensors",
+        "transformer.encoder.layers.27.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
+        "transformer.encoder.layers.27.self_attention.dense.weight": "model-00007-of-00007.safetensors",
+        "transformer.encoder.layers.27.self_attention.query_key_value.bias": "model-00007-of-00007.safetensors",
+        "transformer.encoder.layers.27.self_attention.query_key_value.weight": "model-00007-of-00007.safetensors",
+        "transformer.encoder.layers.3.input_layernorm.weight": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.3.mlp.dense_4h_to_h.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.3.mlp.dense_h_to_4h.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.3.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.3.self_attention.dense.weight": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.3.self_attention.query_key_value.bias": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.3.self_attention.query_key_value.weight": "model-00001-of-00007.safetensors",
+        "transformer.encoder.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.4.mlp.dense_4h_to_h.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.4.mlp.dense_h_to_4h.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.4.self_attention.dense.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.4.self_attention.query_key_value.bias": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.4.self_attention.query_key_value.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.5.mlp.dense_4h_to_h.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.5.mlp.dense_h_to_4h.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.5.self_attention.dense.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.5.self_attention.query_key_value.bias": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.5.self_attention.query_key_value.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.6.mlp.dense_4h_to_h.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.6.mlp.dense_h_to_4h.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.6.self_attention.dense.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.6.self_attention.query_key_value.bias": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.6.self_attention.query_key_value.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.7.mlp.dense_4h_to_h.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.7.mlp.dense_h_to_4h.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.7.self_attention.dense.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.7.self_attention.query_key_value.bias": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.7.self_attention.query_key_value.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.8.input_layernorm.weight": "model-00002-of-00007.safetensors",
+        "transformer.encoder.layers.8.mlp.dense_4h_to_h.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.8.mlp.dense_h_to_4h.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.8.self_attention.dense.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.8.self_attention.query_key_value.bias": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.8.self_attention.query_key_value.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.9.mlp.dense_4h_to_h.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.9.mlp.dense_h_to_4h.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.9.self_attention.dense.weight": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.9.self_attention.query_key_value.bias": "model-00003-of-00007.safetensors",
+        "transformer.encoder.layers.9.self_attention.query_key_value.weight": "model-00003-of-00007.safetensors",
+        "transformer.output_layer.weight": "model-00007-of-00007.safetensors",
+        "transformer.rotary_pos_emb.inv_freq": "model-00001-of-00007.safetensors"
+    }
+}

modeling_chatglm.py

@@ -634,7 +634,8 @@ class GLMTransformer(torch.nn.Module):
                     attention_mask,
                     rotary_pos_emb,
                     kv_caches[index],
-                    use_cache
+                    use_cache,
+                    use_reentrant=False
                 )
             else:
                 layer_ret = layer(
@@ -697,9 +698,9 @@ class ChatGLMPreTrainedModel(PreTrainedModel):
         position_ids = torch.arange(seq_length, dtype=torch.long, device=device).unsqueeze(0).repeat(batch_size, 1)
         return position_ids
 
-    def _set_gradient_checkpointing(self, module, value=False):
-        if isinstance(module, GLMTransformer):
-            module.gradient_checkpointing = value
+    def gradient_checkpointing_enable(self, gradient_checkpointing_kwargs=None):
+        if not self.supports_gradient_checkpointing:
+            raise ValueError(f"{self.__class__.__name__} does not support gradient checkpointing.")
 
 
 class Embedding(torch.nn.Module):
@@ -768,6 +769,9 @@ class ChatGLMModel(ChatGLMPreTrainedModel):
     def get_input_embeddings(self):
         return self.embedding.word_embeddings
 
+    def set_input_embeddings(self, value):
+        self.embedding.word_embeddings = value
+
     def get_prompt(self, batch_size, device, dtype=torch.half):
         prefix_tokens = self.prefix_tokens.unsqueeze(0).expand(batch_size, -1).to(device)
         past_key_values = self.prefix_encoder(prefix_tokens).type(dtype)

tokenization_chatglm.py

@@ -8,6 +8,9 @@ from transformers.utils import logging, PaddingStrategy
 from transformers.tokenization_utils_base import EncodedInput, BatchEncoding
 
 
+logger = logging.get_logger(__name__)
+
+
 class SPTokenizer:
     def __init__(self, model_path: str):
         # reload tokenizer
@@ -29,7 +32,7 @@ class SPTokenizer:
             self.special_tokens[token] = self.n_words
             self.index_special_tokens[self.n_words] = token
             self.n_words += 1
-        self.role_special_token_expression = "|".join([re.escape(token) for token in role_special_tokens])
+        self.role_special_token_expression = "|".join([re.escape(token) for token in special_tokens]) # for apply_chat_template
 
     def tokenize(self, s: str, encode_special_tokens=False):
         if encode_special_tokens:
@@ -89,25 +92,34 @@ class SPTokenizer:
 
 
 class ChatGLMTokenizer(PreTrainedTokenizer):
-    vocab_files_names = {"vocab_file": "tokenizer.model"}
 
+    vocab_files_names = {"vocab_file": "tokenizer.model"}
     model_input_names = ["input_ids", "attention_mask", "position_ids"]
 
-    def __init__(self, vocab_file, padding_side="left", clean_up_tokenization_spaces=False, encode_special_tokens=False,
-                 **kwargs):
+    def __init__(
+        self,
+        vocab_file,
+        padding_side="left",
+        clean_up_tokenization_spaces=False,
+        encode_special_tokens=False,
+        **kwargs
+    ):
         self.name = "GLMTokenizer"
-
         self.vocab_file = vocab_file
         self.tokenizer = SPTokenizer(vocab_file)
         self.special_tokens = {
             "<bos>": self.tokenizer.bos_id,
             "<eos>": self.tokenizer.eos_id,
+            "<unk>": self.tokenizer.pad_id,
             "<pad>": self.tokenizer.pad_id
         }
         self.encode_special_tokens = encode_special_tokens
-        super().__init__(padding_side=padding_side, clean_up_tokenization_spaces=clean_up_tokenization_spaces,
-                         encode_special_tokens=encode_special_tokens,
-                         **kwargs)
+
+        super().__init__(
+            padding_side=padding_side,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            **kwargs
+        )
 
     def get_command(self, token):
         if token in self.special_tokens:
@@ -117,24 +129,40 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
 
     @property
     def unk_token(self) -> str:
-        return "<unk>"
+        return self.tokenizer.sp_model.IdToPiece(self.get_command("<unk>"))
 
     @property
     def pad_token(self) -> str:
-        return "<unk>"
+        return self.tokenizer.sp_model.IdToPiece(self.get_command("<pad>"))
 
     @property
-    def pad_token_id(self):
-        return self.get_command("<pad>")
+    def eos_token(self) -> str:
+        return self.tokenizer.sp_model.IdToPiece(self.get_command("<eos>"))
 
     @property
-    def eos_token(self) -> str:
-        return "</s>"
+    def unk_token_id(self) -> int:
+        return self.get_command("<unk>")
+
+    @property
+    def pad_token_id(self) -> int:
+        return self.get_command("<pad>")
 
     @property
     def eos_token_id(self):
         return self.get_command("<eos>")
 
+    @unk_token.setter
+    def unk_token(self, value):
+        logger.warning("Setting unk_token is not supported, use the default one.")
+
+    @pad_token.setter
+    def pad_token(self, value):
+        logger.warning("Setting pad_token is not supported, use the default one.")
+
+    @eos_token.setter
+    def eos_token(self, value):
+        logger.warning("Setting eos_token is not supported, use the default one.")
+
     @property
     def vocab_size(self):
         return self.tokenizer.n_words
@@ -212,7 +240,7 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
         return self.batch_encode_plus([input_ids], return_tensors="pt", is_split_into_words=True)
 
     def build_inputs_with_special_tokens(
-            self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
+        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
     ) -> List[int]:
         """
         Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
@@ -237,12 +265,12 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
         return token_ids_0
 
     def _pad(
-            self,
-            encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding],
-            max_length: Optional[int] = None,
-            padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
-            pad_to_multiple_of: Optional[int] = None,
-            return_attention_mask: Optional[bool] = None,
+        self,
+        encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding],
+        max_length: Optional[int] = None,
+        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
+        pad_to_multiple_of: Optional[int] = None,
+        return_attention_mask: Optional[bool] = None,
     ) -> dict:
         """
         Pad encoded inputs (on left/right and up to predefined length or max length in the batch)

tokenizer_config.json

@@ -31,6 +31,14 @@
       "rstrip": false,
       "single_word": false,
       "special": false
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
     }
   },
   "auto_map": {