update tokenizer and model weights

added_tokens.json

@@ -0,0 +1,10 @@
+{
+  "<|endoftext|>": 96506,
+  "<|end|>": 96500,
+  "<|message|>": 96501,
+  "<|pad|>": 96505,
+  "<|start|>": 96499,
+  "<|tool_end|>": 96504,
+  "<|tool_excute|>": 96503,
+  "<|tool_start|>": 96502
+}

config.json

@@ -0,0 +1,28 @@
+{
+    "auto_map": {
+      "AutoConfig": "configuration_inflm.INFLMConfig",
+      "AutoModelForCausalLM": "modeling_inflm.INFLMForCausalLM"
+    },
+    "architectures": [
+      "INFLMForCausalLM"
+    ],
+    "bos_token_id": 1,
+    "eos_token_id": 2,
+    "hidden_act": "silu",
+    "hidden_size": 8192,
+    "initializer_range": 0.02,
+    "intermediate_size": 22016,
+    "max_position_embeddings": 4096,
+    "model_type": "inflm",
+    "num_attention_heads": 64,
+    "num_hidden_layers": 48,
+    "num_key_value_heads": 8,
+    "pretraining_tp": 1,
+    "layer_norm_eps": 1e-05,
+    "rope_theta": 500000,
+    "rope_scaling": null,
+    "tie_word_embeddings": false,
+    "torch_dtype": "bfloat16",
+    "use_cache": true,
+    "vocab_size": 96512
+}

configuration_inflm.py

@@ -0,0 +1,36 @@
+# coding=utf-8
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
+#
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
+# and OPT implementations in this library. It has been modified from its
+# original forms to accommodate minor architectural differences compared
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""INFLM model configuration."""
+
+from transformers.models.llama.configuration_llama import LlamaConfig
+
+
+class INFLMConfig(LlamaConfig):
+    model_type = "inflm"
+
+    def __init__(
+        self,
+        layer_norm_eps=1e-5,
+        **kwargs,
+    ):
+        self.layer_norm_eps = layer_norm_eps
+        super().__init__(
+            **kwargs,
+        )

generation_config.json

@@ -0,0 +1,9 @@
+{
+    "eos_token_id": 2,
+    "pad_token_id": 3,
+    "max_new_tokens": 2048,
+    "do_sample": true,
+    "top_k": 0,
+    "top_p": 0.8,
+    "transformers_version": "4.39.0"
+}

model-00001-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:161c857f1b2ab30371cdd41c9a682e8ce9027fa59c752bed06335f98ac459b0b
+size 3988850744

model-00002-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03b2bd2d36f9193c43df8fc5e420a89ac6b8c62f974939b8a973435bef7e81c4
+size 4152561376

model-00003-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:369e960a9db12fb81c9b5857a82eadc9f74a4df9c68ca067024eaa8c0c8c9cc2
+size 4152561376

model-00004-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e952e929eaae30fc484d858dcda84e83fa02ac1ff95b51627ba9a54defdf123
+size 4152561408

model-00005-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9be03db03185bda2724f04f9e18b22bd4981794e1be6d0dbf50f52bf9aecf0f
+size 4152561408

model-00006-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3146ff836155bc6c4b3448834dbbe2e61b7b326125f22adebc68ee25ed24af2b
+size 4152561408

model-00007-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:642420385e52d173cfc79f373412c1a514c752edc8deb102aa2b3dd67e471f26
+size 4152561408

model-00008-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:571775f85501368131427d0937923334c17e9d816ca98c74a1491d9c74189154
+size 4152561408

model-00009-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2dbc5475d252737bdfb2d097d38c901df07b3b6b551af3a47a02d4ff20be0d0e
+size 4152561408

model-00010-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28f7014dda7fc2ea4a93709396ac5c7ca67740d5073e787fa84f9ddf0f334513
+size 4152561408

model-00011-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d52d892f1c71b8a08cc1189f73df07bff1901bfdb499d3e38adb5921b332aeb3
+size 4152561408

model-00012-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec61eb90fddf06abd14c64290cbc891fb01af8fd84a05d5cc543f9f9a7c4b60e
+size 4152561408

model-00013-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ccf632af3c3d42c226927c77045f0df6e0c053a7a566658a4c22cc8a52d29df5
+size 4152561408

model-00014-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfa2a791184b253664128e5388bb06a043dcffd8691ad469508e019d321a08cc
+size 4152561408

model-00015-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ff5b32dbfd9ed096d5a8f38d908c6580efe697955050bbbe5c6353e1eca184a
+size 4152561408

model-00016-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc89e53b2c7279a2adb6029d3460e84dd551c0b3298a8b49d8bd3a3dd67ab681
+size 4152561408

model-00017-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51ae3aebeb804960180b40a2b878d9be56e60199f69f559460043ff50bac5aed
+size 3326249080

model.safetensors.index.json

@@ -0,0 +1,539 @@
+{
+  "metadata": {
+    "total_size": 69603459072
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00017.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00017.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00017.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00017.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00017.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00017.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00017.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00017.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00017.safetensors",
+    "model.layers.0.input_layernorm.bias": "model-00001-of-00017.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00017.safetensors",
+    "model.layers.0.post_attention_layernorm.bias": "model-00001-of-00017.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00017.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00017.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00017.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00017.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00017.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00017.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00002-of-00017.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00002-of-00017.safetensors",
+    "model.layers.1.input_layernorm.bias": "model-00002-of-00017.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00002-of-00017.safetensors",
+    "model.layers.1.post_attention_layernorm.bias": "model-00002-of-00017.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00002-of-00017.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00002-of-00017.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00002-of-00017.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00002-of-00017.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00002-of-00017.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00002-of-00017.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00002-of-00017.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00002-of-00017.safetensors",
+    "model.layers.2.input_layernorm.bias": "model-00002-of-00017.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00002-of-00017.safetensors",
+    "model.layers.2.post_attention_layernorm.bias": "model-00002-of-00017.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00002-of-00017.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00002-of-00017.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00002-of-00017.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00002-of-00017.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00017.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00017.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00017.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00017.safetensors",
+    "model.layers.3.input_layernorm.bias": "model-00002-of-00017.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00017.safetensors",
+    "model.layers.3.post_attention_layernorm.bias": "model-00002-of-00017.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00017.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00017.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00017.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00017.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00017.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00017.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00003-of-00017.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00003-of-00017.safetensors",
+    "model.layers.4.input_layernorm.bias": "model-00003-of-00017.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00003-of-00017.safetensors",
+    "model.layers.4.post_attention_layernorm.bias": "model-00003-of-00017.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00003-of-00017.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00003-of-00017.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00003-of-00017.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00003-of-00017.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00003-of-00017.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00003-of-00017.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00003-of-00017.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00003-of-00017.safetensors",
+    "model.layers.5.input_layernorm.bias": "model-00003-of-00017.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00003-of-00017.safetensors",
+    "model.layers.5.post_attention_layernorm.bias": "model-00003-of-00017.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00003-of-00017.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00003-of-00017.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00003-of-00017.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00003-of-00017.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00003-of-00017.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00003-of-00017.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00003-of-00017.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00003-of-00017.safetensors",
+    "model.layers.6.input_layernorm.bias": "model-00003-of-00017.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00003-of-00017.safetensors",
+    "model.layers.6.post_attention_layernorm.bias": "model-00003-of-00017.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00003-of-00017.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00003-of-00017.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00003-of-00017.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00003-of-00017.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00003-of-00017.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00003-of-00017.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00004-of-00017.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00004-of-00017.safetensors",
+    "model.layers.7.input_layernorm.bias": "model-00004-of-00017.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00004-of-00017.safetensors",
+    "model.layers.7.post_attention_layernorm.bias": "model-00004-of-00017.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00004-of-00017.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00004-of-00017.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00004-of-00017.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00004-of-00017.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00004-of-00017.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00004-of-00017.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00004-of-00017.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00004-of-00017.safetensors",
+    "model.layers.8.input_layernorm.bias": "model-00004-of-00017.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00004-of-00017.safetensors",
+    "model.layers.8.post_attention_layernorm.bias": "model-00004-of-00017.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00004-of-00017.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00004-of-00017.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00004-of-00017.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00004-of-00017.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00004-of-00017.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00004-of-00017.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00004-of-00017.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00004-of-00017.safetensors",
+    "model.layers.9.input_layernorm.bias": "model-00004-of-00017.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00004-of-00017.safetensors",
+    "model.layers.9.post_attention_layernorm.bias": "model-00004-of-00017.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00004-of-00017.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00004-of-00017.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00004-of-00017.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00004-of-00017.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00004-of-00017.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00004-of-00017.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00005-of-00017.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00005-of-00017.safetensors",
+    "model.layers.10.input_layernorm.bias": "model-00005-of-00017.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00005-of-00017.safetensors",
+    "model.layers.10.post_attention_layernorm.bias": "model-00005-of-00017.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00005-of-00017.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00005-of-00017.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00005-of-00017.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00005-of-00017.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00005-of-00017.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00005-of-00017.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00005-of-00017.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00005-of-00017.safetensors",
+    "model.layers.11.input_layernorm.bias": "model-00005-of-00017.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00005-of-00017.safetensors",
+    "model.layers.11.post_attention_layernorm.bias": "model-00005-of-00017.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00005-of-00017.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00005-of-00017.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00005-of-00017.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00005-of-00017.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00005-of-00017.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00005-of-00017.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00005-of-00017.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00005-of-00017.safetensors",
+    "model.layers.12.input_layernorm.bias": "model-00005-of-00017.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00005-of-00017.safetensors",
+    "model.layers.12.post_attention_layernorm.bias": "model-00005-of-00017.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00005-of-00017.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00005-of-00017.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00005-of-00017.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00005-of-00017.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00005-of-00017.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00005-of-00017.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00006-of-00017.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00006-of-00017.safetensors",
+    "model.layers.13.input_layernorm.bias": "model-00006-of-00017.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00006-of-00017.safetensors",
+    "model.layers.13.post_attention_layernorm.bias": "model-00006-of-00017.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00006-of-00017.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00006-of-00017.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00006-of-00017.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00006-of-00017.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00006-of-00017.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00006-of-00017.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00006-of-00017.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00006-of-00017.safetensors",
+    "model.layers.14.input_layernorm.bias": "model-00006-of-00017.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00006-of-00017.safetensors",
+    "model.layers.14.post_attention_layernorm.bias": "model-00006-of-00017.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00006-of-00017.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00006-of-00017.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00006-of-00017.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00006-of-00017.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00006-of-00017.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00006-of-00017.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00006-of-00017.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00006-of-00017.safetensors",
+    "model.layers.15.input_layernorm.bias": "model-00006-of-00017.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00006-of-00017.safetensors",
+    "model.layers.15.post_attention_layernorm.bias": "model-00006-of-00017.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00006-of-00017.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00006-of-00017.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00006-of-00017.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00006-of-00017.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00006-of-00017.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00006-of-00017.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00007-of-00017.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00007-of-00017.safetensors",
+    "model.layers.16.input_layernorm.bias": "model-00007-of-00017.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00007-of-00017.safetensors",
+    "model.layers.16.post_attention_layernorm.bias": "model-00007-of-00017.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00007-of-00017.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00007-of-00017.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00007-of-00017.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00007-of-00017.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00007-of-00017.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00007-of-00017.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00007-of-00017.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00007-of-00017.safetensors",
+    "model.layers.17.input_layernorm.bias": "model-00007-of-00017.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00007-of-00017.safetensors",
+    "model.layers.17.post_attention_layernorm.bias": "model-00007-of-00017.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00007-of-00017.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00007-of-00017.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00007-of-00017.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00007-of-00017.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00007-of-00017.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00007-of-00017.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00007-of-00017.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00007-of-00017.safetensors",
+    "model.layers.18.input_layernorm.bias": "model-00007-of-00017.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00007-of-00017.safetensors",
+    "model.layers.18.post_attention_layernorm.bias": "model-00007-of-00017.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00007-of-00017.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00007-of-00017.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00007-of-00017.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00007-of-00017.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00007-of-00017.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00007-of-00017.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00008-of-00017.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00008-of-00017.safetensors",
+    "model.layers.19.input_layernorm.bias": "model-00008-of-00017.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00008-of-00017.safetensors",
+    "model.layers.19.post_attention_layernorm.bias": "model-00008-of-00017.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00008-of-00017.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00008-of-00017.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00008-of-00017.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00008-of-00017.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00008-of-00017.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00008-of-00017.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00008-of-00017.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00008-of-00017.safetensors",
+    "model.layers.20.input_layernorm.bias": "model-00008-of-00017.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00008-of-00017.safetensors",
+    "model.layers.20.post_attention_layernorm.bias": "model-00008-of-00017.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00008-of-00017.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00008-of-00017.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00008-of-00017.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00008-of-00017.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00008-of-00017.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00008-of-00017.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00008-of-00017.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00008-of-00017.safetensors",
+    "model.layers.21.input_layernorm.bias": "model-00008-of-00017.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00008-of-00017.safetensors",
+    "model.layers.21.post_attention_layernorm.bias": "model-00008-of-00017.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00008-of-00017.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00008-of-00017.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00008-of-00017.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00008-of-00017.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00008-of-00017.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00008-of-00017.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00009-of-00017.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00009-of-00017.safetensors",
+    "model.layers.22.input_layernorm.bias": "model-00009-of-00017.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00009-of-00017.safetensors",
+    "model.layers.22.post_attention_layernorm.bias": "model-00009-of-00017.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00009-of-00017.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00009-of-00017.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00009-of-00017.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00009-of-00017.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00009-of-00017.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00009-of-00017.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00009-of-00017.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00009-of-00017.safetensors",
+    "model.layers.23.input_layernorm.bias": "model-00009-of-00017.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00009-of-00017.safetensors",
+    "model.layers.23.post_attention_layernorm.bias": "model-00009-of-00017.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00009-of-00017.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00009-of-00017.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00009-of-00017.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00009-of-00017.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00009-of-00017.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00009-of-00017.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00009-of-00017.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00009-of-00017.safetensors",
+    "model.layers.24.input_layernorm.bias": "model-00009-of-00017.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00009-of-00017.safetensors",
+    "model.layers.24.post_attention_layernorm.bias": "model-00009-of-00017.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00009-of-00017.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00009-of-00017.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00009-of-00017.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00009-of-00017.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00009-of-00017.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00009-of-00017.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00010-of-00017.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00010-of-00017.safetensors",
+    "model.layers.25.input_layernorm.bias": "model-00010-of-00017.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00010-of-00017.safetensors",
+    "model.layers.25.post_attention_layernorm.bias": "model-00010-of-00017.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00010-of-00017.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00010-of-00017.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00010-of-00017.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00010-of-00017.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00010-of-00017.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00010-of-00017.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00010-of-00017.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00010-of-00017.safetensors",
+    "model.layers.26.input_layernorm.bias": "model-00010-of-00017.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00010-of-00017.safetensors",
+    "model.layers.26.post_attention_layernorm.bias": "model-00010-of-00017.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00010-of-00017.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00010-of-00017.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00010-of-00017.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00010-of-00017.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00010-of-00017.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00010-of-00017.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00010-of-00017.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00010-of-00017.safetensors",
+    "model.layers.27.input_layernorm.bias": "model-00010-of-00017.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00010-of-00017.safetensors",
+    "model.layers.27.post_attention_layernorm.bias": "model-00010-of-00017.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00010-of-00017.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00010-of-00017.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00010-of-00017.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00010-of-00017.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00010-of-00017.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00010-of-00017.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00011-of-00017.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00011-of-00017.safetensors",
+    "model.layers.28.input_layernorm.bias": "model-00011-of-00017.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00011-of-00017.safetensors",
+    "model.layers.28.post_attention_layernorm.bias": "model-00011-of-00017.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00011-of-00017.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00011-of-00017.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00011-of-00017.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00011-of-00017.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00011-of-00017.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00011-of-00017.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00011-of-00017.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00011-of-00017.safetensors",
+    "model.layers.29.input_layernorm.bias": "model-00011-of-00017.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00011-of-00017.safetensors",
+    "model.layers.29.post_attention_layernorm.bias": "model-00011-of-00017.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00011-of-00017.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00011-of-00017.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00011-of-00017.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00011-of-00017.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00011-of-00017.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00011-of-00017.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00011-of-00017.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00011-of-00017.safetensors",
+    "model.layers.30.input_layernorm.bias": "model-00011-of-00017.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00011-of-00017.safetensors",
+    "model.layers.30.post_attention_layernorm.bias": "model-00011-of-00017.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00011-of-00017.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00011-of-00017.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00011-of-00017.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00011-of-00017.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00011-of-00017.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00011-of-00017.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00012-of-00017.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00012-of-00017.safetensors",
+    "model.layers.31.input_layernorm.bias": "model-00012-of-00017.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00012-of-00017.safetensors",
+    "model.layers.31.post_attention_layernorm.bias": "model-00012-of-00017.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00012-of-00017.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00012-of-00017.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00012-of-00017.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00012-of-00017.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00012-of-00017.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00012-of-00017.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00012-of-00017.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00012-of-00017.safetensors",
+    "model.layers.32.input_layernorm.bias": "model-00012-of-00017.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00012-of-00017.safetensors",
+    "model.layers.32.post_attention_layernorm.bias": "model-00012-of-00017.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00012-of-00017.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00012-of-00017.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00012-of-00017.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00012-of-00017.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00012-of-00017.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00012-of-00017.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00012-of-00017.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00012-of-00017.safetensors",
+    "model.layers.33.input_layernorm.bias": "model-00012-of-00017.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00012-of-00017.safetensors",
+    "model.layers.33.post_attention_layernorm.bias": "model-00012-of-00017.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00012-of-00017.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00012-of-00017.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00012-of-00017.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00012-of-00017.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00012-of-00017.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00012-of-00017.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00013-of-00017.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00013-of-00017.safetensors",
+    "model.layers.34.input_layernorm.bias": "model-00013-of-00017.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00013-of-00017.safetensors",
+    "model.layers.34.post_attention_layernorm.bias": "model-00013-of-00017.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00013-of-00017.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00013-of-00017.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00013-of-00017.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00013-of-00017.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00013-of-00017.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00013-of-00017.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00013-of-00017.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00013-of-00017.safetensors",
+    "model.layers.35.input_layernorm.bias": "model-00013-of-00017.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00013-of-00017.safetensors",
+    "model.layers.35.post_attention_layernorm.bias": "model-00013-of-00017.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00013-of-00017.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00013-of-00017.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00013-of-00017.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00013-of-00017.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00013-of-00017.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00013-of-00017.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00013-of-00017.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00013-of-00017.safetensors",
+    "model.layers.36.input_layernorm.bias": "model-00013-of-00017.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00013-of-00017.safetensors",
+    "model.layers.36.post_attention_layernorm.bias": "model-00013-of-00017.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00013-of-00017.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00013-of-00017.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00013-of-00017.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00013-of-00017.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00013-of-00017.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00013-of-00017.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00014-of-00017.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00014-of-00017.safetensors",
+    "model.layers.37.input_layernorm.bias": "model-00014-of-00017.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00014-of-00017.safetensors",
+    "model.layers.37.post_attention_layernorm.bias": "model-00014-of-00017.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00014-of-00017.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00014-of-00017.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00014-of-00017.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00014-of-00017.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00014-of-00017.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00014-of-00017.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00014-of-00017.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00014-of-00017.safetensors",
+    "model.layers.38.input_layernorm.bias": "model-00014-of-00017.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00014-of-00017.safetensors",
+    "model.layers.38.post_attention_layernorm.bias": "model-00014-of-00017.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00014-of-00017.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00014-of-00017.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00014-of-00017.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00014-of-00017.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00014-of-00017.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00014-of-00017.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00014-of-00017.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00014-of-00017.safetensors",
+    "model.layers.39.input_layernorm.bias": "model-00014-of-00017.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00014-of-00017.safetensors",
+    "model.layers.39.post_attention_layernorm.bias": "model-00014-of-00017.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00014-of-00017.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00014-of-00017.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00014-of-00017.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00014-of-00017.safetensors",
+    "model.layers.40.mlp.gate_proj.weight": "model-00014-of-00017.safetensors",
+    "model.layers.40.mlp.up_proj.weight": "model-00014-of-00017.safetensors",
+    "model.layers.40.mlp.down_proj.weight": "model-00015-of-00017.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00015-of-00017.safetensors",
+    "model.layers.40.input_layernorm.bias": "model-00015-of-00017.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00015-of-00017.safetensors",
+    "model.layers.40.post_attention_layernorm.bias": "model-00015-of-00017.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00015-of-00017.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00015-of-00017.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00015-of-00017.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00015-of-00017.safetensors",
+    "model.layers.41.mlp.gate_proj.weight": "model-00015-of-00017.safetensors",
+    "model.layers.41.mlp.up_proj.weight": "model-00015-of-00017.safetensors",
+    "model.layers.41.mlp.down_proj.weight": "model-00015-of-00017.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00015-of-00017.safetensors",
+    "model.layers.41.input_layernorm.bias": "model-00015-of-00017.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00015-of-00017.safetensors",
+    "model.layers.41.post_attention_layernorm.bias": "model-00015-of-00017.safetensors",
+    "model.layers.42.self_attn.q_proj.weight": "model-00015-of-00017.safetensors",
+    "model.layers.42.self_attn.k_proj.weight": "model-00015-of-00017.safetensors",
+    "model.layers.42.self_attn.v_proj.weight": "model-00015-of-00017.safetensors",
+    "model.layers.42.self_attn.o_proj.weight": "model-00015-of-00017.safetensors",
+    "model.layers.42.mlp.gate_proj.weight": "model-00015-of-00017.safetensors",
+    "model.layers.42.mlp.up_proj.weight": "model-00015-of-00017.safetensors",
+    "model.layers.42.mlp.down_proj.weight": "model-00015-of-00017.safetensors",
+    "model.layers.42.input_layernorm.weight": "model-00015-of-00017.safetensors",
+    "model.layers.42.input_layernorm.bias": "model-00015-of-00017.safetensors",
+    "model.layers.42.post_attention_layernorm.weight": "model-00015-of-00017.safetensors",
+    "model.layers.42.post_attention_layernorm.bias": "model-00015-of-00017.safetensors",
+    "model.layers.43.self_attn.q_proj.weight": "model-00015-of-00017.safetensors",
+    "model.layers.43.self_attn.k_proj.weight": "model-00015-of-00017.safetensors",
+    "model.layers.43.self_attn.v_proj.weight": "model-00015-of-00017.safetensors",
+    "model.layers.43.self_attn.o_proj.weight": "model-00015-of-00017.safetensors",
+    "model.layers.43.mlp.gate_proj.weight": "model-00015-of-00017.safetensors",
+    "model.layers.43.mlp.up_proj.weight": "model-00015-of-00017.safetensors",
+    "model.layers.43.mlp.down_proj.weight": "model-00016-of-00017.safetensors",
+    "model.layers.43.input_layernorm.weight": "model-00016-of-00017.safetensors",
+    "model.layers.43.input_layernorm.bias": "model-00016-of-00017.safetensors",
+    "model.layers.43.post_attention_layernorm.weight": "model-00016-of-00017.safetensors",
+    "model.layers.43.post_attention_layernorm.bias": "model-00016-of-00017.safetensors",
+    "model.layers.44.self_attn.q_proj.weight": "model-00016-of-00017.safetensors",
+    "model.layers.44.self_attn.k_proj.weight": "model-00016-of-00017.safetensors",
+    "model.layers.44.self_attn.v_proj.weight": "model-00016-of-00017.safetensors",
+    "model.layers.44.self_attn.o_proj.weight": "model-00016-of-00017.safetensors",
+    "model.layers.44.mlp.gate_proj.weight": "model-00016-of-00017.safetensors",
+    "model.layers.44.mlp.up_proj.weight": "model-00016-of-00017.safetensors",
+    "model.layers.44.mlp.down_proj.weight": "model-00016-of-00017.safetensors",
+    "model.layers.44.input_layernorm.weight": "model-00016-of-00017.safetensors",
+    "model.layers.44.input_layernorm.bias": "model-00016-of-00017.safetensors",
+    "model.layers.44.post_attention_layernorm.weight": "model-00016-of-00017.safetensors",
+    "model.layers.44.post_attention_layernorm.bias": "model-00016-of-00017.safetensors",
+    "model.layers.45.self_attn.q_proj.weight": "model-00016-of-00017.safetensors",
+    "model.layers.45.self_attn.k_proj.weight": "model-00016-of-00017.safetensors",
+    "model.layers.45.self_attn.v_proj.weight": "model-00016-of-00017.safetensors",
+    "model.layers.45.self_attn.o_proj.weight": "model-00016-of-00017.safetensors",
+    "model.layers.45.mlp.gate_proj.weight": "model-00016-of-00017.safetensors",
+    "model.layers.45.mlp.up_proj.weight": "model-00016-of-00017.safetensors",
+    "model.layers.45.mlp.down_proj.weight": "model-00016-of-00017.safetensors",
+    "model.layers.45.input_layernorm.weight": "model-00016-of-00017.safetensors",
+    "model.layers.45.input_layernorm.bias": "model-00016-of-00017.safetensors",
+    "model.layers.45.post_attention_layernorm.weight": "model-00016-of-00017.safetensors",
+    "model.layers.45.post_attention_layernorm.bias": "model-00016-of-00017.safetensors",
+    "model.layers.46.self_attn.q_proj.weight": "model-00016-of-00017.safetensors",
+    "model.layers.46.self_attn.k_proj.weight": "model-00016-of-00017.safetensors",
+    "model.layers.46.self_attn.v_proj.weight": "model-00016-of-00017.safetensors",
+    "model.layers.46.self_attn.o_proj.weight": "model-00016-of-00017.safetensors",
+    "model.layers.46.mlp.gate_proj.weight": "model-00016-of-00017.safetensors",
+    "model.layers.46.mlp.up_proj.weight": "model-00016-of-00017.safetensors",
+    "model.layers.46.mlp.down_proj.weight": "model-00017-of-00017.safetensors",
+    "model.layers.46.input_layernorm.weight": "model-00017-of-00017.safetensors",
+    "model.layers.46.input_layernorm.bias": "model-00017-of-00017.safetensors",
+    "model.layers.46.post_attention_layernorm.weight": "model-00017-of-00017.safetensors",
+    "model.layers.46.post_attention_layernorm.bias": "model-00017-of-00017.safetensors",
+    "model.layers.47.self_attn.q_proj.weight": "model-00017-of-00017.safetensors",
+    "model.layers.47.self_attn.k_proj.weight": "model-00017-of-00017.safetensors",
+    "model.layers.47.self_attn.v_proj.weight": "model-00017-of-00017.safetensors",
+    "model.layers.47.self_attn.o_proj.weight": "model-00017-of-00017.safetensors",
+    "model.layers.47.mlp.gate_proj.weight": "model-00017-of-00017.safetensors",
+    "model.layers.47.mlp.up_proj.weight": "model-00017-of-00017.safetensors",
+    "model.layers.47.mlp.down_proj.weight": "model-00017-of-00017.safetensors",
+    "model.layers.47.input_layernorm.weight": "model-00017-of-00017.safetensors",
+    "model.layers.47.input_layernorm.bias": "model-00017-of-00017.safetensors",
+    "model.layers.47.post_attention_layernorm.weight": "model-00017-of-00017.safetensors",
+    "model.layers.47.post_attention_layernorm.bias": "model-00017-of-00017.safetensors",
+    "model.norm.weight": "model-00017-of-00017.safetensors",
+    "model.norm.bias": "model-00017-of-00017.safetensors",
+    "lm_head.weight": "model-00017-of-00017.safetensors"
+  }
+}

modeling_inflm.py

@@ -0,0 +1,69 @@
+# coding=utf-8
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
+#
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
+# and OPT implementations in this library. It has been modified from its
+# original forms to accommodate minor architectural differences compared
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTorch INFLM model."""
+
+import torch
+from torch import nn
+from transformers.models.llama.modeling_llama import (
+    LlamaDecoderLayer,
+    LlamaModel,
+    LlamaForCausalLM
+)
+from .configuration_inflm import INFLMConfig
+
+_CONFIG_FOR_DOC = "INFLMConfig"
+
+
+class INFLMDecoderLayer(LlamaDecoderLayer):
+    def __init__(self, config: INFLMConfig, layer_idx: int):
+        super().__init__(config, layer_idx)
+        self.input_layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.post_attention_layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+
+
+class INFLMModel(LlamaModel):
+    config_class = INFLMConfig
+    _no_split_modules = ["INFLMDecoderLayer"]
+    
+    def __init__(self, config: INFLMConfig):
+        super().__init__(config)
+        self.padding_idx = config.pad_token_id
+        self.vocab_size = config.vocab_size
+
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
+        self.layers = nn.ModuleList([INFLMDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)])
+        self.norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+
+        self.gradient_checkpointing = False
+        # Initialize weights and apply final processing
+        self.post_init()
+
+
+class INFLMForCausalLM(LlamaForCausalLM):
+    _tied_weights_keys = ["lm_head.weight"]
+
+    def __init__(self, config: INFLMConfig):
+        super().__init__(config)
+        self.model = INFLMModel(config)
+        self.vocab_size = config.vocab_size
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        # Initialize weights and apply final processing
+        self.post_init()

special_tokens_map.json

@@ -0,0 +1,64 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<|start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|message|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|tool_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|tool_excute|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|tool_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

tokenization_inflm.py

@@ -0,0 +1,292 @@
+# coding=utf-8
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
+#
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
+# and OPT implementations in this library. It has been modified from its
+# original forms to accommodate minor architectural differences compared
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tokenization classes for INFLMTokenizer."""
+import os
+from shutil import copyfile
+from typing import Any, Dict, List, Optional, Tuple
+
+import sentencepiece as spm
+
+from transformers.tokenization_utils import PreTrainedTokenizer
+from transformers.utils import logging
+
+from tokenizers import pre_tokenizers,Regex,decoders
+from tokenizers.pre_tokenizers import Digits, Split, ByteLevel
+import os 
+           
+# same as gpt4 cl-base-100k
+PATTERN = Regex("(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+\s+(\S)+")
+                
+logger = logging.get_logger(__name__)
+
+VOCAB_FILES_NAMES = {"vocab_file": "./tokenizer.model"}
+
+PRETRAINED_VOCAB_FILES_MAP = {}
+
+      
+class INFLMTokenizer(PreTrainedTokenizer):
+    """
+    Construct a INFLMTokenizer tokenizer based on sentence-piece 
+
+    Args:
+        vocab_file (`str`):
+            Path to the vocabulary file.
+    """
+
+    vocab_files_names = VOCAB_FILES_NAMES
+    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    model_input_names = ["input_ids", "attention_mask"]
+    _auto_class = "AutoTokenizer"
+
+    def __init__(
+        self,
+        vocab_file,
+        unk_token="<unk>",
+        bos_token="<s>",
+        eos_token="</s>",
+        pad_token="<pad>",
+        sp_model_kwargs: Optional[Dict[str, Any]] = None,
+        add_bos_token=False,
+        add_eos_token=False,
+        decode_with_prefix_space=False,
+        clean_up_tokenization_spaces=False,
+        spaces_between_special_tokens=False,
+        **kwargs,
+    ):
+        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
+        self.vocab_file = vocab_file
+        self.add_bos_token = add_bos_token
+        self.add_eos_token = add_eos_token
+        self.decode_with_prefix_space = decode_with_prefix_space
+        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
+        self.sp_model.Load(vocab_file)
+        self._no_prefix_space_tokens = None
+        self.pre_tokenizer = pre_tokenizers.Sequence([Split(pattern =PATTERN,behavior = "isolated", invert = False)])
+        super().__init__(
+            bos_token=bos_token,
+            eos_token=eos_token,
+            unk_token=unk_token,
+            pad_token=pad_token,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            spaces_between_special_tokens=spaces_between_special_tokens,
+            **kwargs,
+        ) 
+
+        """ Initialisation"""
+
+    @property
+    def no_prefix_space_tokens(self):
+        if self._no_prefix_space_tokens is None:
+            vocab = self.convert_ids_to_tokens(list(range(self.vocab_size)))
+            self._no_prefix_space_tokens = {i for i, tok in enumerate(vocab) if not tok.startswith("▁")}
+        return self._no_prefix_space_tokens
+
+    @property
+    def vocab_size(self):
+        """Returns vocab size"""
+        return self.sp_model.get_piece_size()
+
+    @property
+    def bos_token_id(self) -> Optional[int]:
+        return self.sp_model.bos_id()
+
+    @property
+    def eos_token_id(self) -> Optional[int]:
+        return self.sp_model.eos_id()
+
+    def get_vocab(self):
+        """Returns vocab as a dict"""
+        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+
+    def _tokenize(self, text):
+        """Returns a tokenized string."""
+        
+        splits = self.pre_tokenizer.pre_tokenize_str(text)
+        texts=[]
+       
+        for split in splits:
+            texts.extend(self.sp_model.encode(split[0], out_type=str))
+        return texts
+
+    def _convert_token_to_id(self, token):
+        """Converts a token (str) in an id using the vocab."""
+       
+        return self.sp_model.piece_to_id(token)
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        token = self.sp_model.IdToPiece(index)
+        return token
+
+    def _maybe_add_prefix_space(self, tokens, decoded):
+        if tokens and tokens[0] not in self.no_prefix_space_tokens:
+            return " " + decoded
+        else:
+            return decoded
+
+    def convert_tokens_to_string(self, tokens):
+        """Converts a sequence of tokens (string) in a single string."""
+        current_sub_tokens = []
+        out_string = ""
+        prev_is_special = False
+        for token in tokens:
+            # make sure that special tokens are not decoded using sentencepiece model
+            if token in self.all_special_tokens:
+                out_string += self.sp_model.decode(current_sub_tokens) + token
+                prev_is_special = True
+                current_sub_tokens = []
+            else:
+                current_sub_tokens.append(token)
+                prev_is_special = False
+        out_string += self.sp_model.decode(current_sub_tokens)
+        
+        return out_string
+
+    def save_vocabulary(self, save_directory, filename_prefix: Optional[str] = None) -> Tuple[str]:
+        """
+        Save the vocabulary and special tokens file to a directory.
+
+        Args:
+            save_directory (`str`):
+                The directory in which to save the vocabulary.
+
+        Returns:
+            `Tuple(str)`: Paths to the files saved.
+        """
+        if not os.path.isdir(save_directory):
+            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
+            return
+        out_vocab_file = os.path.join(
+            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
+        )
+
+        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
+            copyfile(self.vocab_file, out_vocab_file)
+        elif not os.path.isfile(self.vocab_file):
+            with open(out_vocab_file, "wb") as fi:
+                content_spiece_model = self.sp_model.serialized_model_proto()
+                fi.write(content_spiece_model)
+
+        return (out_vocab_file,)
+
+    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+        if self.add_bos_token:
+            bos_token_ids = [self.bos_token_id]
+        else:
+            bos_token_ids = []
+
+        output = bos_token_ids + token_ids_0
+
+        if token_ids_1 is not None:
+            output = output + token_ids_1
+
+        if self.add_eos_token:
+            output = output + [self.eos_token_id]
+
+        return output
+
+    def get_special_tokens_mask(
+        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
+    ) -> List[int]:
+        """
+        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
+        special tokens using the tokenizer `prepare_for_model` method.
+
+        Args:
+            token_ids_0 (`List[int]`):
+                List of IDs.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
+                Whether or not the token list is already formatted with special tokens for the model.
+
+        Returns:
+            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
+        """
+        if already_has_special_tokens:
+            return super().get_special_tokens_mask(
+                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
+            )
+
+        eos_token_id = [1] if self.add_eos_token else []
+        if token_ids_1 is None:
+            return  ([0] * len(token_ids_0)) + eos_token_id
+        return  ([0] * len(token_ids_0)) + eos_token_id + ([0] * len(token_ids_1)) + eos_token_id
+    
+
+    def create_token_type_ids_from_sequences(
+        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
+    ) -> List[int]:
+        """
+        Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT
+        sequence pair mask has the following format:
+
+        ```
+        0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
+        | first sequence    | second sequence |
+        ```
+
+        if token_ids_1 is None, only returns the first portion of the mask (0s).
+
+        Note this is only used for back compatiblity, thus list of zero is returned.
+
+        Args:
+            token_ids_0 (`List[int]`):
+                List of ids.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+
+        Returns:
+            `List[int]`: List of zeros.
+        """
+        eos = [self.eos_token_id]
+
+        if token_ids_1 is None:
+            return len(token_ids_0 + eos) * [0]
+        return len(token_ids_0 + eos + token_ids_1 + eos) * [0]
+    
+
+    @property
+    def default_chat_template(self):
+        return None
+
+
+    def decode(
+        self,
+        token_ids,
+        skip_special_tokens: bool = False,
+        clean_up_tokenization_spaces: Optional[bool] = False,
+        spaces_between_special_tokens: bool = False,
+        **kwargs,
+    ) -> str:
+        # default spaces_between_special_tokens should be false.
+        if spaces_between_special_tokens:
+            logger.warning_once('spaces_between_special_tokens is set. \
+                                It has no effect for bos,eos,pad,unk when transformers<=4.38.')
+        return super().decode(
+            token_ids,
+            skip_special_tokens=skip_special_tokens,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            spaces_between_special_tokens=spaces_between_special_tokens,
+            **kwargs,
+        )

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76d43d618fc0c5a7c79dc4e72579f9f29bb803b36e4a4d709d1233626fd8fe2a
+size 1535725

tokenizer_config.json

@@ -0,0 +1,129 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "96499": {
+      "content": "<|start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "96500": {
+      "content": "<|end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "96501": {
+      "content": "<|message|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "96502": {
+      "content": "<|tool_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "96503": {
+      "content": "<|tool_excute|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "96504": {
+      "content": "<|tool_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "96505": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "96506": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|start|>",
+    "<|end|>",
+    "<|message|>",
+    "<|tool_start|>",
+    "<|tool_excute|>",
+    "<|tool_end|>",
+    "<|pad|>",
+    "<|endoftext|>"
+  ],
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_inflm.INFLMTokenizer",
+      null
+    ]
+  },
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": false,
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "model_max_length": 1000000000000000019884624838656,
+  "chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}{% if not loop.first %}{{ '\\n' }}{% endif %}{{'<|start|>user\\n' + message['content'] + '<|end|>\\n' }}{% if (loop.last and add_generation_prompt) %}{{ '<|start|>assistant<|message|>' }}{% endif %}{% elif message['role'] == 'system' %}{{ '<|start|>system\\n' + message['content'] + '<|end|>' }}{% elif message['role'] == 'assistant' %}{{ '<|start|>assistant<|message|>' + message['content'] + '<|end|>' }}{% endif %}{% endfor %}",
+  "pad_token": "<pad>",
+  "return_tensors": true,
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "INFLMTokenizer",
+  "unk_token": "<unk>"
+}