Upload tokenizer_config.json

tokenizer_config.json

@@ -43,7 +43,7 @@
       "special": true
     },
     "32002": {
-      "content": "<|step|>",
+      "content": "<|placeholder1|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": true,
@@ -51,7 +51,7 @@
       "special": true
     },
     "32003": {
-      "content": "<|function_output|>",
+      "content": "<|placeholder2|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": true,
@@ -59,7 +59,7 @@
       "special": true
     },
     "32004": {
-      "content": "<|tag|>",
+      "content": "<|placeholder3|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": true,
@@ -67,7 +67,7 @@
       "special": true
     },
     "32005": {
-      "content": "<|function_call|>",
+      "content": "<|placeholder4|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": true,
@@ -91,7 +91,7 @@
       "special": true
     },
     "32008": {
-      "content": "<|raw|>",
+      "content": "<|placeholder5|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": true,
@@ -99,7 +99,7 @@
       "special": true
     },
     "32009": {
-      "content": "<|continue|>",
+      "content": "<|placeholder6|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": true,
@@ -113,235 +113,15 @@
       "rstrip": true,
       "single_word": false,
       "special": true
-    },
-    "32011": {
-      "content": "<|function_list|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32012": {
-      "content": "<|calc|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32013": {
-      "content": "<|code|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32014": {
-      "content": "<|/code|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32015": {
-      "content": "<|summary|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32016": {
-      "content": "<|resource|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32017": {
-      "content": "<|assistant_mask|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32018": {
-      "content": "<|start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32019": {
-      "content": "<|message|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32020": {
-      "content": "<|fim_prefix|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32021": {
-      "content": "<|fim_middle|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32022": {
-      "content": "<|fim_suffix|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32023": {
-      "content": "<|meta_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32024": {
-      "content": "<|ipynb_marker|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32025": {
-      "content": "<|diff_marker|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32026": {
-      "content": "<|ghissue|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32027": {
-      "content": "<|ghreview|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32028": {
-      "content": "<|disc_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32029": {
-      "content": "<|disc_sep|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32030": {
-      "content": "<|disc_thread|><|query|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32031": {
-      "content": "<|/query|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32032": {
-      "content": "<|data|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32033": {
-      "content": "<|/data|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32034": {
-      "content": "<|sys|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32035": {
-      "content": "<|/sys|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32036": {
-      "content": "<|inst|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32037": {
-      "content": "<|/inst|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
     }
   },
-  "additional_special_tokens": [
-    "<|/inst|>"
-  ],
   "bos_token": "<s>",
-  "chat_template": "{{ bos_token }}{% for message in messages %}{{'<|' + message['role'] + '|>' + '\n' + message['content'] + '<|end|>\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
+  "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
-  "legacy": false,
   "model_max_length": 4096,
-  "pad_token": "<unk>",
-  "padding_side": "right",
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
   "sp_model_kwargs": {},
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",