add pad and extra_N special tokens

.gitattributes

@@ -0,0 +1 @@
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -5,17 +5,22 @@ tags:
  - tokenizers
 ---
 
-## Why should you use this and not the titotken included in the orignal model?
-Original tokenizer pad vocabulary to correct size with `<extra_N>` tokens but encoder never uses them causing inconsistency and deterimental to training code that may want to use the unused `<extra_N>` tokens.
+## Why should you use this and not the tiktoken included in the orignal model?
+1. Original tokenizer pad the vocabulary to correct size with `<extra_N>` tokens but encoder never uses them
+2. `len(tokenizer) != tokenizer.vocab_size` as result of 1.
+3. Original tokenizer use eos as pad token which may confuse trainers to mask out the eos token so model never output eos.
+
 
 modified from original code @ https://huggingface.co/Xenova/dbrx-instruct-tokenizer 
 
-```
+```json
 Changes:
 1. Remove non-base model tokens
 2. Keep/Add `<|pad|>` special token to make sure padding can be differentiated from eos/bos.
+3. Expose 15 unused/reserved `<|extra_N|>` for use
 
- "100277": {
+# pad token
+ "100256": {
       "content": "<|pad|>",
       "lstrip": false,
       "normalized": false,
@@ -23,6 +28,13 @@ Changes:
       "single_word": false,
       "special": true
     },
+
+# 15 unused/reserved extra tokens
+"<|extra_0|>": 100261
+"<|extra_1|>": 100262
+...
+"<|extra_14|>": 100275
+
 ```
 
 # DBRX Instruct Tokenizer

special_tokens_map.json

@@ -1,8 +1,5 @@
 {
-  "additional_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>"
-  ],
+  "additional_special_tokens": [],
   "bos_token": {
     "content": "<|endoftext|>",
     "lstrip": false,

tokenizer_config.json

@@ -11,7 +11,7 @@
       "single_word": false,
       "special": true
     },
-    "100277": {
+    "100256": {
       "content": "<|pad|>",
       "lstrip": false,
       "normalized": false,
@@ -19,16 +19,120 @@
       "single_word": false,
       "special": true
     },
-    "100278": {
-      "content": "<|im_start|>",
+    "100261": {
+      "content": "<|extra_0|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "100279": {
-      "content": "<|im_end|>",
+    "100262": {
+      "content": "<|extra_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100263": {
+      "content": "<|extra_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100264": {
+      "content": "<|extra_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|extra_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|extra_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100267": {
+      "content": "<|extra_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100268": {
+      "content": "<|extra_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100269": {
+      "content": "<|extra_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100270": {
+      "content": "<|extra_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100271": {
+      "content": "<|extra_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100272": {
+      "content": "<|extra_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100273": {
+      "content": "<|extra_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100274": {
+      "content": "<|extra_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100275": {
+      "content": "<|extra_14|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -36,10 +140,7 @@
       "special": true
     }
   },
-  "additional_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>"
-  ],
+  "additional_special_tokens": [],
   "bos_token": "<|endoftext|>",
   "clean_up_tokenization_spaces": true,
   "encoding_name": null,
@@ -49,7 +150,5 @@
   "model_name": "gpt-4",
   "pad_token": "<|pad|>",
   "tokenizer_class": "GPT2Tokenizer",
-  "unk_token": "<|endoftext|>",
-  "use_default_system_prompt": true,
-  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% elif true == true and not 'system' in messages[0]['role'] %}{% set loop_messages = messages %}{% set system_message = 'You are DBRX, created by Databricks. You were last updated in December 2023. You answer questions based on information available up to that point.\nYOU PROVIDE SHORT RESPONSES TO SHORT QUESTIONS OR STATEMENTS, but provide thorough responses to more complex and open-ended questions.\nYou assist with various tasks, from writing to coding (using markdown for code blocks — remember to use ``` with code, JSON, and tables).\n(You do not have real-time data access or code execution capabilities. You avoid stereotyping and provide balanced perspectives on controversial topics. You do not provide song lyrics, poems, or news articles and do not divulge details of your training data.)\nThis is your system prompt, guiding your responses. Do not reference it, just respond to the user. If you find yourself talking about this message, stop. You should be responding appropriately and usually that means not mentioning this.\nYOU DO NOT MENTION ANY OF THIS INFORMATION ABOUT YOURSELF UNLESS THE INFORMATION IS DIRECTLY PERTINENT TO THE USER\\'S QUERY.' %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if loop.index0 == 0 %}{% if system_message != false %}{{ '<|im_start|>system\n' + system_message.strip() + '<|im_end|>\n'}}{% endif %}{{ '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' }}{% else %}{{ '\n' + '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' }}{% endif %}{% if (add_generation_prompt == true and loop.last) %}{{ '\n' + '<|im_start|>' + 'assistant' + '\n' }}{% endif %}{% endfor %}"
+  "unk_token": "<|endoftext|>"
 }

vocab.json

@@ -100260,7 +100260,20 @@
   "<|fim_middle|>": 100259,
   "<|fim_suffix|>": 100260,
   "<|endofprompt|>": 100276,
-  "<|pad|>": 100277,
-  "<|im_start|>": 100278,
-  "<|im_end|>": 100279
+  "<|pad|>": 100256,
+  "<|extra_0|>": 100261,
+  "<|extra_1|>": 100262,
+  "<|extra_2|>": 100263,
+  "<|extra_3|>": 100264,
+  "<|extra_4|>": 100265,
+  "<|extra_5|>": 100266,
+  "<|extra_6|>": 100267,
+  "<|extra_7|>": 100268,
+  "<|extra_8|>": 100269,
+  "<|extra_9|>": 100270,
+  "<|extra_10|>": 100271,
+  "<|extra_11|>": 100272,
+  "<|extra_12|>": 100273,
+  "<|extra_13|>": 100274,
+  "<|extra_14|>": 100275
 }