Upload model trained with Unsloth

Upload model trained with Unsloth 2x faster

special_tokens_map.json

@@ -7,17 +7,17 @@
     "single_word": false
   },
   "eos_token": {
-    "content": "<|endoftext|>",
+    "content": "</s>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "pad_token": {
-    "content": "<|placeholder6|>",
+    "content": "<unk>",
     "lstrip": false,
     "normalized": false,
-    "rstrip": true,
+    "rstrip": false,
     "single_word": false
   },
   "unk_token": {

tokenizer.json

@@ -26,108 +26,9 @@
       "content": "</s>",
       "single_word": false,
       "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": false
-    },
-    {
-      "id": 32000,
-      "content": "<|endoftext|>",
-      "single_word": false,
-      "lstrip": false,
       "rstrip": false,
       "normalized": false,
       "special": true
-    },
-    {
-      "id": 32001,
-      "content": "<|assistant|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32002,
-      "content": "<|placeholder1|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32003,
-      "content": "<|placeholder2|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32004,
-      "content": "<|placeholder3|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32005,
-      "content": "<|placeholder4|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32006,
-      "content": "<|system|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32007,
-      "content": "<|end|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32008,
-      "content": "<|placeholder5|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32009,
-      "content": "<|placeholder6|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32010,
-      "content": "<|user|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
     }
   ],
   "normalizer": {

tokenizer_config.json

@@ -22,106 +22,18 @@
       "content": "</s>",
       "lstrip": false,
       "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": false
-    },
-    "32000": {
-      "content": "<|endoftext|>",
-      "lstrip": false,
-      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
-    },
-    "32001": {
-      "content": "<|assistant|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32002": {
-      "content": "<|placeholder1|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32003": {
-      "content": "<|placeholder2|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32004": {
-      "content": "<|placeholder3|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32005": {
-      "content": "<|placeholder4|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32006": {
-      "content": "<|system|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32007": {
-      "content": "<|end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32008": {
-      "content": "<|placeholder5|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32009": {
-      "content": "<|placeholder6|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32010": {
-      "content": "<|user|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
     }
   },
   "bos_token": "<s>",
-  "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "<|endoftext|>",
+  "eos_token": "</s>",
   "legacy": false,
   "model_max_length": 4096,
-  "pad_token": "<|placeholder6|>",
+  "pad_token": "<unk>",
   "padding_side": "left",
   "sp_model_kwargs": {},
   "tokenizer_class": "LlamaTokenizer",