Upload tokenizer

special_tokens_map.json

@@ -1,6 +1,30 @@
 {
-  "bos_token": "<s>",
-  "eos_token": "<|im_end|>",
-  "pad_token": "<|im_end|>",
-  "unk_token": "<unk>"
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
 }

tokenizer.json

@@ -23,7 +23,7 @@
     },
     {
       "id": 2,
-      "content": "<|im_end|>",
+      "content": "</s>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -137,7 +137,7 @@
     "vocab": {
       "<unk>": 0,
       "<s>": 1,
-      "<|im_end|>": 2,
+      "</s>": 2,
       "<0x00>": 3,
       "<0x01>": 4,
       "<0x02>": 5,

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -19,7 +19,7 @@
       "special": true
     },
     "2": {
-      "content": "<|im_end|>",
+      "content": "</s>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -27,17 +27,18 @@
       "special": true
     }
   },
+  "additional_special_tokens": [],
   "bos_token": "<s>",
-  "chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}{{'<|im_start|>user\n' + message['content'] + '<|im_end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|im_start|>assistant\n' + message['content'] + '<|im_end|>\n' }}{% else %}{{ '<|im_start|>system\n' + message['content'] + '<|im_end|>\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "<|im_end|>",
-  "max_length": 2048,
-  "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "<|im_end|>",
-  "stride": 0,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 32768,
+  "pad_token": "<unk>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",
-  "truncation_side": "right",
-  "truncation_strategy": "longest_first",
   "unk_token": "<unk>",
   "use_default_system_prompt": false
 }