Upload tokenizer

chat_template.jinja

@@ -1,22 +1,15 @@
-
-{{ bos_token }}
-{% if messages[0]['role'] == 'system' %}
-  {{ raise_exception('System role not supported') }}
-{% endif %}
 {% for message in messages %}
-  {% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}
-    {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}
-  {% endif %}
-  {% if message['role'] == 'assistant' %}
-<start_of_turn>model
-{% generation %}{{ message['content'] | trim }}{% endgeneration %}
-<end_of_turn>
-  {% else %}
-<start_of_turn>{{ message['role'] }}
-{{ message['content'] | trim }}
-<end_of_turn>
-  {% endif %}
-{% endfor %}
-{% if add_generation_prompt %}
-<start_of_turn>model
+{% if message['role'] == 'user' %}
+{{ '<|user|>
+' + message['content'] + eos_token }}
+{% elif message['role'] == 'system' %}
+{{ '<|system|>
+' + message['content'] + eos_token }}
+{% elif message['role'] == 'assistant' %}
+{{ '<|assistant|>
+'  + message['content'] + eos_token }}
+{% endif %}
+{% if loop.last and add_generation_prompt %}
+{{ '<|assistant|>' }}
 {% endif %}
+{% endfor %}

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:487cee8724215dcd2dde8888539e8b1bf844ceb5dbbe27f7845abda69eeb060f
-size 34362872
+oid sha256:81bb383c6138159665cd350a59fbcb5aa1fe65712cf5ed9b6481bc802cdbd8e5
+size 3618870

tokenizer_config.json

@@ -1,19 +1,15 @@
 {
+  "add_prefix_space": null,
   "backend": "tokenizers",
-  "bos_token": "<bos>",
+  "bos_token": "<s>",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "<eos>",
-  "extra_special_tokens": [
-    "<start_of_turn>",
-    "<end_of_turn>"
-  ],
-  "is_local": false,
-  "mask_token": "<mask>",
-  "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "<pad>",
+  "eos_token": "</s>",
+  "is_local": true,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "padding_side": "right",
   "sp_model_kwargs": {},
-  "spaces_between_special_tokens": false,
-  "tokenizer_class": "GemmaTokenizer",
+  "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
   "use_default_system_prompt": false
 }