Upload tokenizer
Browse files- tokenizer_config.json +1 -1
tokenizer_config.json
CHANGED
@@ -147,7 +147,7 @@
|
|
147 |
"<start_assistant>",
|
148 |
"<end_message>"
|
149 |
],
|
150 |
-
"chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}{{ '<start_user>' + message['content'].strip() + '<end_message>' }}{% elif message['role'] == 'system' %}{{ '<start_system>' + message['content'].strip() + '<end_message>' }}{% elif message['role'] == 'assistant' %}{{ '<
|
151 |
"clean_up_tokenization_spaces": true,
|
152 |
"eos_token": "<end_message>",
|
153 |
"model_max_length": 8192,
|
|
|
147 |
"<start_assistant>",
|
148 |
"<end_message>"
|
149 |
],
|
150 |
+
"chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}{{ '<start_user>' + message['content'].strip() + '<end_message>' }}{% elif message['role'] == 'system' %}{{ '<start_system>' + message['content'].strip() + '<end_message>' }}{% elif message['role'] == 'assistant' %}{{ '<start_assistant>' + message['content'] + '<end_message>' }}{% else %}{{ raise_exception('Only system, user and assistant roles are supported.') }}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<start_assistant>' }}{% endif %}{% endfor %}",
|
151 |
"clean_up_tokenization_spaces": true,
|
152 |
"eos_token": "<end_message>",
|
153 |
"model_max_length": 8192,
|