SihyunPark
commited on
Commit
•
0c59ea2
1
Parent(s):
36c4d90
Upload tokenizer_config.json with huggingface_hub
Browse files- tokenizer_config.json +5 -0
tokenizer_config.json
CHANGED
@@ -210681,7 +210681,12 @@
|
|
210681 |
"special": false
|
210682 |
}
|
210683 |
},
|
|
|
|
|
|
|
|
|
210684 |
"bos_token": "<|begin_of_text|>",
|
|
|
210685 |
"clean_up_tokenization_spaces": true,
|
210686 |
"eos_token": "<|eot_id|>",
|
210687 |
"model_input_names": [
|
|
|
210681 |
"special": false
|
210682 |
}
|
210683 |
},
|
210684 |
+
"additional_special_tokens": [
|
210685 |
+
"<|eot_id|>",
|
210686 |
+
"<|eom_id|>"
|
210687 |
+
],
|
210688 |
"bos_token": "<|begin_of_text|>",
|
210689 |
+
"chat_template": "{{ '<|begin_of_text|>' }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ '<|start_header_id|>system<|end_header_id|>\n\n' + system_message + '<|eot_id|>' }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|start_header_id|>user<|end_header_id|>\n\n' + content + '<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<|eot_id|>' }}{% endif %}{% endfor %}",
|
210690 |
"clean_up_tokenization_spaces": true,
|
210691 |
"eos_token": "<|eot_id|>",
|
210692 |
"model_input_names": [
|