|
{ |
|
"add_prefix_space": false, |
|
"added_tokens_decoder": { |
|
"151643": { |
|
"content": "<|endoftext|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151644": { |
|
"content": "<|im_start|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151645": { |
|
"content": "<|im_end|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151646": { |
|
"content": "<|eot_id|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
} |
|
}, |
|
"additional_special_tokens": [ |
|
"<|im_start|>", |
|
"<|im_end|>" |
|
], |
|
"bos_token": null, |
|
"chat_template": "{% set system_message = 'You are a helpful assistant.' %}{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ '<|start_header_id|>system<|end_header_id|>\n\n' + system_message + '<|eot_id|>' }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|start_header_id|>user<|end_header_id|>\n\n' + content + '<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<|eot_id|>' }}{% endif %}{% endfor %}", |
|
"clean_up_tokenization_spaces": false, |
|
"eos_token": "<|eot_id|>", |
|
"errors": "replace", |
|
"model_max_length": 32768, |
|
"pad_token": "<|endoftext|>", |
|
"padding_side": "left", |
|
"split_special_tokens": false, |
|
"tokenizer_class": "Qwen2Tokenizer", |
|
"unk_token": null |
|
} |
|
|