|
{ |
|
"add_bos_token": false, |
|
"add_prefix_space": false, |
|
"added_tokens_decoder": { |
|
"151643": { |
|
"content": "<|endoftext|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151644": { |
|
"content": "<|im_start|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151645": { |
|
"content": "<|im_end|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151646": { |
|
"content": "<|object_ref_start|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151647": { |
|
"content": "<|object_ref_end|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151648": { |
|
"content": "<|box_start|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151649": { |
|
"content": "<|box_end|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151650": { |
|
"content": "<|quad_start|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151651": { |
|
"content": "<|quad_end|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151652": { |
|
"content": "<|vision_start|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151653": { |
|
"content": "<|vision_end|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151654": { |
|
"content": "<|vision_pad|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151655": { |
|
"content": "<|image_pad|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151656": { |
|
"content": "<|video_pad|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151657": { |
|
"content": "<tool_call>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"151658": { |
|
"content": "</tool_call>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"151659": { |
|
"content": "<|fim_prefix|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"151660": { |
|
"content": "<|fim_middle|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"151661": { |
|
"content": "<|fim_suffix|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"151662": { |
|
"content": "<|fim_pad|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"151663": { |
|
"content": "<|repo_name|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"151664": { |
|
"content": "<|file_sep|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
} |
|
}, |
|
"additional_special_tokens": [ |
|
"<|im_start|>", |
|
"<|im_end|>", |
|
"<|object_ref_start|>", |
|
"<|object_ref_end|>", |
|
"<|box_start|>", |
|
"<|box_end|>", |
|
"<|quad_start|>", |
|
"<|quad_end|>", |
|
"<|vision_start|>", |
|
"<|vision_end|>", |
|
"<|vision_pad|>", |
|
"<|image_pad|>", |
|
"<|video_pad|>" |
|
], |
|
"bos_token": null, |
|
"chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}", |
|
"clean_up_tokenization_spaces": false, |
|
"eos_token": "<|im_end|>", |
|
"errors": "replace", |
|
"model_max_length": 131072, |
|
"pad_token": "<|endoftext|>", |
|
"split_special_tokens": false, |
|
"tokenizer_class": "Qwen2Tokenizer", |
|
"unk_token": null, |
|
"use_fast": true |
|
} |
|
|