|
{ |
|
"added_tokens_decoder": { |
|
"151329": { |
|
"content": "<|endoftext|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151330": { |
|
"content": "[MASK]", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151331": { |
|
"content": "[gMASK]", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151332": { |
|
"content": "[sMASK]", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151333": { |
|
"content": "<sop>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151334": { |
|
"content": "<eop>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151335": { |
|
"content": "<|system|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151336": { |
|
"content": "<|user|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151337": { |
|
"content": "<|assistant|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151338": { |
|
"content": "<|observation|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151339": { |
|
"content": "<|begin_of_image|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151340": { |
|
"content": "<|end_of_image|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151341": { |
|
"content": "<|begin_of_video|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151342": { |
|
"content": "<|end_of_video|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
} |
|
}, |
|
"additional_special_tokens": [ |
|
"<|endoftext|>", |
|
"[MASK]", |
|
"[gMASK]", |
|
"[sMASK]", |
|
"<sop>", |
|
"<eop>", |
|
"<|system|>", |
|
"<|user|>", |
|
"<|assistant|>", |
|
"<|observation|>", |
|
"<|begin_of_image|>", |
|
"<|end_of_image|>", |
|
"<|begin_of_video|>", |
|
"<|end_of_video|>" |
|
], |
|
"auto_map": { |
|
"AutoTokenizer": [ |
|
"tokenization_chatglm.ChatGLM4Tokenizer", |
|
null |
|
] |
|
}, |
|
"chat_template": "[gMASK]<sop>{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ 'system\n당신은 한국어 문서의 내용을 기반으로 사용자의 질문에 정확한 답변을 하는 지능형 한국어 어시스턴트 KoGLM-4입니다. 답변은 한국어로만 말해 주세요.除非另有说明,请用韩语回答\n' }}{% endif %}{{'' + message['role'] + '\n' + message['content'] + '' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ 'assistant\n' }}{% endif %}", |
|
"clean_up_tokenization_spaces": false, |
|
"do_lower_case": false, |
|
"eos_token": "<|endoftext|>", |
|
"model_max_length": 128000, |
|
"pad_token": "<|endoftext|>", |
|
"padding_side": "left", |
|
"remove_space": false, |
|
"tokenizer_class": "ChatGLM4Tokenizer" |
|
} |
|
|