{ | |
"add_bos_token": false, | |
"add_eos_token": false, | |
"added_tokens_decoder": { | |
"0": { | |
"content": "<unk>", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": true, | |
"special": true | |
}, | |
"1": { | |
"content": "<s>", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"2": { | |
"content": "</s>", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": true, | |
"special": true | |
}, | |
"64000": { | |
"content": "<|CLS|>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"64001": { | |
"content": "<|SEP|>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"64002": { | |
"content": "<|EOD|>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"64003": { | |
"content": "<|MASK|>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"64004": { | |
"content": "<|PAD|>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
} | |
}, | |
"additional_special_tokens": [ | |
"<|CLS|>", | |
"<|SEP|>", | |
"<|EOD|>", | |
"<|MASK|>", | |
"<|PAD|>" | |
], | |
"auto_map": { | |
"AutoTokenizer": [ | |
"m-a-p/neo_7b--tokenization_neo.NEOTokenizer", | |
null | |
] | |
}, | |
"bos_token": "<s>", | |
"chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}", | |
"clean_up_tokenization_spaces": false, | |
"eos_token": "</s>", | |
"model_max_length": 4096, | |
"pad_token": "<unk>", | |
"padding_side": "right", | |
"sp_model_kwargs": {}, | |
"split_special_tokens": false, | |
"tokenizer_class": "NEOTokenizer", | |
"unk_token": "<unk>", | |
"use_fast": false | |
} | |