{ | |
"add_eos_token": false, | |
"add_prefix_space": false, | |
"add_special_tokens": true, | |
"auto_map": { | |
"AutoTokenizer": [ | |
"Salesforce/xgen-7b-8k-base--tokenization_xgen.XgenTokenizer", | |
null | |
] | |
}, | |
"clean_up_tokenization_spaces": true, | |
"eos_token": { | |
"__type": "AddedToken", | |
"content": "<|endoftext|>", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false | |
}, | |
"model_max_length": 8192, | |
"pad_token": null, | |
"tokenizer_class": "XgenTokenizer" | |
} | |