ERNIE-Image-Turbo / tokenizer /tokenizer_config.json
HsiaX's picture
Upload folder using huggingface_hub
b4b4f0c verified
raw
history blame contribute delete
379 Bytes
{
"add_prefix_space": null,
"backend": "tokenizers",
"bos_token": "<s>",
"clean_up_tokenization_spaces": false,
"eos_token": "</s>",
"is_local": true,
"legacy": true,
"model_max_length": 2048,
"pad_token": "<pad>",
"processor_class": "PixtralProcessor",
"tokenizer_class": "TokenizersBackend",
"unk_token": "<unk>",
"use_default_system_prompt": false
}