Update tokenizer_config.json

According to https://huggingface.co/Tongyi-MAI/Z-Image-Turbo/blob/main/tokenizer/tokenizer_config.json, it should be `additional_special_tokens` ? otherwise it will leads to tokenization error.

Files changed (1) hide show

tokenizer/tokenizer_config.json +1 -1

tokenizer/tokenizer_config.json CHANGED Viewed

@@ -6,7 +6,7 @@
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|im_end|>",
   "errors": "replace",
-  "extra_special_tokens": [
     "<|im_start|>",
     "<|im_end|>",
     "<|object_ref_start|>",

   "clean_up_tokenization_spaces": false,
   "eos_token": "<|im_end|>",
   "errors": "replace",
+  "additional_special_tokens": [
     "<|im_start|>",
     "<|im_end|>",
     "<|object_ref_start|>",