ybelkada commited on
Commit
d9ce6ea
1 Parent(s): da3e071

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -0
tokenizer_config.json CHANGED
@@ -16221,6 +16221,7 @@
16221
  ],
16222
  "clean_up_tokenization_spaces": true,
16223
  "eos_token": "<|endoftext|>",
 
16224
  "model_max_length": 32768,
16225
  "tokenizer_class": "PreTrainedTokenizerFast"
16226
  }
 
16221
  ],
16222
  "clean_up_tokenization_spaces": true,
16223
  "eos_token": "<|endoftext|>",
16224
+ "extra_special_tokens": {},
16225
  "model_max_length": 32768,
16226
  "tokenizer_class": "PreTrainedTokenizerFast"
16227
  }