ArthurZ HF staff commited on
Commit
6eb37bc
1 Parent(s): 3122c18

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -0
tokenizer_config.json CHANGED
@@ -308,6 +308,7 @@
308
  "model_max_length": 1000000000000000019884624838656,
309
  "pad_token": "<pad>",
310
  "sp_model_kwargs": {},
 
311
  "tokenizer_class": "T5Tokenizer",
312
  "unk_token": "<unk>"
313
  }
 
308
  "model_max_length": 1000000000000000019884624838656,
309
  "pad_token": "<pad>",
310
  "sp_model_kwargs": {},
311
+ "spaces_between_special_tokens": false,
312
  "tokenizer_class": "T5Tokenizer",
313
  "unk_token": "<unk>"
314
  }