mjbuehler commited on
Commit
51c5570
1 Parent(s): 1ab76f2

Upload tokenizer

Browse files
special_tokens_map.json CHANGED
@@ -1 +1,4 @@
1
- {}
 
 
 
 
1
+ {
2
+ "eos_token": "</s>",
3
+ "pad_token": "<pad>"
4
+ }
tokenizer.json CHANGED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
@@ -1,5 +1,6 @@
1
  {
2
  "clean_up_tokenization_spaces": true,
3
  "model_max_length": 1000000000000000019884624838656,
 
4
  "tokenizer_class": "PreTrainedTokenizerFast"
5
  }
 
1
  {
2
  "clean_up_tokenization_spaces": true,
3
  "model_max_length": 1000000000000000019884624838656,
4
+ "padding": "max_length",
5
  "tokenizer_class": "PreTrainedTokenizerFast"
6
  }