Mxytyu commited on
Commit
933c4fb
1 Parent(s): c065140

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +8 -5
tokenizer_config.json CHANGED
@@ -1,7 +1,10 @@
1
  {
2
- "vocab_size": 42,
3
- "special_tokens": ["<banana>", "<potato>", "<squirrel>"],
4
- "unk_token": "<huh?>",
5
- "bos_token": "<go!>",
6
- "eos_token": "<whew>"
 
 
 
7
  }
 
1
  {
2
+ "do_lower_case": false,
3
+ "model_max_length": 512,
4
+ "padding_side": "right",
5
+ "special_tokens_map_file": null,
6
+ "tokenizer_file": "tokenizer.json",
7
+ "unk_token": "<unk>",
8
+ "bos_token": "<s>",
9
+ "eos_token": "</s>"
10
  }