diwank commited on
Commit
f8ed169
1 Parent(s): 86090d7

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -0
tokenizer_config.json CHANGED
@@ -17,6 +17,7 @@
17
  },
18
  "clean_up_tokenization_spaces": true,
19
  "eos_token": "<|endoftext|>",
 
20
  "max_length": 1024,
21
  "model_max_length": 1024,
22
  "stride": 0,
 
17
  },
18
  "clean_up_tokenization_spaces": true,
19
  "eos_token": "<|endoftext|>",
20
+ "pad_token": "<|endoftext|>",
21
  "max_length": 1024,
22
  "model_max_length": 1024,
23
  "stride": 0,