Commit
•
c62b708
1
Parent(s):
6921f66
Update tokenizer_config.json
Browse files- tokenizer_config.json +1 -1
tokenizer_config.json
CHANGED
|
@@ -77,7 +77,7 @@
|
|
| 77 |
"legacy": true,
|
| 78 |
"max_length": 1536,
|
| 79 |
"model_max_length": 1000000000000000019884624838656,
|
| 80 |
-
"pad_token": "<|
|
| 81 |
"sp_model_kwargs": {},
|
| 82 |
"spaces_between_special_tokens": false,
|
| 83 |
"stride": 0,
|
|
|
|
| 77 |
"legacy": true,
|
| 78 |
"max_length": 1536,
|
| 79 |
"model_max_length": 1000000000000000019884624838656,
|
| 80 |
+
"pad_token": "<|endoftext|>",
|
| 81 |
"sp_model_kwargs": {},
|
| 82 |
"spaces_between_special_tokens": false,
|
| 83 |
"stride": 0,
|