Do not add EOS token when tokenizine by default (#4)
Browse files- Do not add EOS token when tokenizine by default (bcbb248ce2d4ced5b66280f88de70872bd8841de)
Co-authored-by: Plat <p1atdev@users.noreply.huggingface.co>
- tokenizer_config.json +1 -1
tokenizer_config.json
CHANGED
@@ -131,5 +131,5 @@
|
|
131 |
"add_dummy_prefix_space": false,
|
132 |
"legacy": false,
|
133 |
"add_bos_token": false,
|
134 |
-
"add_eos_token":
|
135 |
}
|
|
|
131 |
"add_dummy_prefix_space": false,
|
132 |
"legacy": false,
|
133 |
"add_bos_token": false,
|
134 |
+
"add_eos_token": false
|
135 |
}
|