Do not add EOS token when tokenizine by default
#4
by
p1atdev
- opened
- tokenizer_config.json +1 -1
tokenizer_config.json
CHANGED
@@ -131,5 +131,5 @@
|
|
131 |
"add_dummy_prefix_space": false,
|
132 |
"legacy": false,
|
133 |
"add_bos_token": false,
|
134 |
-
"add_eos_token":
|
135 |
}
|
|
|
131 |
"add_dummy_prefix_space": false,
|
132 |
"legacy": false,
|
133 |
"add_bos_token": false,
|
134 |
+
"add_eos_token": false
|
135 |
}
|