Commit
•
6f101e5
1
Parent(s):
e5b0673
Update tokenizer_config.json
Browse files- tokenizer_config.json +1 -1
tokenizer_config.json
CHANGED
|
@@ -12,7 +12,7 @@
|
|
| 12 |
"mask_token": "[MASK]",
|
| 13 |
"word_form_type": "dictionary_and_surface",
|
| 14 |
"sudachi_kwargs": {
|
| 15 |
-
"sudachi_split_mode": "
|
| 16 |
"sudachi_dict_type": "core"
|
| 17 |
},
|
| 18 |
"use_fast": false,
|
|
|
|
| 12 |
"mask_token": "[MASK]",
|
| 13 |
"word_form_type": "dictionary_and_surface",
|
| 14 |
"sudachi_kwargs": {
|
| 15 |
+
"sudachi_split_mode": "B",
|
| 16 |
"sudachi_dict_type": "core"
|
| 17 |
},
|
| 18 |
"use_fast": false,
|