default unidic_lite

distilbert_japanese_tokenizer.py

@@ -440,7 +440,7 @@ class MecabTokenizer:
         do_lower_case=False,
         never_split=None,
         normalize_text=True,
-        mecab_dic: Optional[str] = "unidic",
+        mecab_dic: Optional[str] = "unidic_lite",
         mecab_option: Optional[str] = None,
     ):
         """
@@ -454,7 +454,7 @@ class MecabTokenizer:
                 [`PreTrainedTokenizer.tokenize`]) List of tokens not to split.
             **normalize_text**: (*optional*) boolean (default True)
                 Whether to apply unicode normalization to text before tokenization.
-            **mecab_dic**: (*optional*) string (default "unidic")
+            **mecab_dic**: (*optional*) string (default "unidic_lite")
                 Name of dictionary to be used for MeCab initialization. If you are using a system-installed dictionary,
                 set this option to `None` and modify *mecab_option*.
             **mecab_option**: (*optional*) string