tokenizer_config.json

{
 "use_fast": false,
 "tokenizer_class": "sudachitra.tokenization_electra_sudachipy.ElectraSudachipyTokenizer",
 "do_lower_case": false,
 "do_word_tokenize": true,
 "do_subword_tokenize": true,
 "word_tokenizer_type": "sudachipy",
 "subword_tokenizer_type": "wordpiece",
 "word_form_type": "dictionary_and_surface",
 "sudachipy_kwargs": {
  "split_mode": "A",
  "dict_type": "core"
 },
 "unk_token": "[UNK]",
 "sep_token": "[SEP]",
 "pad_token": "[PAD]",
 "cls_token": "[CLS]",
 "mask_token": "[MASK]"
}