fasttext-classification / tokenizer_config.json
Taizo Kaneko
commit files to HF hub
97c46f0
{
"auto_map": {
"AutoTokenizer": [
"fasttext_jp_tokenizer.FastTextJpTokenizer",
null
]
},
"model_max_length": 128,
"pad_token": "*",
"tokenizer_class": "FastTextJpTokenizer",
"unk_token": "*"
}