{ | |
"eos_token": "</s>", | |
"unk_token": "<unk>", | |
"pad_token": "<pad>", | |
"extra_ids": 0, | |
"additional_special_tokens": null, | |
"sp_model_kwargs": {}, | |
"tokenizer_class": "Midm_bitext_Tokenizer", | |
"auto_map": { | |
"AutoTokenizer": [ | |
"midm_bitext_tokenization.Midm_bitext_Tokenizer", | |
null | |
] | |
} | |
} | |