tokenizer_config.json

{
    "eos_token": "</s>", 
    "unk_token": "<unk>", 
    "pad_token": "<pad>", 
    "extra_ids": 0, 
    "additional_special_tokens": null, 
    "sp_model_kwargs": {}, 
    "tokenizer_class": "Midm_bitext_Tokenizer",
    "auto_map": {
      "AutoTokenizer": [
        "midm_bitext_tokenization.Midm_bitext_Tokenizer",
        null
      ]
    }
}