{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "N", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "E", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "F", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": { "type": "Sequence", "normalizers": [ { "type": "NFD" }, { "type": "Strip", "strip_left": true, "strip_right": true } ] }, "pre_tokenizer": { "type": "Split", "pattern": { "Regex": "[a-zA-Z]" }, "behavior": "Isolated", "invert": false }, "post_processor": null, "decoder": { "type": "WordPiece", "prefix": "", "cleanup": true }, "model": { "type": "WordPiece", "unk_token": "N", "continuing_subword_prefix": "##", "max_input_chars_per_word": 100, "vocab": { "N": 0, "E": 1, "F": 2, "A": 3, "C": 4, "G": 5, "T": 6 } } }