{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": true }, "post_processor": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "decoder": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "model": { "type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "vocab": { "": 0, "A": 1, "R": 2, "N": 3, "D": 4, "B": 5, "C": 6, "E": 7, "Q": 8, "Z": 9, "G": 10, "H": 11, "I": 12, "L": 13, "K": 14, "M": 15, "F": 16, "P": 17, "S": 18, "T": 19, "W": 20, "Y": 21, "V": 22, "1": 23, "2": 24, "3": 25 }, "merges": [] } }