{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [], "normalizer": null, "pre_tokenizer": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "post_processor": null, "decoder": null, "model": { "type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "vocab": { "<|endoftext|>": 0, " A": 1, " B": 2, " C": 3, " D": 4, " E": 5, " F": 6, " G": 7, " H": 8, " I": 9, " J": 10, " K": 11, " L": 12, " M": 13, " N": 14, " O": 15, " P": 16, " Q": 17, " R": 18, " S": 19, " T": 20, " U": 21, " V": 22, " W": 23, " X": 24, " Y": 25, " Z": 26, " a": 27, " b": 28, " c": 29, " d": 30 }, "merges": [] } }