{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "[UNK]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "[CLS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "[SEP]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "[PAD]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 4, "content": "[MASK]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 5, "content": "[BOS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 6, "content": "[EOS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "Whitespace" }, "post_processor": { "type": "TemplateProcessing", "single": [ { "Sequence": { "id": "A", "type_id": 0 } }, { "SpecialToken": { "id": "[CLS]", "type_id": 0 } } ], "pair": [ { "Sequence": { "id": "A", "type_id": 0 } }, { "SpecialToken": { "id": "[CLS]", "type_id": 0 } }, { "Sequence": { "id": "B", "type_id": 1 } }, { "SpecialToken": { "id": "[EOS]", "type_id": 1 } } ], "special_tokens": { "[CLS]": { "id": "[CLS]", "ids": [ 0 ], "tokens": [ "[CLS]" ] }, "[EOS]": { "id": "[EOS]", "ids": [ 1 ], "tokens": [ "[EOS]" ] } } }, "decoder": null, "model": { "type": "BPE", "dropout": null, "unk_token": "[UNK]", "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "vocab": { "[UNK]": 0, "[CLS]": 1, "[SEP]": 2, "[PAD]": 3, "[MASK]": 4, "[BOS]": 5, "[EOS]": 6, "+": 7, "-": 8, "0": 9, "1": 10, "2": 11, "3": 12, "4": 13, "5": 14, "6": 15, "7": 16, "8": 17, "9": 18, "=": 19, "99": 20, "10": 21, "98": 22, "11": 23, "97": 24, "12": 25, "96": 26, "13": 27, "95": 28, "14": 29, "94": 30, "15": 31, "93": 32, "16": 33, "92": 34, "17": 35, "91": 36, "18": 37, "19": 38, "90": 39, "20": 40, "89": 41, "21": 42, "88": 43, "22": 44, "87": 45, "23": 46, "86": 47, "85": 48, "24": 49, "25": 50, "84": 51, "26": 52, "83": 53, "27": 54, "82": 55 }, "merges": [ "9 9", "1 0", "9 8", "1 1", "9 7", "1 2", "9 6", "1 3", "9 5", "1 4", "9 4", "1 5", "9 3", "1 6", "9 2", "1 7", "9 1", "1 8", "1 9", "9 0", "2 0", "8 9", "2 1", "8 8", "2 2", "8 7", "2 3", "8 6", "8 5", "2 4", "2 5", "8 4", "2 6", "8 3", "2 7", "8 2" ] } }