{ | |
"version": "1.0", | |
"truncation": null, | |
"padding": null, | |
"added_tokens": [ | |
{ | |
"id": 48, | |
"content": "[PAD]", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
}, | |
{ | |
"id": 49, | |
"content": "[CLS]", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
}, | |
{ | |
"id": 50, | |
"content": "[EOS]", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
} | |
], | |
"normalizer": null, | |
"pre_tokenizer": { | |
"type": "Whitespace" | |
}, | |
"post_processor": null, | |
"decoder": null, | |
"model": { | |
"type": "BPE", | |
"dropout": null, | |
"unk_token": null, | |
"continuing_subword_prefix": null, | |
"end_of_word_suffix": null, | |
"fuse_unk": false, | |
"byte_fallback": false, | |
"vocab": { | |
"+": 0, | |
"-": 1, | |
"0": 2, | |
"1": 3, | |
"2": 4, | |
"3": 5, | |
"4": 6, | |
"5": 7, | |
"6": 8, | |
"7": 9, | |
"8": 10, | |
"9": 11, | |
"=": 12, | |
"10": 13, | |
"99": 14, | |
"11": 15, | |
"98": 16, | |
"12": 17, | |
"97": 18, | |
"96": 19, | |
"13": 20, | |
"14": 21, | |
"95": 22, | |
"15": 23, | |
"94": 24, | |
"16": 25, | |
"93": 26, | |
"17": 27, | |
"92": 28, | |
"18": 29, | |
"91": 30, | |
"19": 31, | |
"90": 32, | |
"20": 33, | |
"89": 34, | |
"88": 35, | |
"21": 36, | |
"22": 37, | |
"87": 38, | |
"23": 39, | |
"86": 40, | |
"24": 41, | |
"85": 42, | |
"84": 43, | |
"25": 44, | |
"26": 45, | |
"83": 46, | |
"27": 47 | |
}, | |
"merges": [ | |
"1 0", | |
"9 9", | |
"1 1", | |
"9 8", | |
"1 2", | |
"9 7", | |
"9 6", | |
"1 3", | |
"1 4", | |
"9 5", | |
"1 5", | |
"9 4", | |
"1 6", | |
"9 3", | |
"1 7", | |
"9 2", | |
"1 8", | |
"9 1", | |
"1 9", | |
"9 0", | |
"2 0", | |
"8 9", | |
"8 8", | |
"2 1", | |
"2 2", | |
"8 7", | |
"2 3", | |
"8 6", | |
"2 4", | |
"8 5", | |
"8 4", | |
"2 5", | |
"2 6", | |
"8 3", | |
"2 7" | |
] | |
} | |
} |