{ | |
"version": "1.0", | |
"truncation": null, | |
"padding": null, | |
"added_tokens": [ | |
{ | |
"id": 0, | |
"content": "[UNK]", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
}, | |
{ | |
"id": 1, | |
"content": "[CLS]", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
}, | |
{ | |
"id": 2, | |
"content": "[SEP]", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
}, | |
{ | |
"id": 3, | |
"content": "[PAD]", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
}, | |
{ | |
"id": 4, | |
"content": "[MASK]", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
}, | |
{ | |
"id": 5, | |
"content": "[BOS]", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
}, | |
{ | |
"id": 6, | |
"content": "[EOS]", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
} | |
], | |
"normalizer": null, | |
"pre_tokenizer": { | |
"type": "Whitespace" | |
}, | |
"post_processor": null, | |
"decoder": null, | |
"model": { | |
"type": "BPE", | |
"dropout": null, | |
"unk_token": "[UNK]", | |
"continuing_subword_prefix": null, | |
"end_of_word_suffix": null, | |
"fuse_unk": false, | |
"byte_fallback": false, | |
"vocab": { | |
"[UNK]": 0, | |
"[CLS]": 1, | |
"[SEP]": 2, | |
"[PAD]": 3, | |
"[MASK]": 4, | |
"[BOS]": 5, | |
"[EOS]": 6, | |
"+": 7, | |
"-": 8, | |
"0": 9, | |
"1": 10, | |
"2": 11, | |
"3": 12, | |
"4": 13, | |
"5": 14, | |
"6": 15, | |
"7": 16, | |
"8": 17, | |
"9": 18, | |
"=": 19, | |
"99": 20, | |
"10": 21, | |
"98": 22, | |
"11": 23, | |
"12": 24, | |
"97": 25, | |
"13": 26, | |
"96": 27, | |
"95": 28, | |
"14": 29, | |
"94": 30, | |
"15": 31, | |
"16": 32, | |
"93": 33, | |
"17": 34, | |
"92": 35, | |
"91": 36, | |
"18": 37, | |
"19": 38, | |
"90": 39, | |
"20": 40, | |
"89": 41, | |
"21": 42, | |
"88": 43, | |
"22": 44, | |
"87": 45, | |
"23": 46, | |
"86": 47, | |
"24": 48, | |
"85": 49, | |
"84": 50, | |
"25": 51, | |
"26": 52, | |
"83": 53, | |
"27": 54, | |
"82": 55 | |
}, | |
"merges": [ | |
"9 9", | |
"1 0", | |
"9 8", | |
"1 1", | |
"1 2", | |
"9 7", | |
"1 3", | |
"9 6", | |
"9 5", | |
"1 4", | |
"9 4", | |
"1 5", | |
"1 6", | |
"9 3", | |
"1 7", | |
"9 2", | |
"9 1", | |
"1 8", | |
"1 9", | |
"9 0", | |
"2 0", | |
"8 9", | |
"2 1", | |
"8 8", | |
"2 2", | |
"8 7", | |
"2 3", | |
"8 6", | |
"2 4", | |
"8 5", | |
"8 4", | |
"2 5", | |
"2 6", | |
"8 3", | |
"2 7", | |
"8 2" | |
] | |
} | |
} |