|
{ |
|
"version": "1.0", |
|
"truncation": null, |
|
"padding": null, |
|
"added_tokens": [ |
|
{ |
|
"id": 0, |
|
"content": "[EOS]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 1, |
|
"content": "[UNK]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 2, |
|
"content": "[CLS]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 3, |
|
"content": "[PAD]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
} |
|
], |
|
"normalizer": null, |
|
"pre_tokenizer": { |
|
"type": "Whitespace" |
|
}, |
|
"post_processor": { |
|
"type": "TemplateProcessing", |
|
"single": [ |
|
{ |
|
"Sequence": { |
|
"id": "A", |
|
"type_id": 0 |
|
} |
|
}, |
|
{ |
|
"SpecialToken": { |
|
"id": "[EOS]", |
|
"type_id": 0 |
|
} |
|
} |
|
], |
|
"pair": [ |
|
{ |
|
"Sequence": { |
|
"id": "A", |
|
"type_id": 0 |
|
} |
|
}, |
|
{ |
|
"Sequence": { |
|
"id": "B", |
|
"type_id": 1 |
|
} |
|
} |
|
], |
|
"special_tokens": { |
|
"[EOS]": { |
|
"id": "[EOS]", |
|
"ids": [ |
|
0 |
|
], |
|
"tokens": [ |
|
"[EOS]" |
|
] |
|
} |
|
} |
|
}, |
|
"decoder": null, |
|
"model": { |
|
"type": "BPE", |
|
"dropout": null, |
|
"unk_token": "[UNK]", |
|
"continuing_subword_prefix": null, |
|
"end_of_word_suffix": null, |
|
"fuse_unk": false, |
|
"byte_fallback": false, |
|
"vocab": { |
|
"[EOS]": 0, |
|
"[UNK]": 1, |
|
"[CLS]": 2, |
|
"[PAD]": 3, |
|
"+": 4, |
|
"-": 5, |
|
"0": 6, |
|
"1": 7, |
|
"2": 8, |
|
"3": 9, |
|
"4": 10, |
|
"5": 11, |
|
"6": 12, |
|
"7": 13, |
|
"8": 14, |
|
"9": 15, |
|
"10": 16, |
|
"99": 17, |
|
"11": 18, |
|
"98": 19, |
|
"97": 20, |
|
"12": 21, |
|
"96": 22, |
|
"13": 23, |
|
"95": 24, |
|
"14": 25, |
|
"15": 26, |
|
"94": 27, |
|
"16": 28, |
|
"93": 29, |
|
"17": 30, |
|
"92": 31, |
|
"91": 32, |
|
"18": 33, |
|
"90": 34, |
|
"19": 35, |
|
"20": 36, |
|
"89": 37, |
|
"21": 38, |
|
"88": 39, |
|
"87": 40, |
|
"22": 41, |
|
"23": 42, |
|
"86": 43, |
|
"24": 44, |
|
"85": 45, |
|
"25": 46, |
|
"84": 47, |
|
"83": 48, |
|
"26": 49, |
|
"82": 50 |
|
}, |
|
"merges": [ |
|
"1 0", |
|
"9 9", |
|
"1 1", |
|
"9 8", |
|
"9 7", |
|
"1 2", |
|
"9 6", |
|
"1 3", |
|
"9 5", |
|
"1 4", |
|
"1 5", |
|
"9 4", |
|
"1 6", |
|
"9 3", |
|
"1 7", |
|
"9 2", |
|
"9 1", |
|
"1 8", |
|
"9 0", |
|
"1 9", |
|
"2 0", |
|
"8 9", |
|
"2 1", |
|
"8 8", |
|
"8 7", |
|
"2 2", |
|
"2 3", |
|
"8 6", |
|
"2 4", |
|
"8 5", |
|
"2 5", |
|
"8 4", |
|
"8 3", |
|
"2 6", |
|
"8 2" |
|
] |
|
} |
|
} |