{ | |
"version": "1.0", | |
"truncation": null, | |
"padding": null, | |
"added_tokens": [ | |
{ | |
"id": 0, | |
"content": "<pad>", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
}, | |
{ | |
"id": 1, | |
"content": "<cls>", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
}, | |
{ | |
"id": 2, | |
"content": "<sep>", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
}, | |
{ | |
"id": 3, | |
"content": "<unk>", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
}, | |
{ | |
"id": 4, | |
"content": "<mask>", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
} | |
], | |
"normalizer": null, | |
"pre_tokenizer": null, | |
"post_processor": { | |
"type": "TemplateProcessing", | |
"single": [ | |
{ | |
"SpecialToken": { | |
"id": "<cls>", | |
"type_id": 0 | |
} | |
}, | |
{ | |
"Sequence": { | |
"id": "A", | |
"type_id": 0 | |
} | |
}, | |
{ | |
"SpecialToken": { | |
"id": "<sep>", | |
"type_id": 0 | |
} | |
} | |
], | |
"pair": [ | |
{ | |
"SpecialToken": { | |
"id": "<cls>", | |
"type_id": 0 | |
} | |
}, | |
{ | |
"Sequence": { | |
"id": "A", | |
"type_id": 0 | |
} | |
}, | |
{ | |
"SpecialToken": { | |
"id": "<sep>", | |
"type_id": 0 | |
} | |
}, | |
{ | |
"Sequence": { | |
"id": "B", | |
"type_id": 1 | |
} | |
}, | |
{ | |
"SpecialToken": { | |
"id": "<sep>", | |
"type_id": 1 | |
} | |
} | |
], | |
"special_tokens": { | |
"<cls>": { | |
"id": "<cls>", | |
"ids": [ | |
1 | |
], | |
"tokens": [ | |
"<cls>" | |
] | |
}, | |
"<sep>": { | |
"id": "<sep>", | |
"ids": [ | |
2 | |
], | |
"tokens": [ | |
"<sep>" | |
] | |
} | |
} | |
}, | |
"decoder": { | |
"type": "Metaspace", | |
"replacement": "▁", | |
"add_prefix_space": true | |
}, | |
"model": { | |
"type": "Unigram", | |
"unk_id": 3, | |
"vocab": [ | |
[ | |
"<pad>", | |
0.0 | |
], | |
[ | |
"<cls>", | |
0.0 | |
], | |
[ | |
"<sep>", | |
0.0 | |
], | |
[ | |
"<unk>", | |
0.0 | |
], | |
[ | |
"<mask>", | |
0.0 | |
], | |
[ | |
"K", | |
-2.8926598139924984 | |
], | |
[ | |
"P", | |
-2.937301100450547 | |
], | |
[ | |
"R", | |
-2.981133587027994 | |
], | |
[ | |
"N", | |
-2.984398973523902 | |
], | |
[ | |
"F", | |
-3.0405482512940942 | |
], | |
[ | |
"Q", | |
-3.0522167753465475 | |
], | |
[ | |
"T", | |
-3.057893521866589 | |
], | |
[ | |
"D", | |
-3.221572989674069 | |
], | |
[ | |
"V", | |
-3.242957729449966 | |
], | |
[ | |
"I", | |
-3.2639228157131353 | |
], | |
[ | |
"G", | |
-3.292222729720054 | |
], | |
[ | |
"Y", | |
-3.3242430893206976 | |
], | |
[ | |
"E", | |
-3.329998422834164 | |
], | |
[ | |
"A", | |
-3.3315100842234955 | |
], | |
[ | |
"L", | |
-3.354133164459414 | |
], | |
[ | |
"S", | |
-3.374619245946816 | |
], | |
[ | |
"H", | |
-3.6029119248880974 | |
], | |
[ | |
"M", | |
-3.6330880431340073 | |
], | |
[ | |
"C", | |
-4.03859206801765 | |
], | |
[ | |
"W", | |
-4.123791187360837 | |
], | |
[ | |
"AA", | |
-4.973167425197216 | |
], | |
[ | |
"LL", | |
-5.036049835340242 | |
], | |
[ | |
"LA", | |
-5.194188976627833 | |
], | |
[ | |
"LS", | |
-5.284020121682021 | |
], | |
[ | |
"AL", | |
-5.320469682162143 | |
], | |
[ | |
"SS", | |
-5.32453022570105 | |
], | |
[ | |
"SL", | |
-5.499862731096803 | |
], | |
[ | |
"LV", | |
-5.564795933199976 | |
], | |
[ | |
"AG", | |
-5.570161883025232 | |
], | |
[ | |
"GG", | |
-5.586688874755673 | |
], | |
[ | |
"AV", | |
-5.633866882153034 | |
], | |
[ | |
"LG", | |
-5.6617838822297415 | |
], | |
[ | |
"EL", | |
-5.710679099236735 | |
], | |
[ | |
"AS", | |
-5.716264001404388 | |
], | |
[ | |
"VL", | |
-5.7256437689651545 | |
], | |
[ | |
"SG", | |
-5.727475385041664 | |
], | |
[ | |
"LR", | |
-5.728549131116866 | |
], | |
[ | |
"LE", | |
-5.743482374609412 | |
], | |
[ | |
"GL", | |
-5.743495521291919 | |
], | |
[ | |
"SA", | |
-5.761398906433778 | |
], | |
[ | |
"GA", | |
-5.779501575037608 | |
], | |
[ | |
"EE", | |
-5.783251373522621 | |
], | |
[ | |
"RL", | |
-5.787802487132645 | |
], | |
[ | |
"VA", | |
-5.822397606320607 | |
], | |
[ | |
"TL", | |
-5.826633127578567 | |
], | |
[ | |
"GS", | |
-5.836663706756646 | |
], | |
[ | |
"EA", | |
-5.854012392063035 | |
], | |
[ | |
"TA", | |
-5.87623583133286 | |
], | |
[ | |
"LD", | |
-5.884057317313408 | |
], | |
[ | |
"VV", | |
-5.886122471861352 | |
], | |
[ | |
"LT", | |
-5.890501009220628 | |
], | |
[ | |
"IL", | |
-5.893641541215938 | |
], | |
[ | |
"DL", | |
-5.908589432452223 | |
], | |
[ | |
"SV", | |
-5.928508582466128 | |
], | |
[ | |
"LP", | |
-5.930607331306064 | |
], | |
[ | |
"GV", | |
-5.931353514901755 | |
], | |
[ | |
"IS", | |
-5.934577159636316 | |
], | |
[ | |
"RR", | |
-5.937181139038799 | |
], | |
[ | |
"PA", | |
-5.95165892093522 | |
], | |
[ | |
"LI", | |
-5.956444805023796 | |
], | |
[ | |
"DG", | |
-5.959872500616557 | |
], | |
[ | |
"LK", | |
-5.977419747714027 | |
], | |
[ | |
"AE", | |
-5.988346330589572 | |
], | |
[ | |
"VS", | |
-5.992644676581763 | |
], | |
[ | |
"TV", | |
-5.992771047038934 | |
], | |
[ | |
"TS", | |
-5.992862744926201 | |
], | |
[ | |
"DA", | |
-6.005980312620514 | |
], | |
[ | |
"TG", | |
-6.020054424800335 | |
], | |
[ | |
"IA", | |
-6.029275865888938 | |
], | |
[ | |
"PS", | |
-6.0558861763675385 | |
], | |
[ | |
"RA", | |
-6.091301969244897 | |
], | |
[ | |
"EK", | |
-6.143807832748726 | |
], | |
[ | |
"EV", | |
-6.144172113809702 | |
], | |
[ | |
"ST", | |
-6.1659183042917824 | |
], | |
[ | |
"DE", | |
-6.175007806118581 | |
], | |
[ | |
"SI", | |
-6.196141783532548 | |
], | |
[ | |
"IV", | |
-6.203941668151083 | |
], | |
[ | |
"DV", | |
-6.2062585412907065 | |
], | |
[ | |
"AD", | |
-6.214482574668974 | |
], | |
[ | |
"RS", | |
-6.219463683778585 | |
], | |
[ | |
"EI", | |
-6.2248807356059555 | |
], | |
[ | |
"GE", | |
-6.253174496738863 | |
], | |
[ | |
"IE", | |
-6.264016428440042 | |
], | |
[ | |
"ES", | |
-6.272022775598392 | |
], | |
[ | |
"SE", | |
-6.288530115689479 | |
], | |
[ | |
"IG", | |
-6.300749800545876 | |
], | |
[ | |
"SD", | |
-6.305378344633647 | |
], | |
[ | |
"VG", | |
-6.307254140261982 | |
], | |
[ | |
"VE", | |
-6.322491387400987 | |
], | |
[ | |
"GI", | |
-6.346224238884332 | |
], | |
[ | |
"X", | |
-7.588889207971681 | |
], | |
[ | |
"B", | |
-14.434103952435104 | |
], | |
[ | |
"Z", | |
-15.661068824545652 | |
], | |
[ | |
"U", | |
-16.34051994314413 | |
], | |
[ | |
"O", | |
-19.83562802135349 | |
] | |
] | |
} | |
} |