deep_bpe_100 / tokenizer.json
GinnM's picture
Upload tokenizer
76068e7
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "<pad>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "<cls>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "<sep>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "<unk>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 4,
"content": "<mask>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": null,
"post_processor": {
"type": "TemplateProcessing",
"single": [
{
"SpecialToken": {
"id": "<cls>",
"type_id": 0
}
},
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "<sep>",
"type_id": 0
}
}
],
"pair": [
{
"SpecialToken": {
"id": "<cls>",
"type_id": 0
}
},
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "<sep>",
"type_id": 0
}
},
{
"Sequence": {
"id": "B",
"type_id": 1
}
},
{
"SpecialToken": {
"id": "<sep>",
"type_id": 1
}
}
],
"special_tokens": {
"<cls>": {
"id": "<cls>",
"ids": [
1
],
"tokens": [
"<cls>"
]
},
"<sep>": {
"id": "<sep>",
"ids": [
2
],
"tokens": [
"<sep>"
]
}
}
},
"decoder": {
"type": "BPEDecoder",
"suffix": "</w>"
},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": null,
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"vocab": {
"<pad>": 0,
"<cls>": 1,
"<sep>": 2,
"<unk>": 3,
"<mask>": 4,
"A": 5,
"B": 6,
"C": 7,
"D": 8,
"E": 9,
"F": 10,
"G": 11,
"H": 12,
"I": 13,
"K": 14,
"L": 15,
"M": 16,
"N": 17,
"O": 18,
"P": 19,
"Q": 20,
"R": 21,
"S": 22,
"T": 23,
"U": 24,
"V": 25,
"W": 26,
"X": 27,
"Y": 28,
"Z": 29,
"AA": 30,
"LL": 31,
"SS": 32,
"LA": 33,
"LS": 34,
"GG": 35,
"LV": 36,
"LR": 37,
"LE": 38,
"LG": 39,
"LT": 40,
"AV": 41,
"AG": 42,
"LP": 43,
"LD": 44,
"LI": 45,
"AS": 46,
"EE": 47,
"RR": 48,
"LK": 49,
"VV": 50,
"AT": 51,
"AE": 52,
"SG": 53,
"DG": 54,
"AR": 55,
"KK": 56,
"AI": 57,
"AP": 58,
"TT": 59,
"LN": 60,
"LF": 61,
"LQ": 62,
"AD": 63,
"SV": 64,
"ST": 65,
"SI": 66,
"SP": 67,
"DE": 68,
"TG": 69,
"TV": 70,
"SR": 71,
"SE": 72,
"DV": 73,
"PP": 74,
"KE": 75,
"KI": 76,
"DI": 77,
"GV": 78,
"AQ": 79,
"AF": 80,
"RV": 81,
"TI": 82,
"NI": 83,
"PV": 84,
"RE": 85,
"SF": 86,
"KV": 87,
"NG": 88,
"LY": 89,
"RI": 90,
"PG": 91,
"RG": 92,
"DD": 93,
"PE": 94,
"AK": 95,
"NV": 96,
"SD": 97,
"TE": 98,
"QQ": 99
},
"merges": [
"A A",
"L L",
"S S",
"L A",
"L S",
"G G",
"L V",
"L R",
"L E",
"L G",
"L T",
"A V",
"A G",
"L P",
"L D",
"L I",
"A S",
"E E",
"R R",
"L K",
"V V",
"A T",
"A E",
"S G",
"D G",
"A R",
"K K",
"A I",
"A P",
"T T",
"L N",
"L F",
"L Q",
"A D",
"S V",
"S T",
"S I",
"S P",
"D E",
"T G",
"T V",
"S R",
"S E",
"D V",
"P P",
"K E",
"K I",
"D I",
"G V",
"A Q",
"A F",
"R V",
"T I",
"N I",
"P V",
"R E",
"S F",
"K V",
"N G",
"L Y",
"R I",
"P G",
"R G",
"D D",
"P E",
"A K",
"N V",
"S D",
"T E",
"Q Q"
]
}
}