clvp_dev / vocab.json
susnato's picture
Uploaded new saved tokenizer
75a9688
raw
history blame
3.34 kB
{
"!": 3,
"'": 4,
"(": 5,
")": 6,
",": 7,
"-": 8,
".": 9,
"/": 10,
":": 11,
";": 12,
"?": 13,
"[SPACE]": 2,
"[STOP]": 0,
"[UNK]": 1,
"a": 14,
"ab": 109,
"about": 215,
"ac": 77,
"ace": 238,
"ack": 157,
"ad": 68,
"ag": 143,
"ain": 137,
"ak": 213,
"ake": 245,
"al": 56,
"all": 108,
"ally": 195,
"am": 122,
"an": 43,
"and": 53,
"ant": 159,
"any": 229,
"ap": 155,
"ar": 59,
"ard": 177,
"are": 127,
"as": 55,
"ast": 233,
"at": 48,
"ate": 186,
"ation": 158,
"ay": 88,
"b": 15,
"back": 241,
"be": 67,
"been": 237,
"bl": 251,
"bo": 165,
"br": 243,
"but": 128,
"by": 185,
"c": 16,
"ca": 183,
"can": 201,
"ce": 117,
"ch": 71,
"ck": 197,
"co": 174,
"com": 170,
"con": 135,
"could": 221,
"d": 17,
"de": 126,
"der": 234,
"did": 235,
"do": 134,
"e": 18,
"ed": 49,
"el": 249,
"em": 225,
"en": 50,
"end": 204,
"ent": 86,
"er": 44,
"ere": 138,
"ers": 173,
"es": 61,
"ess": 164,
"est": 218,
"ex": 169,
"f": 19,
"fe": 132,
"fo": 182,
"for": 87,
"fr": 130,
"from": 154,
"g": 20,
"ge": 112,
"get": 254,
"gh": 82,
"ght": 104,
"go": 119,
"gr": 198,
"h": 21,
"ha": 102,
"had": 116,
"have": 148,
"he": 62,
"her": 92,
"him": 156,
"his": 94,
"i": 22,
"ic": 91,
"id": 83,
"ide": 208,
"ie": 236,
"if": 140,
"ight": 196,
"il": 151,
"ill": 216,
"im": 106,
"in": 41,
"ind": 168,
"ine": 203,
"ing": 52,
"ink": 239,
"into": 228,
"ion": 107,
"ir": 98,
"is": 54,
"it": 60,
"ity": 240,
"j": 23,
"just": 226,
"k": 24,
"ke": 93,
"know": 209,
"l": 25,
"ld": 79,
"le": 64,
"li": 75,
"like": 162,
"ll": 84,
"lo": 99,
"ly": 70,
"m": 26,
"man": 176,
"me": 80,
"mo": 115,
"more": 244,
"my": 125,
"n": 27,
"ne": 111,
"no": 95,
"not": 149,
"now": 145,
"o": 28,
"od": 207,
"of": 58,
"ok": 166,
"om": 65,
"ome": 129,
"on": 47,
"one": 110,
"ong": 222,
"op": 161,
"or": 57,
"other": 227,
"ou": 45,
"ough": 202,
"ould": 113,
"oun": 146,
"ound": 189,
"our": 160,
"ous": 206,
"out": 105,
"over": 253,
"ow": 69,
"own": 190,
"p": 29,
"pe": 124,
"per": 205,
"pl": 178,
"po": 139,
"pp": 246,
"pro": 187,
"q": 30,
"qu": 194,
"r": 31,
"re": 46,
"red": 219,
"res": 175,
"ri": 101,
"ro": 89,
"s": 32,
"sa": 150,
"said": 252,
"se": 66,
"sh": 120,
"she": 118,
"si": 212,
"so": 123,
"some": 171,
"sp": 248,
"st": 63,
"su": 133,
"t": 33,
"te": 136,
"ted": 188,
"ter": 114,
"th": 40,
"that": 73,
"the": 42,
"their": 224,
"them": 217,
"then": 247,
"ther": 131,
"there": 172,
"they": 141,
"thing": 153,
"this": 147,
"ti": 78,
"ting": 242,
"tion": 181,
"to": 51,
"tr": 144,
"ts": 192,
"tw": 232,
"ty": 210,
"u": 34,
"ul": 167,
"um": 231,
"un": 97,
"up": 152,
"ur": 121,
"us": 142,
"use": 250,
"ust": 163,
"ut": 96,
"v": 35,
"ve": 76,
"ven": 200,
"ver": 90,
"very": 211,
"w": 36,
"was": 81,
"way": 180,
"we": 100,
"were": 184,
"wh": 72,
"what": 193,
"when": 199,
"whi": 230,
"who": 214,
"wi": 85,
"with": 103,
"wor": 179,
"would": 191,
"x": 37,
"y": 38,
"ye": 220,
"you": 74,
"your": 223,
"z": 39
}