paruwka's picture
Upload tokenizer
38662ed verified
raw
history blame
No virus
1.06 kB
{
"&": 80,
"'": 37,
"(": 54,
")": 8,
"=": 67,
"[PAD]": 89,
"[UNK]": 88,
"a": 74,
"b": 33,
"c": 82,
"d": 47,
"e": 49,
"f": 68,
"g": 58,
"h": 13,
"i": 63,
"j": 77,
"k": 61,
"l": 81,
"m": 24,
"n": 17,
"o": 12,
"p": 16,
"q": 69,
"r": 39,
"s": 20,
"t": 1,
"u": 53,
"v": 11,
"w": 29,
"x": 86,
"y": 27,
"z": 64,
"|": 48,
"«": 40,
"´": 10,
"»": 7,
"ß": 76,
"à": 25,
"á": 14,
"â": 44,
"ã": 79,
"ä": 15,
"å": 4,
"æ": 18,
"ç": 42,
"è": 71,
"é": 85,
"ê": 5,
"ë": 83,
"ì": 45,
"î": 35,
"ï": 46,
"ð": 22,
"ñ": 84,
"ó": 72,
"ô": 6,
"ö": 66,
"ø": 59,
"ù": 9,
"ú": 38,
"û": 78,
"ü": 31,
"þ": 23,
"ÿ": 26,
"ă": 3,
"ć": 87,
"č": 28,
"ē": 65,
"ğ": 52,
"ō": 73,
"œ": 56,
"ş": 62,
"š": 19,
"ū": 51,
"ž": 50,
"ș": 57,
"ʻ": 41,
"̀": 30,
"́": 36,
"̂": 70,
"̧": 75,
"–": 2,
"—": 43,
"’": 55,
"“": 34,
"”": 60,
"…": 21,
"の": 0,
"ひ": 32
}