BauyrjanQ's picture
Upload tokenizer
13d0731 verified
{
"\t": 0,
"\n": 1,
"\r": 2,
"#": 4,
"&": 5,
"(": 6,
")": 7,
"+": 8,
"/": 9,
"0": 10,
"1": 11,
"2": 12,
"3": 13,
"4": 14,
"5": 15,
"6": 16,
"7": 17,
"8": 18,
"9": 19,
"[": 20,
"[PAD]": 122,
"[UNK]": 121,
"\\": 21,
"]": 22,
"_": 23,
"{": 24,
"|": 3,
"}": 25,
"~": 26,
" ": 27,
"¥": 28,
"©": 29,
"¬": 30,
"­": 31,
"·": 32,
"ë": 33,
"ŋ": 34,
"ɕ": 35,
"ə": 36,
"ɵ": 37,
"ʨ": 38,
"̀": 39,
"́": 40,
"̃": 41,
"̆": 42,
"̇": 43,
"̈": 44,
"̜": 45,
"̟": 46,
"̥": 47,
"̩": 48,
"̰": 49,
"̶": 50,
"̹": 51,
"υ": 52,
"а": 53,
"б": 54,
"в": 55,
"г": 56,
"д": 57,
"е": 58,
"ж": 59,
"з": 60,
"и": 61,
"й": 62,
"к": 63,
"л": 64,
"м": 65,
"н": 66,
"о": 67,
"п": 68,
"р": 69,
"с": 70,
"т": 71,
"у": 72,
"ф": 73,
"х": 74,
"ц": 75,
"ч": 76,
"ш": 77,
"щ": 78,
"ъ": 79,
"ы": 80,
"ь": 81,
"э": 82,
"ю": 83,
"я": 84,
"ё": 85,
"і": 86,
"ї": 87,
"ў": 88,
"ѳ": 89,
"ғ": 90,
"қ": 91,
"ң": 92,
"ү": 93,
"ұ": 94,
"һ": 95,
"ӊ": 96,
"ӏ": 97,
"ә": 98,
"ө": 99,
"​": 100,
"‎": 101,
"‐": 102,
"‒": 103,
"–": 104,
"—": 105,
"―": 106,
"’": 107,
"‚": 108,
"„": 109,
"…": 110,
"№": 111,
"−": 112,
"─": 113,
"●": 114,
"☺": 115,
"♥": 116,
"⠀": 117,
"": 118,
"": 119,
"-": 120
}