anzorq's picture
Upload tokenizer
9bb67bb verified
raw
history blame
569 Bytes
{
"(": 1,
")": 2,
"1": 3,
"2": 4,
"3": 5,
"4": 6,
"6": 7,
"9": 8,
"[PAD]": 47,
"[UNK]": 46,
"|": 0,
"а": 9,
"б": 10,
"в": 11,
"г": 12,
"д": 13,
"е": 14,
"ж": 15,
"з": 16,
"и": 17,
"й": 18,
"к": 19,
"л": 20,
"м": 21,
"н": 22,
"о": 23,
"п": 24,
"р": 25,
"с": 26,
"т": 27,
"у": 28,
"ф": 29,
"х": 30,
"ц": 31,
"ч": 32,
"ш": 33,
"щ": 34,
"ъ": 35,
"ы": 36,
"ь": 37,
"э": 38,
"ю": 39,
"я": 40,
"ё": 41,
"ӏ": 42,
"–": 43,
"—": 44,
"…": 45
}