Jubliano's picture
Upload tokenizer
34d5fa0 verified
raw
history blame
1.62 kB
{
" ": 110,
"(": 99,
")": 95,
".": 93,
"2": 107,
"4": 109,
"5": 104,
"6": 105,
"[PAD]": 134,
"[UNK]": 133,
"a": 84,
"b": 1,
"c": 6,
"d": 3,
"e": 69,
"f": 29,
"g": 9,
"h": 47,
"i": 60,
"j": 54,
"k": 8,
"l": 56,
"m": 14,
"n": 16,
"o": 74,
"p": 0,
"q": 10,
"r": 22,
"s": 33,
"t": 2,
"u": 65,
"v": 30,
"w": 100,
"x": 41,
"y": 61,
"z": 34,
"|": 132,
"ä": 98,
"æ": 82,
"ç": 39,
"ð": 32,
"ø": 70,
"ħ": 45,
"ĩ": 131,
"ŋ": 19,
"œ": 77,
"ũ": 130,
"ɐ": 83,
"ɑ": 86,
"ɒ": 87,
"ɔ": 81,
"ɕ": 94,
"ɖ": 5,
"ɗ": 106,
"ɘ": 71,
"ə": 75,
"ɚ": 96,
"ɛ": 76,
"ɜ": 78,
"ɞ": 79,
"ɟ": 7,
"ɡ": 102,
"ɢ": 11,
"ɣ": 42,
"ɤ": 73,
"ɦ": 48,
"ɨ": 62,
"ɪ": 66,
"ɫ": 108,
"ɬ": 49,
"ɭ": 57,
"ɮ": 50,
"ɯ": 64,
"ɰ": 55,
"ɱ": 15,
"ɲ": 18,
"ɳ": 17,
"ɴ": 20,
"ɵ": 72,
"ɶ": 85,
"ɸ": 27,
"ɹ": 52,
"ɻ": 53,
"ɽ": 26,
"ɾ": 25,
"ʀ": 23,
"ʁ": 44,
"ʂ": 37,
"ʃ": 35,
"ʈ": 4,
"ʉ": 63,
"ʊ": 68,
"ʋ": 51,
"ʌ": 80,
"ʎ": 58,
"ʏ": 67,
"ʐ": 38,
"ʑ": 103,
"ʒ": 36,
"ʔ": 13,
"ʕ": 46,
"ʙ": 21,
"ʝ": 40,
"ʟ": 59,
"ʡ": 12,
"ʰ": 88,
"ʲ": 90,
"ʷ": 89,
"ː": 97,
"ˠ": 91,
"ˤ": 92,
"̃": 129,
"̈": 118,
"̊": 112,
"̘": 124,
"̙": 125,
"̜": 115,
"̝": 123,
"̞": 122,
"̟": 116,
"̠": 117,
"̤": 126,
"̥": 111,
"̩": 120,
"̬": 113,
"̯": 121,
"̰": 127,
"̹": 114,
"̼": 128,
"̽": 119,
"β": 28,
"θ": 31,
"χ": 43,
"ᵝ": 101,
"ⱱ": 24
}