Upload tokenizer
004ff58
verified
|
{
|
|
" ": 16,
|
|
"!": 5,
|
|
"\"": 11,
|
|
"'": 176,
|
|
",": 3,
|
|
".": 4,
|
|
":": 2,
|
|
";": 1,
|
|
"?": 6,
|
|
"A": 17,
|
|
"B": 18,
|
|
"C": 19,
|
|
"D": 20,
|
|
"E": 21,
|
|
"F": 22,
|
|
"G": 23,
|
|
"H": 24,
|
|
"I": 25,
|
|
"J": 26,
|
|
"K": 27,
|
|
"L": 28,
|
|
"M": 29,
|
|
"N": 30,
|
|
"O": 31,
|
|
"P": 32,
|
|
"Q": 33,
|
|
"R": 34,
|
|
"S": 35,
|
|
"T": 36,
|
|
"U": 37,
|
|
"V": 38,
|
|
"W": 39,
|
|
"X": 40,
|
|
"Y": 41,
|
|
"Z": 42,
|
|
"_": 0,
|
|
"a": 43,
|
|
"b": 44,
|
|
"c": 45,
|
|
"d": 46,
|
|
"e": 47,
|
|
"f": 48,
|
|
"g": 49,
|
|
"h": 50,
|
|
"i": 51,
|
|
"j": 52,
|
|
"k": 53,
|
|
"l": 54,
|
|
"m": 55,
|
|
"n": 56,
|
|
"o": 57,
|
|
"p": 58,
|
|
"q": 59,
|
|
"r": 60,
|
|
"s": 61,
|
|
"t": 62,
|
|
"u": 63,
|
|
"v": 64,
|
|
"w": 65,
|
|
"x": 66,
|
|
"y": 67,
|
|
"z": 68,
|
|
"¡": 7,
|
|
"«": 12,
|
|
"»": 13,
|
|
"¿": 8,
|
|
"æ": 72,
|
|
"ç": 78,
|
|
"ð": 81,
|
|
"ø": 116,
|
|
"ħ": 98,
|
|
"ŋ": 112,
|
|
"œ": 120,
|
|
"ǀ": 152,
|
|
"ǁ": 153,
|
|
"ǂ": 154,
|
|
"ǃ": 155,
|
|
"ɐ": 70,
|
|
"ɑ": 69,
|
|
"ɒ": 71,
|
|
"ɓ": 73,
|
|
"ɔ": 76,
|
|
"ɕ": 77,
|
|
"ɖ": 80,
|
|
"ɗ": 79,
|
|
"ɘ": 84,
|
|
"ə": 83,
|
|
"ɚ": 85,
|
|
"ɛ": 86,
|
|
"ɜ": 87,
|
|
"ɝ": 88,
|
|
"ɞ": 89,
|
|
"ɟ": 90,
|
|
"ɠ": 93,
|
|
"ɡ": 92,
|
|
"ɢ": 94,
|
|
"ɣ": 139,
|
|
"ɤ": 140,
|
|
"ɥ": 99,
|
|
"ɦ": 96,
|
|
"ɧ": 97,
|
|
"ɨ": 101,
|
|
"ɪ": 102,
|
|
"ɫ": 106,
|
|
"ɬ": 105,
|
|
"ɭ": 104,
|
|
"ɮ": 107,
|
|
"ɯ": 110,
|
|
"ɰ": 111,
|
|
"ɱ": 109,
|
|
"ɲ": 114,
|
|
"ɳ": 113,
|
|
"ɴ": 115,
|
|
"ɵ": 117,
|
|
"ɶ": 121,
|
|
"ɸ": 118,
|
|
"ɹ": 123,
|
|
"ɺ": 124,
|
|
"ɻ": 126,
|
|
"ɽ": 129,
|
|
"ɾ": 125,
|
|
"ʀ": 127,
|
|
"ʁ": 128,
|
|
"ʂ": 130,
|
|
"ʃ": 131,
|
|
"ʄ": 91,
|
|
"ʈ": 132,
|
|
"ʉ": 134,
|
|
"ʊ": 135,
|
|
"ʋ": 136,
|
|
"ʌ": 138,
|
|
"ʍ": 141,
|
|
"ʎ": 143,
|
|
"ʏ": 144,
|
|
"ʐ": 146,
|
|
"ʑ": 145,
|
|
"ʒ": 147,
|
|
"ʔ": 148,
|
|
"ʕ": 150,
|
|
"ʘ": 122,
|
|
"ʙ": 74,
|
|
"ʛ": 95,
|
|
"ʜ": 100,
|
|
"ʝ": 103,
|
|
"ʟ": 108,
|
|
"ʡ": 149,
|
|
"ʢ": 151,
|
|
"ʤ": 82,
|
|
"ʧ": 133,
|
|
"ʰ": 162,
|
|
"ʱ": 163,
|
|
"ʲ": 164,
|
|
"ʴ": 161,
|
|
"ʷ": 165,
|
|
"ʼ": 160,
|
|
"ˈ": 156,
|
|
"ˌ": 157,
|
|
"ː": 158,
|
|
"ˑ": 159,
|
|
"˞": 168,
|
|
"ˠ": 166,
|
|
"ˤ": 167,
|
|
"̩": 175,
|
|
"β": 75,
|
|
"θ": 119,
|
|
"χ": 142,
|
|
"ᵻ": 177,
|
|
"—": 9,
|
|
"“": 14,
|
|
"”": 15,
|
|
"…": 10,
|
|
"↑": 170,
|
|
"→": 171,
|
|
"↓": 169,
|
|
"↗": 172,
|
|
"↘": 173,
|
|
"ⱱ": 137
|
|
}
|
|
|