Upload tokenizer
4edd597
|
{ |
|
" ": 16, |
|
"!": 5, |
|
"\"": 11, |
|
"'": 176, |
|
",": 3, |
|
".": 4, |
|
":": 2, |
|
";": 1, |
|
"<pad>": 177, |
|
"<unk>": 178, |
|
"?": 6, |
|
"A": 17, |
|
"B": 18, |
|
"C": 19, |
|
"D": 20, |
|
"E": 21, |
|
"F": 22, |
|
"G": 23, |
|
"H": 24, |
|
"I": 25, |
|
"J": 26, |
|
"K": 27, |
|
"L": 28, |
|
"M": 29, |
|
"N": 30, |
|
"O": 31, |
|
"P": 32, |
|
"Q": 33, |
|
"R": 34, |
|
"S": 35, |
|
"T": 36, |
|
"U": 37, |
|
"V": 38, |
|
"W": 39, |
|
"X": 40, |
|
"Y": 41, |
|
"Z": 42, |
|
"_": 0, |
|
"a": 43, |
|
"b": 44, |
|
"c": 45, |
|
"d": 46, |
|
"e": 47, |
|
"f": 48, |
|
"g": 49, |
|
"h": 50, |
|
"i": 51, |
|
"j": 52, |
|
"k": 53, |
|
"l": 54, |
|
"m": 55, |
|
"n": 56, |
|
"o": 57, |
|
"p": 58, |
|
"q": 59, |
|
"r": 60, |
|
"s": 61, |
|
"t": 62, |
|
"u": 63, |
|
"v": 64, |
|
"w": 65, |
|
"x": 66, |
|
"y": 67, |
|
"z": 68, |
|
"¡": 7, |
|
"«": 12, |
|
"»": 13, |
|
"¿": 8, |
|
"æ": 72, |
|
"ç": 78, |
|
"ð": 81, |
|
"ø": 116, |
|
"ħ": 98, |
|
"ŋ": 112, |
|
"œ": 120, |
|
"ǀ": 152, |
|
"ǁ": 153, |
|
"ǂ": 154, |
|
"ǃ": 155, |
|
"ɐ": 70, |
|
"ɑ": 69, |
|
"ɒ": 71, |
|
"ɓ": 73, |
|
"ɔ": 76, |
|
"ɕ": 77, |
|
"ɖ": 80, |
|
"ɗ": 79, |
|
"ɘ": 84, |
|
"ə": 83, |
|
"ɚ": 85, |
|
"ɛ": 86, |
|
"ɜ": 87, |
|
"ɝ": 88, |
|
"ɞ": 89, |
|
"ɟ": 90, |
|
"ɠ": 93, |
|
"ɡ": 92, |
|
"ɢ": 94, |
|
"ɣ": 139, |
|
"ɤ": 140, |
|
"ɥ": 99, |
|
"ɦ": 96, |
|
"ɧ": 97, |
|
"ɨ": 101, |
|
"ɪ": 102, |
|
"ɫ": 106, |
|
"ɬ": 105, |
|
"ɭ": 104, |
|
"ɮ": 107, |
|
"ɯ": 110, |
|
"ɰ": 111, |
|
"ɱ": 109, |
|
"ɲ": 114, |
|
"ɳ": 113, |
|
"ɴ": 115, |
|
"ɵ": 117, |
|
"ɶ": 121, |
|
"ɸ": 118, |
|
"ɹ": 123, |
|
"ɺ": 124, |
|
"ɻ": 126, |
|
"ɽ": 129, |
|
"ɾ": 125, |
|
"ʀ": 127, |
|
"ʁ": 128, |
|
"ʂ": 130, |
|
"ʃ": 131, |
|
"ʄ": 91, |
|
"ʈ": 132, |
|
"ʉ": 134, |
|
"ʊ": 135, |
|
"ʋ": 136, |
|
"ʌ": 138, |
|
"ʍ": 141, |
|
"ʎ": 143, |
|
"ʏ": 144, |
|
"ʐ": 146, |
|
"ʑ": 145, |
|
"ʒ": 147, |
|
"ʔ": 148, |
|
"ʕ": 150, |
|
"ʘ": 122, |
|
"ʙ": 74, |
|
"ʛ": 95, |
|
"ʜ": 100, |
|
"ʝ": 103, |
|
"ʟ": 108, |
|
"ʡ": 149, |
|
"ʢ": 151, |
|
"ʤ": 82, |
|
"ʧ": 133, |
|
"ʰ": 162, |
|
"ʱ": 163, |
|
"ʲ": 164, |
|
"ʴ": 161, |
|
"ʷ": 165, |
|
"ʼ": 160, |
|
"ˈ": 156, |
|
"ˌ": 157, |
|
"ː": 158, |
|
"ˑ": 159, |
|
"˞": 168, |
|
"ˠ": 166, |
|
"ˤ": 167, |
|
"̩": 175, |
|
"β": 75, |
|
"θ": 119, |
|
"χ": 142, |
|
"ᵻ": 177, |
|
"—": 9, |
|
"“": 14, |
|
"”": 15, |
|
"…": 10, |
|
"↑": 170, |
|
"→": 171, |
|
"↓": 169, |
|
"↗": 172, |
|
"↘": 173, |
|
"ⱱ": 137 |
|
} |
|
|
