Trifon's picture
add tokenizer
4615038
raw
history blame
2.12 kB
{"Ƅ": 0, "ŏ": 1, "ľ": 2, "ç": 3, "Ø": 4, "ų": 5, "ů": 6, "ź": 7, "Ķ": 8, "Ę": 9, "ū": 10, "Ģ": 11, "È": 12, "Ğ": 13, "Ē": 14, "ŀ": 15, "é": 16, "Ũ": 17, "Ý": 18, "ė": 19, "ij": 20, "IJ": 21, "ƀ": 22, "Ą": 23, "ž": 24, "ä": 25, "Ė": 26, "ĸ": 27, "Ŵ": 28, "Ƃ": 29, "Î": 30, "Ţ": 31, "Ņ": 32, "Þ": 33, "ģ": 34, "Ŷ": 35, "Ɔ": 36, "à": 37, "Ł": 38, "Ÿ": 39, "Ľ": 40, "ĝ": 41, "å": 42, "Ĺ": 43, "ŵ": 44, "ō": 45, "Ų": 46, "ß": 47, "Ŕ": 48, "á": 49, "ÿ": 50, "ś": 51, "Œ": 52, "ř": 53, "Ċ": 54, "Ö": 55, "ü": 56, "Ť": 57, "ũ": 58, "ń": 59, "Ƈ": 60, "ĭ": 61, "ħ": 62, "ĩ": 63, "Š": 64, "ġ": 65, "Ù": 66, "ı": 67, "ŕ": 68, "Ò": 69, "Ǝ": 70, "ù": 71, "Ā": 72, "ñ": 73, "ý": 74, "É": 75, "ŷ": 76, "ĉ": 77, "į": 78, "Ļ": 79, "ƍ": 80, "Ƌ": 81, "Ū": 82, "Ú": 83, "Ŀ": 84, "ę": 85, "ŋ": 86, "Ë": 87, "ķ": 88, "ú": 89, "Ɗ": 90, "Í": 91, "Ŗ": 92, "â": 93, "î": 94, "ſ": 95, "Ô": 96, "Ń": 97, "Ć": 98, "ó": 99, "ã": 100, "ţ": 101, "ŧ": 102, "ƈ": 103, "Ì": 104, "ð": 105, "ö": 106, "ğ": 107, "Ñ": 108, "Ĉ": 109, "ŝ": 110, "ƌ": 111, "Õ": 112, "Ġ": 113, "Č": 114, "ļ": 115, "ĺ": 116, "ě": 117, "Ű": 118, "Ɓ": 119, "ĵ": 120, "š": 121, "ē": 122, "Ê": 123, "ć": 124, "×": 125, "Ĕ": 126, "Ŭ": 127, "Ď": 128, "Ě": 129, "í": 130, "ĕ": 131, "Ă": 132, "Ś": 133, "ő": 134, "ê": 135, "ŗ": 136, "Ə": 137, "ë": 138, "ĥ": 139, "Ň": 140, "Į": 141, "û": 142, "÷": 143, "Đ": 144, "Ī": 145, "Ž": 146, "đ": 147, "Û": 148, "Ź": 149, "ą": 150, "Ï": 151, "Ĩ": 152, "è": 153, "č": 154, "ť": 155, "Ó": 156, "Ĵ": 157, "œ": 158, "ø": 159, "ş": 160, "Ż": 161, "Ŏ": 162, "ò": 163, "Ĭ": 164, "ƅ": 165, "þ": 166, "ċ": 167, "Ɖ": 168, "æ": 169, "Ŝ": 170, "ŭ": 171, "Ĝ": 172, "Ĥ": 173, "Ŧ": 174, "ű": 175, "ƃ": 176, "ď": 177, "ʼn": 178, "ż": 179, "Ő": 180, "ă": 181, "Ð": 182, "Ů": 183, "ï": 184, "ô": 185, "Ř": 186, "ī": 187, "ň": 188, "Ō": 189, "Ħ": 190, "Ŋ": 191, "İ": 192, "ņ": 193, "ā": 194, "Ş": 195, "ì": 196, "Ü": 197, "õ": 198, "ł": 199, "[UNK]": 200, "[PAD]": 201}