add tokenizer

vocab.json

@@ -1 +1 @@
-{"h": 0, "ы": 1, "r": 2, "ʻ": 3, "ł": 4, "ת": 5, "p": 6, "ס": 7, "ã": 8, "т": 9, "ם": 10, "ź": 11, "í": 12, "x": 13, "ä": 15, "מ": 16, "—": 17, "'": 18, "н": 19, "m": 20, "ê": 21, "k": 22, "ü": 23, "ּ": 24, "ć": 25, "d": 26, "š": 27, "–": 28, "ó": 29, "ž": 30, "ו": 31, "u": 32, "t": 33, "l": 34, "ө": 35, "о": 36, "w": 37, "f": 38, "ø": 39, "o": 40, "נ": 41, "к": 42, "¿": 43, "ш": 44, "°": 45, "б": 46, "n": 47, "é": 48, "y": 49, "ú": 50, "ë": 51, "ö": 52, "ñ": 53, "毵": 54, "ל": 55, "ş": 56, "a": 57, "â": 58, "œ": 59, "ò": 60, "j": 61, "g": 62, "р": 63, "ő": 64, "ָ": 65, "z": 66, "ֹ": 67, "כ": 68, "e": 69, "ė": 70, "á": 71, "s": 72, "¡": 73, "ְ": 74, "q": 75, "v": 76, "и": 77, "п": 78, "ש": 79, "=": 80, "´": 81, "ð": 82, "ō": 83, "i": 84, "ə": 85, "ū": 86, "c": 87, "е": 88, "ô": 89, "а": 90, "ֵ": 91, "ү": 92, "ï": 93, "л": 94, "ě": 95, "ִ": 96, "b": 97, "’": 98, "|": 14, "[UNK]": 99, "[PAD]": 100}
+{"l": 0, "ã": 1, "¿": 2, "毵": 3, "m": 4, "j": 5, "z": 6, "g": 7, "ü": 8, "f": 9, "b": 10, "ə": 11, "ñ": 12, "á": 13, "o": 14, "°": 15, "–": 16, "d": 17, "k": 18, "¡": 19, "ū": 20, "é": 21, "u": 22, "ë": 23, "q": 24, "ú": 25, "í": 26, "i": 27, "w": 28, "t": 29, "v": 30, "h": 31, "=": 33, "ō": 34, "’": 35, "n": 36, "e": 37, "x": 38, "'": 39, "p": 40, "ö": 41, "ó": 42, "ş": 43, "s": 44, "r": 45, "a": 46, "c": 47, "y": 48, "|": 32, "[UNK]": 49, "[PAD]": 50}