add tokenizer

added_tokens.json

@@ -1 +1 @@
-{"<s>": 51, "</s>": 52}
+{"<s>": 47, "</s>": 48}

vocab.json

@@ -1 +1 @@
-{"`": 1, "a": 2, "b": 3, "c": 4, "d": 5, "e": 6, "f": 7, "g": 8, "h": 9, "i": 10, "j": 11, "k": 12, "l": 13, "m": 14, "n": 15, "o": 16, "p": 17, "q": 18, "r": 19, "s": 20, "t": 21, "u": 22, "v": 23, "w": 24, "x": 25, "y": 26, "z": 27, "³": 28, "á": 29, "ä": 30, "é": 31, "í": 32, "ó": 33, "ô": 34, "ö": 35, "ú": 36, "ý": 37, "č": 38, "ď": 39, "ĺ": 40, "ľ": 41, "ň": 42, "ŕ": 43, "š": 44, "ť": 45, "ž": 46, "̌": 47, "„": 48, "|": 0, "[UNK]": 49, "[PAD]": 50}
+{"c": 1, "i": 2, "o": 3, "y": 4, "«": 5, "»": 6, "а": 7, "б": 8, "в": 9, "г": 10, "д": 11, "е": 12, "ж": 13, "з": 14, "и": 15, "й": 16, "к": 17, "л": 18, "м": 19, "н": 20, "о": 21, "п": 22, "р": 23, "с": 24, "т": 25, "у": 26, "ф": 27, "х": 28, "ц": 29, "ч": 30, "ш": 31, "щ": 32, "ы": 33, "ь": 34, "э": 35, "ю": 36, "я": 37, "і": 38, "ј": 39, "ҕ": 40, "ҥ": 41, "ү": 42, "һ": 43, "ө": 44, "|": 0, "[UNK]": 45, "[PAD]": 46}