add tokenizer

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

added_tokens.json

@@ -1 +1 @@
-{"<s>": 67, "</s>": 68}
+{"<s>": 34, "</s>": 35}

vocab.json

@@ -1 +1 @@
-{"!": 1, ",": 2, "-": 3, ".": 4, ":": 5, ";": 6, "?": 7, "A": 8, "B": 9, "C": 10, "D": 11, "E": 12, "F": 13, "G": 14, "H": 15, "I": 16, "J": 17, "K": 18, "L": 19, "M": 20, "N": 21, "O": 22, "P": 23, "R": 24, "S": 25, "T": 26, "U": 27, "V": 28, "W": 29, "X": 30, "Z": 31, "a": 32, "b": 33, "c": 34, "d": 35, "e": 36, "f": 37, "g": 38, "h": 39, "i": 40, "j": 41, "k": 42, "l": 43, "m": 44, "n": 45, "o": 46, "p": 47, "r": 48, "s": 49, "t": 50, "u": 51, "v": 52, "w": 53, "y": 54, "z": 55, "«": 56, "»": 57, "Č": 58, "č": 59, "Š": 60, "š": 61, "Ž": 62, "ž": 63, "–": 64, "|": 0, "[UNK]": 65, "[PAD]": 66}
+{"a": 1, "b": 2, "c": 3, "d": 4, "e": 5, "f": 6, "g": 7, "h": 8, "i": 9, "j": 10, "k": 11, "l": 12, "m": 13, "n": 14, "o": 15, "p": 16, "r": 17, "s": 18, "t": 19, "u": 20, "v": 21, "w": 22, "x": 23, "y": 24, "z": 25, "«": 26, "»": 27, "č": 28, "š": 29, "ž": 30, "–": 31, "|": 0, "[UNK]": 32, "[PAD]": 33}