add tokenizer

added_tokens.json

@@ -1 +1 @@
-{"<s>": 68, "</s>": 69}
+{"<s>": 33, "</s>": 34}

vocab.json

@@ -1 +1 @@
-{"!": 1, "\"": 2, "'": 3, ",": 4, "-": 5, ".": 6, ":": 7, ";": 8, "?": 9, "A": 10, "B": 11, "C": 12, "D": 13, "E": 14, "F": 15, "G": 16, "H": 17, "I": 18, "J": 19, "K": 20, "L": 21, "M": 22, "N": 23, "O": 24, "P": 25, "Q": 26, "R": 27, "S": 28, "T": 29, "U": 30, "V": 31, "W": 32, "Y": 33, "Z": 34, "a": 35, "b": 36, "c": 37, "d": 38, "e": 39, "f": 40, "g": 41, "h": 42, "i": 43, "j": 44, "k": 45, "l": 46, "m": 47, "n": 48, "o": 49, "p": 50, "q": 51, "r": 52, "s": 53, "t": 54, "u": 55, "v": 56, "w": 57, "x": 58, "y": 59, "z": 60, "·": 61, "é": 62, "ö": 63, "“": 64, "”": 65, "|": 0, "[UNK]": 66, "[PAD]": 67}
+{"a": 1, "b": 2, "c": 3, "d": 4, "e": 5, "f": 6, "g": 7, "h": 8, "i": 9, "j": 10, "k": 11, "l": 12, "m": 13, "n": 14, "o": 15, "p": 16, "q": 17, "r": 18, "s": 19, "t": 20, "u": 21, "v": 22, "w": 23, "x": 24, "y": 25, "z": 26, "á": 27, "é": 28, "ł": 29, "ń": 30, "|": 0, "[UNK]": 31, "[PAD]": 32}