add tokenizer

vocab.json

@@ -1 +1 @@
-{"v": 0, ";": 1, "Y": 2, "W": 3, "f": 4, ",": 5, "M": 6, "L": 7, "m": 8, "q": 9, "e": 10, "p": 11, "F": 13, "j": 14, "N": 15, "z": 16, "-": 17, "?": 18, "A": 19, "S": 20, "'": 21, "G": 22, "x": 23, "B": 24, "n": 25, "U": 26, "Q": 27, "g": 28, "J": 29, "o": 30, ":": 31, "b": 32, "c": 33, ".": 34, "C": 35, "H": 36, "D": 37, "w": 38, "t": 39, "\"": 40, "K": 41, "X": 42, "u": 43, "I": 44, "T": 45, "y": 46, "d": 47, "l": 48, "O": 49, "i": 50, "r": 51, "E": 52, "k": 53, "R": 54, "s": 55, "h": 56, "P": 57, "V": 58, "a": 59, "!": 60, "|": 12, "[UNK]": 61, "[PAD]": 62}
+{"g": 0, "s": 1, "V": 2, "c": 3, "?": 4, ",": 5, "K": 6, ":": 7, "p": 8, "h": 9, "j": 10, "t": 11, ";": 12, "u": 13, "k": 14, "Y": 15, "y": 16, "S": 17, "W": 18, "M": 19, "J": 20, "N": 21, "L": 22, "f": 23, "B": 24, "O": 25, "!": 26, "o": 27, "a": 28, "D": 29, "w": 30, "X": 31, "R": 32, "l": 33, "i": 34, ".": 35, "m": 36, "\"": 37, "d": 38, "T": 39, "q": 40, "b": 41, "n": 42, "G": 43, "z": 44, "F": 45, "P": 46, "H": 47, "e": 48, "I": 49, "Q": 50, "-": 51, "C": 52, "x": 53, "'": 54, "A": 55, "v": 56, "E": 57, "U": 58, "r": 59, "|": 60, "[UNK]": 61, "[PAD]": 62}