add tokenizer

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

runs/Sep04_18-23-55_7f7b93ce1371/1662315917.2589/events.out.tfevents.1662315917.7f7b93ce1371.57.5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:855907fa4381351903fe6eea51fd05a6c93c67890f32723fbf120af44c21e03b
+size 4612

runs/Sep04_18-23-55_7f7b93ce1371/events.out.tfevents.1662315917.7f7b93ce1371.57.4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:362b8a23ded7e5e6b49d5045edb61b17e5eb99511f28ac9c9786074cd4bf8a26
+size 4268

vocab.json

@@ -1 +1 @@
-{"n": 0, "s": 1, "v": 2, "5": 3, "x": 4, "c": 5, "2": 6, "a": 7, "l": 8, "p": 9, "/": 10, "d": 11, "g": 12, "y": 13, "o": 14, "k": 15, "(": 16, "i": 17, ")": 18, "j": 20, "f": 21, "0": 22, "4": 23, "b": 24, "=": 25, "t": 26, "e": 27, "r": 28, "w": 29, "1": 30, "u": 31, "m": 32, "h": 33, "|": 19, "[UNK]": 34, "[PAD]": 35}
+{"w": 0, "b": 1, "4": 2, "u": 3, "y": 4, ")": 5, "v": 6, "x": 7, "i": 8, "s": 9, "a": 10, "f": 11, "2": 12, "k": 13, "j": 14, "n": 15, "t": 16, "0": 17, "o": 18, "1": 19, "(": 20, "m": 21, "d": 22, "r": 23, "h": 24, "5": 25, "c": 26, "e": 27, "p": 28, "g": 29, "=": 30, "l": 31, "/": 33, "|": 32, "[UNK]": 34, "[PAD]": 35}