add tokenizer

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

runs/Nov25_06-40-53_932797b7b350/1637822547.034949/events.out.tfevents.1637822547.932797b7b350.83.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:549422aeca4fc262ca2d32487388ddc322a672caca3728e99cafd0019aa239bb
+size 4519

runs/Nov25_06-40-53_932797b7b350/events.out.tfevents.1637822547.932797b7b350.83.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1116d5d3091d92e364210e764d5eca2ffc607d043c1b52459a128592924cfedc
+size 4426

runs/Nov25_06-43-45_932797b7b350/1637822654.3984928/events.out.tfevents.1637822654.932797b7b350.83.3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4637c5a6d0f71337bacb9a53f511cbee5627093f531254c805480c800e2cfee4
+size 4519

runs/Nov25_06-43-45_932797b7b350/events.out.tfevents.1637822654.932797b7b350.83.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddfb4600d410f7f9601403b18385db1c30a0d88ab48b1050fb495a86b9ada223
+size 4426

vocab.json

@@ -1 +1 @@
-{"n": 1, "b": 2, "i": 3, "e": 4, "v": 5, "k": 6, "t": 7, "f": 8, "d": 9, "h": 10, "p": 11, "a": 12, "j": 13, "c": 14, "z": 15, "o": 16, "y": 17, "r": 18, "s": 19, "l": 20, "q": 21, "w": 22, "m": 23, "x": 24, "g": 25, "u": 26, "|": 0, "[UNK]": 27, "[PAD]": 28}
+{"z": 0, "y": 1, "w": 2, "u": 3, "t": 4, "f": 5, "l": 6, "b": 7, "a": 8, "q": 9, "g": 10, "h": 11, "c": 12, "j": 13, "v": 14, "s": 16, "d": 17, "o": 18, "e": 19, "p": 20, "x": 21, "k": 22, "r": 23, "m": 24, "i": 25, "n": 26, "|": 15, "[UNK]": 27, "[PAD]": 28}