add tokenizer

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

runs/Feb02_07-39-40_3e1d9e601274/1643787681.9722958/events.out.tfevents.1643787681.3e1d9e601274.392.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf08ce507dc4cb55be3a0d5f5a60adcf6923de8b36046baa1c786000737a17d4
+size 4534

runs/Feb02_07-39-40_3e1d9e601274/events.out.tfevents.1643787681.3e1d9e601274.392.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:637599d6c8d57d90a3baedc7d376d6036a26354e45330fa44174bb00147680b2
+size 4443

runs/Feb02_07-46-39_3e1d9e601274/1643788038.9581788/events.out.tfevents.1643788038.3e1d9e601274.392.3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d26ed2196c69aba46b748cfe00ad7e3b52a0105b47ff57f73f5d10f3bb565733
+size 4534

runs/Feb02_07-46-39_3e1d9e601274/events.out.tfevents.1643788038.3e1d9e601274.392.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:755ea83597483493161b3bcdce0335b843359ac7cfc2245453ae5dc0ef838a01
+size 4443

runs/Feb02_07-59-39_3e1d9e601274/1643788811.715984/events.out.tfevents.1643788811.3e1d9e601274.392.5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:258f16da3450141d926c1c01d9a6c3909fb077f5e71f7908cdaef689bacf1163
+size 4534

runs/Feb02_07-59-39_3e1d9e601274/events.out.tfevents.1643788811.3e1d9e601274.392.4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50ff4cb464b0010ba5a839b547f19b279844daf6ef432621fb87788b87eb14ac
+size 4442

runs/Feb02_08-06-34_3e1d9e601274/1643789228.465631/events.out.tfevents.1643789228.3e1d9e601274.392.7

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8720a73c65f99296ec3878584fccfe3364a3e16fc7d929e19aea24a62fc1c793
+size 4534

runs/Feb02_08-06-34_3e1d9e601274/events.out.tfevents.1643789228.3e1d9e601274.392.6

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86b847c360f86a91c76ee01e85cc892b59e7009c549569da7fab9f4044c4d073
+size 4442

runs/Feb02_08-09-44_3e1d9e601274/1643789411.8119798/events.out.tfevents.1643789411.3e1d9e601274.392.9

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8414dd5f9b6adf06e3debcdc1ff0ba13f78a0fff1675bb59be3eb497cce38897
+size 4534

runs/Feb02_08-09-44_3e1d9e601274/events.out.tfevents.1643789411.3e1d9e601274.392.8

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:832835923b16664c7d33b3fa66693b8cf53a0622e31574095c88f9ac8cd4d215
+size 4442

runs/Feb02_08-11-27_3e1d9e601274/1643789514.3955863/events.out.tfevents.1643789514.3e1d9e601274.392.11

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc657f0081abd3ef9b487b87a265deff287d6261c47301d8199b1be05b0e93a5
+size 4534

runs/Feb02_08-11-27_3e1d9e601274/events.out.tfevents.1643789514.3e1d9e601274.392.10

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfb03c38b254528d4f8a66aafdd37229cd0e9b268059fa1bec60ee39613486bc
+size 4442

vocab.json

@@ -1 +1 @@
-{"n": 0, "y": 1, "p": 2, "q": 3, "t": 4, "m": 5, "d": 6, "j": 7, "e": 8, "x": 9, "_": 10, "o": 11, "l": 12, "u": 13, "g": 14, "v": 15, "a": 16, "7": 18, "h": 19, "f": 20, "s": 21, "i": 22, "z": 23, "r": 24, "k": 25, "b": 26, "|": 17, "[UNK]": 27, "[PAD]": 28}
+{"n": 0, "o": 1, "j": 2, "b": 3, "7": 4, "h": 5, "f": 6, "_": 7, "u": 8, "x": 9, "i": 10, "z": 11, "s": 13, "y": 14, "v": 15, "p": 16, "d": 17, "q": 18, "l": 19, "a": 20, "r": 21, "m": 22, "e": 23, "t": 24, "g": 25, "k": 26, "|": 12, "[UNK]": 27, "[PAD]": 28}