add tokenizer

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

runs/Nov08_05-03-37_bc3fe9087fdc/1667884207.2054553/events.out.tfevents.1667884207.bc3fe9087fdc.429.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:180a2343c44bfa78b14944ec8db08f81c0da06525895e705053fbd43526c7e01
+size 4567

runs/Nov08_05-03-37_bc3fe9087fdc/1667884578.5979955/events.out.tfevents.1667884578.bc3fe9087fdc.429.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd3690bbe7b1643ac666805335ce1f355e94bfeffd1b86a9195e0a270de98b2f
+size 4567

runs/Nov08_05-03-37_bc3fe9087fdc/1667884694.5535138/events.out.tfevents.1667884694.bc3fe9087fdc.429.3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afbceb5ab3cac4d27b5016e3d1b7556c7c19329a75e0f009d86d579da985d7e4
+size 4567

runs/Nov08_05-03-37_bc3fe9087fdc/events.out.tfevents.1667884207.bc3fe9087fdc.429.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:633b5e815b68a439e0e6b518cdfc808c902ed854e2a7805a5f593c37f4d93265
+size 12500