add tokenizer

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

runs/Dec06_13-11-36_nvu72klxfx/1670332392.1904578/events.out.tfevents.1670332392.nvu72klxfx.145.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d24ffcec9ce20cc2e7cc46e87be323a9934052d12c1fa15167e3596ed24f6da5
+size 4520

runs/Dec06_13-11-36_nvu72klxfx/events.out.tfevents.1670332392.nvu72klxfx.145.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98a74e661d3a894fad56fb1f9caaf7dee344ca69acd3898e834e3ffd94186430
+size 4425

runs/Dec06_13-14-38_nvu72klxfx/1670332577.1337323/events.out.tfevents.1670332577.nvu72klxfx.145.3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4cd4934145d0a60257111352f9ee5681d1c9dfd0af0167b696d21e579441707
+size 4520

runs/Dec06_13-14-38_nvu72klxfx/events.out.tfevents.1670332577.nvu72klxfx.145.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef9eeb12c9aa865e8decc3473ba66fef61f9bbcae9168b0e3bf783f5347b1282
+size 40