add tokenizer

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

runs/Jun01_08-22-35_075fda315076/1654071777.8526492/events.out.tfevents.1654071777.075fda315076.72.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41fc9fb2c96d0de7a8ee0b04f194b15cb931e70bef2b9647e662c8139b9de010
+size 4573

runs/Jun01_08-22-35_075fda315076/events.out.tfevents.1654071777.075fda315076.72.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4818a968e7f3bd18639daf92fb2264e05554d8d9915eec475cbc86a040fc3bf4
+size 4873

runs/Jun01_08-23-36_075fda315076/1654071827.2440298/events.out.tfevents.1654071827.075fda315076.72.3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a29675bf7bc7b13c2b0e1cfc02cb33a45f6e265f1d5d69c829096b995e32f559
+size 4573

runs/Jun01_08-23-36_075fda315076/events.out.tfevents.1654071827.075fda315076.72.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62962b13ef1e6f4afc6d6554091d168ec5909b2bdd39c5c7dac800f1480e3820
+size 4427