add tokenizer

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

runs/Nov24_01-40-42_843fb2fe0f9e/1637718132.0757496/events.out.tfevents.1637718132.843fb2fe0f9e.73.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d09a2dc72653626f86e3e2cb6ebad3d610f431d416a74ee1ea921e3aff4b0624
+size 4567

runs/Nov24_01-40-42_843fb2fe0f9e/events.out.tfevents.1637718132.843fb2fe0f9e.73.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8a313d93119a25155ad9c49873376c4e45570c2754746876454716c2d1c64d8
+size 4613

runs/Nov24_02-10-07_843fb2fe0f9e/1637719887.863317/events.out.tfevents.1637719887.843fb2fe0f9e.73.3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dba7f1f7b6c663d9435265d9a630c06b052a5c75e8df80661c4742805a684d22
+size 4567

runs/Nov24_02-10-07_843fb2fe0f9e/events.out.tfevents.1637719887.843fb2fe0f9e.73.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:374205a2ed67329ebae1e55396fd57360dfd68fb4b0b19e64655624856ec07e3
+size 4613

vocab.json

@@ -1 +1 @@
-{"ay": 1, "p": 2, "aw": 3, "t": 4, "ae": 5, "uh": 6, "hh": 7, "f": 8, "m": 9, "s": 10, "eh": 11, "k": 12, "g": 13, "zh": 14, "ey": 15, "ah": 16, "y": 17, "v": 18, "n": 19, "iy": 20, "z": 21, "ih": 22, "ch": 23, "er": 24, "aa": 25, "b": 26, "th": 27, "r": 28, "l": 29, "sh": 30, "jh": 31, "dh": 32, "uw": 33, "ng": 34, "ow": 35, "oy": 36, "d": 37, "w": 38, "|": 0, "[UNK]": 39, "[PAD]": 40}
+{"sh": 1, "th": 2, "er": 3, "n": 4, "v": 5, "p": 6, "jh": 7, "d": 8, "b": 9, "z": 10, "ow": 11, "g": 12, "uw": 13, "hh": 14, "ah": 15, "iy": 16, "ih": 17, "f": 18, "l": 19, "zh": 20, "y": 21, "eh": 22, "s": 23, "k": 24, "uh": 25, "m": 26, "aa": 27, "ch": 28, "oy": 29, "w": 30, "ae": 31, "ey": 32, "r": 33, "aw": 34, "ay": 35, "ng": 36, "t": 37, "dh": 38, "|": 0, "[UNK]": 39, "[PAD]": 40}