add tokenizer

runs/Apr04_16-37-05_76b1a7fb30b7/1680626241.56045/events.out.tfevents.1680626241.76b1a7fb30b7.1260.9

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55aa99e0fa09c8bacd0a97578d4b73e39319a8a396894b2a086cd797fe0bbdc8
+size 4786

runs/Apr04_16-37-05_76b1a7fb30b7/events.out.tfevents.1680626241.76b1a7fb30b7.1260.8

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd08061aa5108c23d2405766fb548c0a50edf7f43fa19f648bd616dbf7001017
+size 5153

vocab.json

@@ -1 +1 @@
-{"g": 0, "'": 1, "z": 2, "a": 3, "n": 4, "v": 5, "m": 6, "j": 7, "w": 8, "e": 9, "y": 11, "£": 12, "l": 13, "t": 14, "f": 15, "d": 16, "q": 17, "k": 18, "i": 19, "&": 20, "s": 21, "r": 22, "h": 23, "p": 24, "u": 25, "c": 26, "b": 27, "o": 28, "x": 29, "|": 10, "[UNK]": 30, "[PAD]": 31}
+{"g": 0, "z": 1, "a": 2, "n": 3, "v": 4, "m": 5, "j": 6, "w": 7, "e": 8, "y": 10, "£": 11, "l": 12, "t": 13, "f": 14, "d": 15, "q": 16, "k": 17, "i": 18, "&": 19, "s": 20, "r": 21, "h": 22, "p": 23, "u": 24, "c": 25, "b": 26, "o": 27, "x": 28, "|": 9, "[UNK]": 29, "[PAD]": 30}