PrajwalS commited on
Commit
fad7288
1 Parent(s): cd8dbf9

add tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +1 -1
vocab.json CHANGED
@@ -1 +1 @@
1
- {"u": 0, "'": 1, "w": 2, "r": 3, "l": 4, "t": 5, "z": 6, "g": 7, "x": 8, "v": 9, "q": 10, "e": 11, "i": 12, "f": 13, "h": 14, "p": 15, "m": 16, "k": 17, "y": 18, "o": 19, "b": 20, "d": 21, "a": 22, "n": 23, "j": 24, "c": 25, "s": 26, "|": 27, "[UNK]": 28, "[PAD]": 29}
 
1
+ {"k": 0, "b": 1, "h": 3, "a": 4, "u": 5, "v": 6, "r": 7, "w": 8, "s": 9, "n": 10, "x": 11, "i": 12, "m": 13, "c": 14, "f": 15, "z": 16, "o": 17, "d": 18, "j": 19, "g": 20, "q": 21, "l": 22, "e": 23, "'": 24, "p": 25, "y": 26, "t": 27, "|": 2, "[UNK]": 28, "[PAD]": 29}