jcsilva commited on
Commit
d0b1504
1 Parent(s): 43e0f91

add tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +1 -1
vocab.json CHANGED
@@ -1 +1 @@
1
- {"y": 0, "g": 1, "f": 2, "w": 3, "u": 4, "i": 5, "o": 6, "x": 7, "t": 8, "q": 9, "'": 11, "m": 12, "l": 13, "s": 14, "k": 15, "p": 16, "r": 17, "b": 18, "e": 19, "h": 20, "j": 21, "c": 22, "a": 23, "d": 24, "z": 25, "v": 26, "n": 27, "|": 10, "[UNK]": 28, "[PAD]": 29}
 
1
+ {"k": 0, "r": 1, "s": 2, "a": 4, "m": 5, "z": 6, "g": 7, "x": 8, "w": 9, "e": 10, "h": 11, "d": 12, "q": 13, "n": 14, "'": 15, "c": 16, "o": 17, "l": 18, "t": 19, "f": 20, "y": 21, "p": 22, "i": 23, "u": 24, "b": 25, "j": 26, "v": 27, "|": 3, "[UNK]": 28, "[PAD]": 29}