mprzibilla commited on
Commit
bb56df9
1 Parent(s): 51e5c95

add tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +1 -1
vocab.json CHANGED
@@ -1 +1 @@
1
- {"o": 0, "b": 1, "a": 2, "g": 3, "n": 4, "y": 5, "s": 6, "q": 7, "h": 8, "l": 9, "k": 10, "j": 11, "v": 12, "w": 13, "r": 14, "p": 15, "i": 16, "f": 17, "m": 18, "c": 19, "e": 20, "d": 21, "u": 22, "x": 24, "t": 25, "|": 23, "[UNK]": 26, "[PAD]": 27}
 
1
+ {"n": 0, "p": 1, "a": 2, "s": 3, "x": 4, "k": 5, "b": 6, "g": 7, "m": 8, "c": 9, "y": 10, "f": 11, "w": 12, "e": 13, "q": 15, "i": 16, "r": 17, "t": 18, "v": 19, "j": 20, "d": 21, "u": 22, "l": 23, "h": 24, "o": 25, "|": 14, "[UNK]": 26, "[PAD]": 27}