mprzibilla commited on
Commit
5406dbb
1 Parent(s): 876c701

add tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +1 -1
vocab.json CHANGED
@@ -1 +1 @@
1
- {"x": 0, "v": 1, "o": 2, "e": 3, "f": 4, "k": 5, "y": 6, "m": 7, "s": 8, "b": 9, "n": 10, "p": 11, "i": 13, "l": 14, "d": 15, "h": 16, "u": 17, "t": 18, "c": 19, "g": 20, "w": 21, "a": 22, "q": 23, "r": 24, "j": 25, "|": 12, "[UNK]": 26, "[PAD]": 27}
 
1
+ {"h": 0, "b": 1, "r": 2, "x": 3, "p": 4, "t": 5, "j": 6, "o": 7, "m": 8, "k": 9, "v": 10, "n": 11, "s": 12, "w": 13, "y": 15, "g": 16, "c": 17, "f": 18, "e": 19, "q": 20, "u": 21, "l": 22, "i": 23, "a": 24, "d": 25, "|": 14, "[UNK]": 26, "[PAD]": 27}