mprzibilla commited on
Commit
1760ada
1 Parent(s): 0aa868c

add tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +1 -1
vocab.json CHANGED
@@ -1 +1 @@
1
- {"i": 0, "b": 1, "t": 2, "d": 3, "v": 4, "l": 5, "g": 6, "a": 7, "y": 8, "f": 9, "s": 10, "k": 11, "o": 12, "e": 13, "n": 14, "z": 15, "q": 16, "x": 17, "p": 19, "j": 20, "r": 21, "w": 22, "m": 23, "h": 24, "u": 25, "c": 26, "|": 18, "[UNK]": 27, "[PAD]": 28}
 
1
+ {"u": 0, "l": 1, "w": 3, "k": 4, "d": 5, "n": 6, "m": 7, "x": 8, "j": 9, "t": 10, "o": 11, "y": 12, "g": 13, "e": 14, "a": 15, "b": 16, "p": 17, "c": 18, "v": 19, "s": 20, "i": 21, "q": 22, "h": 23, "r": 24, "f": 25, "|": 2, "[UNK]": 26, "[PAD]": 27}