mprzibilla commited on
Commit
ec6488d
1 Parent(s): 172e503

add tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +1 -1
vocab.json CHANGED
@@ -1 +1 @@
1
- {"i": 0, "h": 1, "o": 2, "s": 3, "d": 4, "r": 5, "f": 6, "n": 7, "a": 8, "t": 9, "b": 10, "m": 11, "e": 12, " ": 13, "p": 14, "c": 15, "k": 16, "l": 17, "|": 13, "UNK": 19, "[PAD]": 20}
 
1
+ {"i": 0, "k": 1, "o": 2, "r": 3, "h": 4, "c": 5, "t": 6, "l": 7, "n": 8, "m": 9, "s": 10, "p": 11, "f": 12, "a": 13, "d": 14, "b": 15, "e": 16, " ": 17, "|": 17, "UNK": 19, "[PAD]": 20}