202015004's picture
add tokenizer
3d456ca
{"h": 0, "q": 1, "z": 2, "i": 3, "a": 5, "w": 6, "u": 7, "j": 8, "e": 9, "'": 10, "l": 11, "v": 12, "s": 13, "x": 14, "p": 15, "r": 16, "k": 17, "y": 18, "m": 19, "b": 20, "d": 21, "t": 22, "n": 23, "f": 24, "c": 25, "o": 26, "g": 27, "|": 4, "[UNK]": 28, "[PAD]": 29}