ToToKr's picture
add tokenizer
447fde3
{"b": 0, "v": 1, "k": 2, "p": 3, "q": 4, "n": 5, "h": 6, "s": 7, "i": 8, "x": 9, "l": 10, "m": 11, "r": 12, "\t": 13, "j": 14, "'": 15, "o": 16, "t": 17, "d": 19, "y": 20, "`": 21, "e": 22, "u": 23, "f": 24, "w": 25, "a": 26, "z": 27, "c": 28, "g": 29, "|": 18, "[UNK]": 30, "[PAD]": 31}