wrice's picture
add tokenizer
c74a9a7
{"P": 0, "D": 1, "R": 2, "N": 3, "S": 4, ";": 5, "l": 6, "U": 7, "g": 8, "I": 9, "O": 10, "e": 11, "q": 12, "-": 13, "n": 14, "t": 15, "i": 16, "z": 17, "j": 18, "G": 19, "E": 20, "x": 21, "W": 22, ".": 23, "a": 24, "p": 25, "M": 26, "c": 27, "y": 28, "Y": 29, "d": 30, "h": 31, "?": 32, "C": 33, "w": 34, "v": 35, "k": 36, "H": 37, "F": 38, ",": 39, "B": 40, "b": 41, "u": 42, "Q": 43, "T": 44, "V": 45, "o": 46, "'": 47, "A": 48, "m": 49, "L": 50, "!": 51, "X": 53, "f": 54, "\"": 55, "J": 56, "s": 57, "K": 58, ":": 59, "r": 60, "|": 52, "[UNK]": 61, "[PAD]": 62, "<s>": 63, "</s>": 64}