wrice's picture
add tokenizer
f5abe6d
raw
history blame
566 Bytes
{"g": 0, "s": 1, "V": 2, "c": 3, "?": 4, ",": 5, "K": 6, ":": 7, "p": 8, "h": 9, "j": 10, "t": 11, ";": 12, "u": 13, "k": 14, "Y": 15, "y": 16, "S": 17, "W": 18, "M": 19, "J": 20, "N": 21, "L": 22, "f": 23, "B": 24, "O": 25, "!": 26, "o": 27, "a": 28, "D": 29, "w": 30, "X": 31, "R": 32, "l": 33, "i": 34, ".": 35, "m": 36, "\"": 37, "d": 38, "T": 39, "q": 40, "b": 41, "n": 42, "G": 43, "z": 44, "F": 45, "P": 46, "H": 47, "e": 48, "I": 49, "Q": 50, "-": 51, "C": 52, "x": 53, "'": 54, "A": 55, "v": 56, "E": 57, "U": 58, "r": 59, "|": 60, "[UNK]": 61, "[PAD]": 62}