reach-vb's picture
reach-vb HF staff
add tokenizer
57fdacb
raw history blame
No virus
611 Bytes
{"!": 1, ",": 2, "-": 3, ".": 4, ":": 5, ";": 6, "?": 7, "A": 8, "B": 9, "C": 10, "D": 11, "E": 12, "F": 13, "G": 14, "H": 15, "I": 16, "J": 17, "K": 18, "L": 19, "M": 20, "N": 21, "O": 22, "P": 23, "R": 24, "S": 25, "T": 26, "U": 27, "V": 28, "W": 29, "X": 30, "Z": 31, "a": 32, "b": 33, "c": 34, "d": 35, "e": 36, "f": 37, "g": 38, "h": 39, "i": 40, "j": 41, "k": 42, "l": 43, "m": 44, "n": 45, "o": 46, "p": 47, "r": 48, "s": 49, "t": 50, "u": 51, "v": 52, "w": 53, "y": 54, "z": 55, "«": 56, "»": 57, "Č": 58, "č": 59, "Š": 60, "š": 61, "Ž": 62, "ž": 63, "–": 64, "|": 0, "[UNK]": 65, "[PAD]": 66}