emre's picture
add tokenizer
2835684
{"n": 0, "y": 1, "e": 2, "̇": 3, ")": 4, "h": 5, "(": 6, "p": 7, "x": 8, "i": 9, "…": 10, "'": 11, "t": 12, "r": 13, "b": 14, "j": 15, "ü": 16, "â": 17, "ç": 18, "q": 20, "v": 21, "k": 22, "ë": 23, "f": 24, "m": 25, "g": 26, "w": 27, "î": 28, "d": 29, "s": 30, "o": 31, "ş": 32, "u": 33, "z": 34, "û": 35, "a": 36, "ğ": 37, "’": 38, "ö": 39, "c": 40, "l": 41, "ı": 42, "é": 43, "|": 19, "[UNK]": 44, "[PAD]": 45}