reichenbach's picture
add tokenizer
38519d1
{"&": 1, "'": 2, "-": 3, "a": 4, "b": 5, "c": 6, "d": 7, "e": 8, "f": 9, "g": 10, "h": 11, "i": 12, "j": 13, "k": 14, "l": 15, "m": 16, "n": 17, "o": 18, "p": 19, "r": 20, "s": 21, "t": 22, "u": 23, "v": 24, "w": 25, "x": 26, "y": 27, "z": 28, "|": 29, "ँ": 30, "ं": 31, "ः": 32, "अ": 33, "आ": 34, "इ": 35, "ई": 36, "उ": 37, "ऊ": 38, "ऋ": 39, "ए": 40, "ऐ": 41, "ऑ": 42, "ओ": 43, "औ": 44, "क": 45, "ख": 46, "ग": 47, "घ": 48, "च": 49, "छ": 50, "ज": 51, "झ": 52, "ञ": 53, "ट": 54, "ठ": 55, "ड": 56, "ढ": 57, "ण": 58, "त": 59, "थ": 60, "द": 61, "ध": 62, "न": 63, "प": 64, "फ": 65, "ब": 66, "भ": 67, "म": 68, "य": 69, "र": 70, "ल": 71, "व": 72, "श": 73, "ष": 74, "स": 75, "ह": 76, "़": 77, "ा": 78, "ि": 79, "ी": 80, "ु": 81, "ू": 82, "ृ": 83, "ॅ": 84, "े": 85, "ै": 86, "ॉ": 87, "ो": 88, "ौ": 89, "्": 90, "क़": 91, "ग़": 92, "ज़": 93, "ड़": 94, "ढ़": 95, "^": 0, "[UNK]": 96, "[PAD]": 97}