yaswanth's picture
add tokenizer
d68a397
{"फ": 0, "y": 1, "व": 2, "झ": 3, "o": 4, "ड": 5, "ड़": 6, "v": 7, "ु": 8, "छ": 10, "य": 11, "ो": 12, "'": 13, "न": 14, "ऊ": 15, "ँ": 16, "m": 17, "ढ़": 18, "क़": 19, "स": 20, "ट": 21, "d": 22, "ई": 23, "उ": 24, "क": 25, "ण": 26, "ओ": 27, "z": 28, "h": 29, "ल": 30, "आ": 31, "ऐ": 32, "e": 33, "ष": 34, "द": 35, "श": 36, "b": 37, "ञ": 38, "p": 39, "्": 40, "l": 41, "u": 42, "g": 43, "ए": 44, "n": 45, "s": 46, "म": 47, "त": 48, "ज": 49, "ृ": 50, "ौ": 51, "ह": 52, "अ": 53, "ध": 54, "t": 55, "k": 56, "ॉ": 57, "ः": 58, "ऑ": 59, "ठ": 60, "थ": 61, "ॅ": 62, "र": 63, "ऋ": 64, "i": 65, "च": 66, "ै": 67, "f": 68, "प": 69, "ज़": 70, "a": 71, "j": 72, "ि": 73, "इ": 74, "ख": 75, "औ": 76, "w": 77, "ब": 78, "ग़": 79, "ढ": 80, "c": 81, "ं": 82, "े": 83, "भ": 84, "ग": 85, "r": 86, "ा": 87, "x": 88, "ू": 89, "ी": 90, "घ": 91, "़": 92, "|": 9, "[UNK]": 93, "[PAD]": 94}