yaswanth's picture
add tokenizer
9e63e40
{"ः": 0, "ल": 1, "उ": 2, "z": 3, "च": 4, "b": 5, "आ": 6, "ज़": 7, "ऋ": 8, "a": 9, "ग": 10, "ू": 11, "'": 12, "ऐ": 13, "ग़": 14, "े": 15, "इ": 16, "ै": 17, "ढ़": 18, "ौ": 19, "j": 20, "ब": 21, "घ": 22, "s": 23, "x": 24, "ि": 25, "ा": 26, "क": 27, "v": 28, "ऑ": 29, "r": 30, "ण": 31, "ञ": 32, "ड": 33, "t": 34, "c": 35, "त": 36, "श": 37, "थ": 38, "ॉ": 39, "ई": 40, "ॅ": 41, "र": 42, "y": 43, "व": 44, "्": 45, "p": 46, "m": 47, "ँ": 48, "य": 49, "ध": 50, "ं": 51, "ृ": 52, "ओ": 53, "l": 54, "भ": 55, "ो": 56, "ठ": 57, "i": 58, "ष": 59, "औ": 60, "ज": 61, "छ": 62, "स": 63, "ढ": 64, "म": 65, "अ": 66, "h": 67, "क़": 68, "ी": 69, "द": 70, "प": 71, "d": 72, "g": 73, "e": 74, "ड़": 75, "f": 76, "़": 77, "फ": 78, "ए": 79, "न": 80, "u": 81, "o": 83, "ह": 84, "n": 85, "ख": 86, "w": 87, "ु": 88, "ऊ": 89, "k": 90, "झ": 91, "ट": 92, "|": 82, "[UNK]": 93, "[PAD]": 94}