nvshubhsharma's picture
add tokenizer
3517a17
{"ई": 0, "थ": 1, "म": 3, "ॉ": 4, "ठ": 6, "ण": 7, "ी": 8, "त": 9, "प": 10, "ल": 11, "ै": 12, "य": 13, "ं": 14, "ख": 15, "घ": 16, "ट": 18, "ः": 21, "छ": 22, "आ": 23, "स": 24, "ौ": 26, "ग": 27, "ऑ": 28, "च": 29, "'": 30, "न": 31, "र": 34, "ए": 35, "द": 36, "ड़": 37, "क": 38, "ब": 40, "्": 41, "भ": 42, "़": 44, "झ": 45, "ह": 46, "ड": 47, "व": 48, "ओ": 49, "ढ": 50, "ँ": 52, "अ": 53, "औ": 54, "श": 55, "इ": 56, "ज": 57, "फ": 58, "उ": 59, "ध": 60, "ऊ": 61, "ो": 62, "ृ": 63, "ा": 64, "ु": 65, "ि": 66, "ू": 67, "े": 68, "ऐ": 69, "ष": 70, "|": 51, "[UNK]": 61, "[PAD]": 62}