rafiulrumy's picture
add tokenizer
9632da4
{"र": 0, "प": 1, "झ": 2, "य": 3, "a": 4, "त": 5, "r": 6, "ौ": 7, "म": 8, "'": 9, "ट": 10, "ई": 11, "ह": 12, "ा": 13, "ण": 14, "f": 15, "u": 16, "ू": 17, "ध": 18, "उ": 19, "ि": 20, "ै": 21, "ष": 22, "भ": 23, "इ": 24, "ढ": 25, "क": 26, "ऑ": 28, "ठ": 29, "श": 30, "l": 31, "ल": 32, "e": 33, "आ": 34, "औ": 35, "ख": 36, "ओ": 37, "द": 38, "ु": 39, "ँ": 40, "ड": 41, "m": 42, "ॉ": 43, "घ": 44, "छ": 45, "न": 46, "थ": 47, "अ": 48, "p": 49, "ं": 50, "्": 51, "w": 52, "फ": 53, "ः": 54, "i": 55, "ऐ": 56, "ो": 57, "ए": 58, "ऊ": 59, "ड़": 60, "ज": 61, "े": 62, "ृ": 63, "।": 64, "स": 65, "च": 66, "ी": 67, "़": 68, "व": 69, "ब": 70, "ग": 71, "|": 27, "[UNK]": 72, "[PAD]": 73}