rafiulrumy's picture
add tokenizer
60e688f
{"ु": 0, "ड़": 1, "ह": 2, "ि": 3, "न": 4, "श": 5, "a": 6, "ौ": 7, "w": 8, "ृ": 9, "ी": 10, "ै": 11, "ख": 12, "ॉ": 13, "इ": 14, "ट": 15, "p": 16, "े": 17, "ष": 18, "ा": 19, "औ": 20, "भ": 21, "म": 22, "अ": 23, "ँ": 24, "र": 25, "ऊ": 26, "ध": 27, "झ": 28, "व": 29, "य": 30, "u": 31, "ल": 32, "ड": 33, "द": 34, "्": 35, "क": 36, "ण": 37, "ए": 38, "ढ": 39, "ो": 40, "f": 41, "i": 42, "प": 43, "स": 44, "थ": 45, "r": 46, "ऑ": 47, "ई": 48, "त": 49, "ठ": 50, "छ": 51, "ं": 52, "m": 53, "आ": 54, "घ": 55, "l": 56, "उ": 57, "ऐ": 58, "ः": 59, "।": 60, "़": 61, "फ": 62, "ज": 63, "ू": 64, "ब": 65, "e": 67, "ग": 68, "च": 69, "ओ": 70, "|": 66, "[UNK]": 71, "[PAD]": 72}