rafiulrumy's picture
add tokenizer
b01774a
{"ः": 0, "ओ": 1, "व": 2, "r": 3, "।": 4, "ै": 5, "ग": 6, "ु": 7, "ी": 8, "उ": 9, "न": 10, "ख": 11, "i": 12, "ई": 13, "त": 14, "ज": 15, "ृ": 16, "़": 17, "ू": 18, "ष": 19, "झ": 20, "m": 21, "f": 22, "र": 23, "आ": 24, "क": 25, "ि": 26, "स": 27, "च": 28, "p": 29, "ठ": 30, "छ": 31, "ह": 32, "e": 33, "ऐ": 35, "ल": 36, "w": 37, "द": 38, "ड": 39, "ऊ": 40, "ण": 41, "य": 42, "अ": 43, "ो": 44, "घ": 45, "ड़": 46, "्": 47, "म": 48, "थ": 49, "a": 50, "े": 51, "l": 52, "ढ": 53, "ँ": 54, "ध": 55, "ऑ": 56, "ौ": 57, "ए": 58, "ॉ": 59, "u": 60, "ा": 61, "'": 62, "ट": 63, "प": 64, "इ": 65, "भ": 66, "ं": 67, "ब": 68, "औ": 69, "फ": 70, "श": 71, "|": 34, "[UNK]": 72, "[PAD]": 73}