add tokenizer

vocab.json

@@ -1 +1 @@
-{"ऊ": 0, "g": 1, "ए": 2, "फ": 3, "क": 4, "श": 5, "f": 6, "t": 7, "व": 8, "क़": 9, "c": 10, "ट": 11, "ष": 12, "v": 13, "झ": 14, "ल": 15, "आ": 16, "j": 17, "ह": 18, "ृ": 19, "l": 20, "z": 21, "ू": 22, "a": 23, "अ": 24, "s": 25, "ॅ": 26, "ज़": 27, "o": 28, "ः": 29, "n": 30, "k": 31, "र": 32, "x": 33, "्": 34, "य": 35, "u": 36, "i": 37, "ढ": 38, "ा": 39, "r": 40, "इ": 41, "p": 42, "छ": 43, "ॉ": 44, "थ": 45, "ओ": 46, "े": 47, "च": 48, "ब": 49, "ऑ": 50, "m": 52, "ठ": 53, "ि": 54, "ख": 55, "ं": 56, "ु": 57, "ो": 58, "ौ": 59, "y": 60, "उ": 61, "न": 62, "ज": 63, "ण": 64, "ड़": 65, "w": 66, "ञ": 67, "द": 68, "त": 69, "e": 70, "ध": 71, "'": 72, "़": 73, "ग": 74, "म": 75, "ऋ": 76, "ी": 77, "भ": 78, "प": 79, "d": 80, "ै": 81, "ऐ": 82, "ग़": 83, "घ": 84, "ई": 85, "ँ": 86, "स": 87, "h": 88, "औ": 89, "ढ़": 90, "b": 91, "ड": 92, "|": 51, "[UNK]": 93, "[PAD]": 94}
+{"ॅ": 0, "र": 1, "अ": 2, "z": 3, "ौ": 4, "आ": 5, "य": 6, "m": 7, "क": 8, "ण": 9, "्": 10, "c": 11, "ञ": 12, "ज": 13, "द": 14, "व": 15, "ै": 16, "ँ": 17, "ं": 18, "v": 19, "श": 20, "ऐ": 21, "प": 22, "ई": 23, "ओ": 24, "त": 25, "w": 26, "स": 27, "y": 28, "d": 29, "ल": 30, "k": 31, "े": 32, "s": 33, "थ": 34, "ड": 35, "फ": 36, "ु": 37, "i": 38, "ा": 40, "ः": 41, "ऊ": 42, "u": 43, "ए": 44, "औ": 45, "e": 46, "j": 47, "म": 48, "t": 49, "ढ": 50, "ड़": 51, "ी": 52, "ट": 53, "न": 54, "उ": 55, "ध": 56, "ह": 57, "़": 58, "ढ़": 59, "इ": 60, "ू": 61, "भ": 62, "झ": 63, "ृ": 64, "f": 65, "x": 66, "ऑ": 67, "ब": 68, "ष": 69, "ॉ": 70, "l": 71, "छ": 72, "a": 73, "च": 74, "h": 75, "r": 76, "ऋ": 77, "ज़": 78, "ग़": 79, "ग": 80, "ि": 81, "ख": 82, "क़": 83, "'": 84, "n": 85, "o": 86, "b": 87, "g": 88, "p": 89, "ठ": 90, "घ": 91, "ो": 92, "|": 39, "[UNK]": 93, "[PAD]": 94}