add tokenizer

vocab.json

@@ -1 +1 @@
-{"ी": 0, "ं": 1, "ै": 2, "अ": 3, "ृ": 4, "ँ": 5, "ठ": 6, "म": 7, "ध": 8, "f": 9, "थ": 10, "ग": 11, "ष": 12, "ो": 13, "ए": 14, "ऑ": 15, "र": 16, "p": 17, "ु": 18, "ा": 19, "ौ": 20, "न": 21, "ि": 22, "त": 23, "झ": 24, "आ": 25, "ः": 26, "e": 27, "ओ": 28, "u": 29, "ब": 30, "w": 31, "्": 32, "m": 33, "r": 34, "व": 35, "इ": 36, "े": 37, "औ": 38, "।": 39, "ू": 41, "i": 42, "उ": 43, "ड़": 44, "य": 45, "प": 46, "श": 47, "ढ": 48, "a": 49, "क": 50, "ल": 51, "स": 52, "ॉ": 53, "भ": 54, "ह": 55, "l": 56, "च": 57, "ई": 58, "'": 59, "द": 60, "ण": 61, "छ": 62, "ड": 63, "ऊ": 64, "ख": 65, "ऐ": 66, "़": 67, "ट": 68, "फ": 69, "ज": 70, "घ": 71, "|": 40, "[UNK]": 72, "[PAD]": 73}
+{"त": 0, "प": 1, "झ": 2, "p": 3, "ं": 4, "ख": 5, "ढ": 6, "r": 7, "a": 8, "e": 9, "म": 10, "श": 11, "ऐ": 12, "इ": 13, "ण": 14, "ठ": 15, "ई": 16, "ओ": 17, "अ": 18, "ँ": 19, "स": 20, "'": 21, "ू": 22, "क": 23, "ॉ": 24, "ए": 25, "ष": 26, "ऑ": 27, "w": 29, "ि": 30, "भ": 31, "उ": 32, "े": 33, "च": 34, "घ": 35, "ड": 36, "छ": 37, "ल": 38, "य": 39, "थ": 40, "।": 41, "f": 42, "ु": 43, "ः": 44, "व": 45, "न": 46, "द": 47, "ड़": 48, "ा": 49, "ह": 50, "ौ": 51, "u": 52, "m": 53, "ब": 54, "ृ": 55, "ग": 56, "ी": 57, "ऊ": 58, "्": 59, "र": 60, "i": 61, "औ": 62, "l": 63, "फ": 64, "ज": 65, "ट": 66, "़": 67, "ो": 68, "ै": 69, "आ": 70, "ध": 71, "|": 28}