add tokenizer

vocab.json

@@ -1 +1 @@
-{"w": 0, "ठ": 1, "र": 2, "अ": 3, "य": 4, "u": 5, "e": 6, "औ": 7, "भ": 8, "ण": 9, "ल": 10, "ग": 11, "ब": 12, "च": 13, "व": 14, "p": 15, "इ": 16, "ॉ": 17, "ड़": 18, "ख": 19, "ौ": 20, "ओ": 21, "i": 22, "न": 23, "'": 24, "ु": 25, "उ": 26, "घ": 27, "े": 28, "ट": 29, "ी": 30, "r": 31, "स": 32, "ध": 33, "ः": 35, "ष": 36, "m": 37, "छ": 38, "ि": 39, "ै": 40, "l": 41, "प": 42, "ज": 43, "ई": 44, "फ": 45, "झ": 46, "ऑ": 47, "त": 48, "ो": 49, "ऐ": 50, "ह": 51, "a": 52, "ा": 53, "्": 54, "ू": 55, "़": 56, "श": 57, "ँ": 58, "क": 59, "f": 60, "ं": 61, "ढ": 62, "ए": 63, "ड": 64, "ऊ": 65, "म": 66, "थ": 67, "आ": 68, "द": 69, "ृ": 70, "|": 34, "[UNK]": 71, "[PAD]": 72}
+{"ी": 0, "र": 1, "m": 2, "औ": 3, "च": 4, "a": 5, "प": 6, "ु": 7, "म": 8, "ा": 9, "इ": 10, "घ": 11, "ौ": 12, "त": 13, "स": 14, "l": 15, "ृ": 16, "आ": 17, "e": 18, "ड़": 19, "उ": 20, "द": 21, "ट": 22, "ऊ": 23, "छ": 24, "ं": 25, "्": 26, "ह": 27, "ि": 28, "क": 29, "ए": 30, "ख": 32, "'": 33, "ै": 34, "अ": 35, "श": 36, "ः": 37, "ग": 38, "ँ": 39, "ठ": 40, "ई": 41, "ढ": 42, "ब": 43, "p": 44, "ू": 45, "े": 46, "ध": 47, "भ": 48, "w": 49, "ऐ": 50, "ओ": 51, "य": 52, "ण": 53, "थ": 54, "f": 55, "ड": 56, "i": 57, "व": 58, "झ": 59, "r": 60, "ज": 61, "़": 62, "ष": 63, "फ": 64, "ॉ": 65, "ऑ": 66, "ो": 67, "न": 68, "u": 69, "ल": 70, "|": 31, "[UNK]": 71, "[PAD]": 72}