add tokenizer

vocab.json

@@ -1 +1 @@
-{"ऑ": 0, "a": 1, "ढ": 2, "च": 3, "ह": 4, "ा": 5, "ल": 6, "अ": 7, "औ": 8, "ठ": 9, "र": 10, "ऐ": 11, "आ": 12, "ए": 13, "ओ": 14, "ौ": 15, "फ": 16, "ड": 17, "थ": 18, "ू": 19, "भ": 20, "e": 21, "ँ": 23, "ृ": 24, "ु": 25, "ि": 26, "w": 27, "ी": 28, "य": 29, "म": 30, "छ": 31, "े": 32, "l": 33, "स": 34, "ब": 35, "उ": 36, "।": 37, "इ": 38, "घ": 39, "m": 40, "श": 41, "व": 42, "ष": 43, "द": 44, "्": 45, "f": 46, "ड़": 47, "r": 48, "ः": 49, "ज": 50, "p": 51, "ग": 52, "i": 53, "ट": 54, "़": 55, "'": 56, "ई": 57, "ख": 58, "ध": 59, "त": 60, "ं": 61, "ो": 62, "ॉ": 63, "ै": 64, "क": 65, "ण": 66, "न": 67, "u": 68, "ऊ": 69, "प": 70, "झ": 71, "|": 22, "[UNK]": 72, "[PAD]": 73}
+{"ः": 0, "ओ": 1, "व": 2, "r": 3, "।": 4, "ै": 5, "ग": 6, "ु": 7, "ी": 8, "उ": 9, "न": 10, "ख": 11, "i": 12, "ई": 13, "त": 14, "ज": 15, "ृ": 16, "़": 17, "ू": 18, "ष": 19, "झ": 20, "m": 21, "f": 22, "र": 23, "आ": 24, "क": 25, "ि": 26, "स": 27, "च": 28, "p": 29, "ठ": 30, "छ": 31, "ह": 32, "e": 33, "ऐ": 35, "ल": 36, "w": 37, "द": 38, "ड": 39, "ऊ": 40, "ण": 41, "य": 42, "अ": 43, "ो": 44, "घ": 45, "ड़": 46, "्": 47, "म": 48, "थ": 49, "a": 50, "े": 51, "l": 52, "ढ": 53, "ँ": 54, "ध": 55, "ऑ": 56, "ौ": 57, "ए": 58, "ॉ": 59, "u": 60, "ा": 61, "'": 62, "ट": 63, "प": 64, "इ": 65, "भ": 66, "ं": 67, "ब": 68, "औ": 69, "फ": 70, "श": 71, "|": 34, "[UNK]": 72, "[PAD]": 73}