add tokenizer

vocab.json

@@ -1 +1 @@
-{"श": 0, "र": 1, "ऋ": 2, "ऊ": 3, "इ": 4, "च": 5, "ी": 6, "ध": 7, "ड": 8, "ब": 9, "द": 10, "म": 11, "ह": 12, "ग": 13, "ा": 14, "ः": 15, "आ": 16, "‌": 17, "थ": 18, "झ": 19, "ृ": 20, "ौ": 21, "न": 22, "८": 23, "‍": 24, "ऐ": 25, "्": 26, "ँ": 27, "ॠ": 28, "ढ": 29, "ङ": 30, "े": 32, "प": 33, "ै": 34, "अ": 35, "उ": 36, "स": 37, "ो": 38, "ू": 39, "भ": 40, "क": 41, "ओ": 42, "ए": 43, "ु": 44, "त": 45, "य": 46, "फ": 47, "औ": 48, "ि": 49, "ं": 50, "छ": 51, "ख": 52, "व": 53, "ज": 54, "ञ": 55, "घ": 56, "ट": 57, "ठ": 58, "ष": 59, "ल": 60, "ई": 61, "ण": 62, "|": 31, "[UNK]": 63, "[PAD]": 64}
+{"ञ": 0, "ए": 1, "ग": 2, "औ": 3, "्": 4, "ओ": 5, "ु": 6, "ष": 7, "ज": 8, "प": 9, "य": 10, "भ": 11, "म": 12, "‌": 13, "थ": 14, "ो": 15, "ः": 16, "इ": 17, "ी": 18, "आ": 19, "छ": 20, "र": 21, "त": 22, "ौ": 23, "व": 24, "ि": 25, "ू": 26, "े": 27, "घ": 28, "अ": 29, "‍": 31, "ह": 32, "द": 33, "ख": 34, "फ": 35, "क": 36, "ङ": 37, "झ": 38, "स": 39, "ठ": 40, "ड": 41, "ै": 42, "ा": 43, "ल": 44, "न": 45, "ढ": 46, "ृ": 47, "श": 48, "ब": 49, "च": 50, "ं": 51, "ँ": 52, "ण": 53, "ध": 54, "ई": 55, "उ": 56, "ट": 57, "|": 30, "[UNK]": 58, "[PAD]": 59}