add tokenizer

vocab.json

@@ -1 +1 @@
-{"्": 0, "स": 1, "ग": 2, "ऊ": 3, "व": 4, "W": 5, "़": 6, "ः": 7, "ष": 8, "ई": 9, "थ": 10, "ब": 11, "श": 12, "प": 13, "ऐ": 14, "l": 15, "ठ": 16, "ल": 17, "च": 18, "F": 19, ":": 20, "औ": 21, "ु": 22, "र": 23, "ू": 24, "आ": 25, "झ": 26, "य": 27, "ँ": 28, "ओ": 29, "-": 30, "i": 31, "ज": 32, "r": 33, "?": 34, "द": 35, "!": 36, "ध": 37, "फ": 38, "छ": 39, "इ": 40, "a": 41, "ो": 42, "घ": 43, "भ": 44, "m": 45, "p": 46, "ॉ": 47, "।": 48, ",": 49, "ण": 50, "त": 51, "न": 52, "अ": 53, "\"": 54, "ड": 55, "'": 56, "ृ": 57, "े": 58, "ं": 59, ".": 60, "उ": 61, "ी": 62, "क": 63, "ि": 64, "ा": 65, "ड़": 66, "ै": 67, "ह": 68, "e": 69, "म": 70, "u": 71, "ए": 72, "ढ": 73, "ऑ": 74, "M": 75, "ौ": 76, "ट": 77, "ख": 79, "|": 78, "[UNK]": 80, "[PAD]": 81}
+{"द": 0, "l": 1, "।": 2, "!": 3, "p": 4, "प": 5, "ा": 6, "ी": 7, "ँ": 8, "झ": 9, "F": 10, "ग": 11, "े": 12, "य": 13, "ु": 14, "अ": 15, "आ": 16, "ऐ": 17, "ॉ": 18, "म": 19, "a": 20, "ब": 21, "उ": 22, "स": 23, "u": 24, "ष": 25, "ण": 26, ",": 27, "त": 28, "भ": 29, "ज": 30, "थ": 31, "़": 32, "ई": 33, "व": 34, ":": 35, "m": 36, "r": 37, "ठ": 38, ".": 39, "फ": 40, "ए": 41, "M": 42, "ओ": 43, "ऊ": 44, "न": 45, "ड़": 46, "क": 47, "्": 48, "ं": 49, "ऑ": 50, "ढ": 51, "इ": 52, "?": 53, "च": 54, "i": 55, "ै": 56, "ड": 57, "ह": 58, "e": 59, "घ": 60, "'": 61, "ल": 62, "औ": 63, "-": 64, "ि": 65, "ख": 66, "ू": 67, "\"": 68, "W": 69, "ध": 70, "ौ": 71, "ो": 72, "र": 73, "छ": 74, "ट": 75, "श": 76, "ृ": 77, "ः": 78, "|": 79, "[UNK]": 80, "[PAD]": 81}