add tokenizer

added_tokens.json

@@ -1 +1 @@
-{"<s>": 97, "</s>": 98}
+{"<s>": 98, "</s>": 99}

vocab.json

@@ -1 +1 @@
-{"a": 1, "b": 2, "c": 3, "d": 4, "e": 5, "f": 6, "g": 7, "h": 8, "i": 9, "j": 10, "k": 11, "l": 12, "m": 13, "n": 14, "o": 15, "p": 16, "r": 17, "s": 18, "t": 19, "u": 20, "v": 21, "w": 22, "x": 23, "y": 24, "z": 25, "|": 26, "ँ": 27, "ं": 28, "ः": 29, "अ": 30, "आ": 31, "इ": 32, "ई": 33, "उ": 34, "ऊ": 35, "ऋ": 36, "ए": 37, "ऐ": 38, "ऑ": 39, "ओ": 40, "औ": 41, "क": 42, "ख": 43, "ग": 44, "घ": 45, "च": 46, "छ": 47, "ज": 48, "झ": 49, "ञ": 50, "ट": 51, "ठ": 52, "ड": 53, "ढ": 54, "ण": 55, "त": 56, "थ": 57, "द": 58, "ध": 59, "न": 60, "प": 61, "फ": 62, "ब": 63, "भ": 64, "म": 65, "य": 66, "र": 67, "ल": 68, "व": 69, "श": 70, "ष": 71, "स": 72, "ह": 73, "़": 74, "ा": 75, "ि": 76, "ी": 77, "ु": 78, "ू": 79, "ृ": 80, "ॅ": 81, "े": 82, "ै": 83, "ॉ": 84, "ो": 85, "ौ": 86, "्": 87, "क़": 88, "ग़": 89, "ज़": 90, "ड़": 91, "ढ़": 92, "।": 93, "’": 94, "^": 0, "[UNK]": 95, "[PAD]": 96}
+{"&": 1, "'": 2, "-": 3, "a": 4, "b": 5, "c": 6, "d": 7, "e": 8, "f": 9, "g": 10, "h": 11, "i": 12, "j": 13, "k": 14, "l": 15, "m": 16, "n": 17, "o": 18, "p": 19, "r": 20, "s": 21, "t": 22, "u": 23, "v": 24, "w": 25, "x": 26, "y": 27, "z": 28, "|": 29, "ँ": 30, "ं": 31, "ः": 32, "अ": 33, "आ": 34, "इ": 35, "ई": 36, "उ": 37, "ऊ": 38, "ऋ": 39, "ए": 40, "ऐ": 41, "ऑ": 42, "ओ": 43, "औ": 44, "क": 45, "ख": 46, "ग": 47, "घ": 48, "च": 49, "छ": 50, "ज": 51, "झ": 52, "ञ": 53, "ट": 54, "ठ": 55, "ड": 56, "ढ": 57, "ण": 58, "त": 59, "थ": 60, "द": 61, "ध": 62, "न": 63, "प": 64, "फ": 65, "ब": 66, "भ": 67, "म": 68, "य": 69, "र": 70, "ल": 71, "व": 72, "श": 73, "ष": 74, "स": 75, "ह": 76, "़": 77, "ा": 78, "ि": 79, "ी": 80, "ु": 81, "ू": 82, "ृ": 83, "ॅ": 84, "े": 85, "ै": 86, "ॉ": 87, "ो": 88, "ौ": 89, "्": 90, "क़": 91, "ग़": 92, "ज़": 93, "ड़": 94, "ढ़": 95, "^": 0, "[UNK]": 96, "[PAD]": 97}