add tokenizer

added_tokens.json

@@ -1 +1 @@
-{"<s>": 49, "</s>": 50}
+{"<s>": 46, "</s>": 47}

vocab.json

@@ -1 +1 @@
-{"e": 1, "n": 2, "o": 3, "ء": 4, "آ": 5, "ؤ": 6, "ئ": 7, "ا": 8, "ب": 9, "ت": 10, "ث": 11, "ج": 12, "ح": 13, "خ": 14, "د": 15, "ذ": 16, "ر": 17, "ز": 18, "س": 19, "ش": 20, "ص": 21, "ض": 22, "ط": 23, "ظ": 24, "ع": 25, "غ": 26, "ف": 27, "ق": 28, "ل": 29, "م": 30, "ن": 31, "و": 32, "ٹ": 33, "پ": 34, "چ": 35, "ڈ": 36, "ڑ": 37, "ژ": 38, "ک": 39, "گ": 40, "ں": 41, "ھ": 42, "ہ": 43, "ۂ": 44, "ی": 45, "ے": 46, "|": 0, "[UNK]": 47, "[PAD]": 48}
+{"ء": 1, "آ": 2, "ؤ": 3, "ئ": 4, "ا": 5, "ب": 6, "ت": 7, "ث": 8, "ج": 9, "ح": 10, "خ": 11, "د": 12, "ذ": 13, "ر": 14, "ز": 15, "س": 16, "ش": 17, "ص": 18, "ض": 19, "ط": 20, "ظ": 21, "ع": 22, "غ": 23, "ف": 24, "ق": 25, "ل": 26, "م": 27, "ن": 28, "و": 29, "ٹ": 30, "پ": 31, "چ": 32, "ڈ": 33, "ڑ": 34, "ژ": 35, "ک": 36, "گ": 37, "ں": 38, "ھ": 39, "ہ": 40, "ۂ": 41, "ی": 42, "ے": 43, "|": 0, "[UNK]": 44, "[PAD]": 45}