add tokenizer

vocab.json

@@ -1 +1 @@
-{"ط": 0, "ُ": 1, "ٔ": 2, "س": 3, "ٹ": 4, "ز": 5, "ش": 6, "خ": 7, "ژ": 8, "ظ": 9, "ڈ": 10, "م": 11, "ء": 12, "ہ": 13, "ے": 14, "د": 15, "ق": 16, "ک": 17, "ا": 18, "چ": 19, "َ": 20, "ع": 21, "ً": 22, "ل": 23, "ن": 24, "غ": 25, "ی": 26, "ئ": 27, "گ": 28, "ح": 29, "ث": 30, "ّ": 31, "ف": 32, "‌": 33, "ب": 34, "ذ": 35, "ض": 36, "آ": 37, "ت": 38, "ج": 39, "ؤ": 40, "ھ": 41, "ِ": 42, "ر": 43, "و": 45, "ں": 46, "پ": 47, "ڑ": 48, "ص": 49, "|": 44, "[UNK]": 50, "[PAD]": 51}
+{"ے": 0, "ئ": 1, "ٔ": 2, "َ": 3, "م": 4, "ی": 5, "ڑ": 7, "ل": 8, "ڈ": 9, "ط": 10, "ژ": 11, "ح": 12, "ا": 13, "ص": 14, "ن": 15, "س": 16, "‌": 17, "ذ": 18, "ض": 19, "ں": 20, "ز": 21, "ٹ": 22, "ّ": 23, "ت": 24, "ء": 25, "پ": 26, "ُ": 27, "ظ": 28, "ً": 29, "و": 30, "چ": 31, "ف": 32, "آ": 33, "ک": 34, "ش": 35, "ث": 36, "ر": 37, "ہ": 38, "ؤ": 39, "خ": 40, "ِ": 41, "غ": 42, "گ": 43, "ھ": 44, "ق": 45, "ع": 46, "د": 47, "ج": 48, "ب": 49, "|": 6, "[UNK]": 50, "[PAD]": 51}