add tokenizer

vocab.json

@@ -1 +1 @@
-{"د": 0, "ی": 1, "ش": 2, "ظ": 3, "گ": 4, "ً": 5, "ف": 6, "خ": 7, "ت": 8, "ص": 9, "م": 10, "ع": 12, "ن": 13, "ک": 14, "ز": 15, "‌": 16, "ے": 17, "ؤ": 18, "ٔ": 19, "ڑ": 20, "ڈ": 21, "ا": 22, "ر": 23, "آ": 24, "ہ": 25, "ھ": 26, "ئ": 27, "ّ": 28, "ج": 29, "ب": 30, "ق": 31, "ط": 32, "ض": 33, "ِ": 34, "ح": 35, "ل": 36, "س": 37, "و": 38, "َ": 39, "ژ": 40, "ء": 41, "غ": 42, "ں": 43, "چ": 44, "ُ": 45, "ث": 46, "ٹ": 47, "ذ": 48, "پ": 49, "|": 11, "[UNK]": 50, "[PAD]": 51}
+{"ا": 0, "د": 1, "ث": 2, "غ": 3, "ز": 4, "ؤ": 5, "ل": 6, "ی": 7, "ِ": 9, "ڑ": 10, "ں": 11, "ج": 12, "ر": 13, "ت": 14, "ّ": 15, "ش": 16, "ٔ": 17, "ک": 18, "س": 19, "ہ": 20, "ع": 21, "ط": 22, "ء": 23, "خ": 24, "ب": 25, "ژ": 26, "گ": 27, "ص": 28, "ے": 29, "ح": 30, "ُ": 31, "ٹ": 32, "و": 33, "م": 34, "ظ": 35, "ئ": 36, "ق": 37, "ذ": 38, "ً": 39, "ڈ": 40, "ف": 41, "ض": 42, "ھ": 43, "چ": 44, "پ": 45, "آ": 46, "‌": 47, "َ": 48, "ن": 49, "|": 8, "[UNK]": 50, "[PAD]": 51}