add tokenizer

vocab.json

@@ -1 +1 @@
-{",": 1, "-": 2, ".": 3, ";": 4, "?": 5, "؛": 6, "ء": 7, "آ": 8, "أ": 9, "ؤ": 10, "إ": 11, "ئ": 12, "ا": 13, "ب": 14, "ة": 15, "ت": 16, "ث": 17, "ج": 18, "ح": 19, "خ": 20, "د": 21, "ذ": 22, "ر": 23, "ز": 24, "س": 25, "ش": 26, "ص": 27, "ض": 28, "ط": 29, "ظ": 30, "ع": 31, "غ": 32, "ف": 33, "ق": 34, "ك": 35, "ل": 36, "م": 37, "ن": 38, "ه": 39, "و": 40, "ى": 41, "ي": 42, "ٌ": 43, "ْ": 44, "ٰ": 45, "چ": 46, "ڨ": 47, "ک": 48, "ھ": 49, "ی": 50, "ۘ": 51, "ۛ": 52, "ﺃ": 53, "ﻻ": 54, "|": 0, "<unk>": 55, "<pad>": 56, "<s>": 57, "</s>": 58}
+{",": 1, "ء": 2, "آ": 3, "أ": 4, "ؤ": 5, "إ": 6, "ئ": 7, "ا": 8, "ب": 9, "ة": 10, "ت": 11, "ث": 12, "ج": 13, "ح": 14, "خ": 15, "د": 16, "ذ": 17, "ر": 18, "ز": 19, "س": 20, "ش": 21, "ص": 22, "ض": 23, "ط": 24, "ظ": 25, "ع": 26, "غ": 27, "ف": 28, "ق": 29, "ك": 30, "ل": 31, "م": 32, "ن": 33, "ه": 34, "و": 35, "ى": 36, "ي": 37, "ٌ": 38, "ْ": 39, "چ": 40, "ڨ": 41, "ک": 42, "ھ": 43, "ی": 44, "ۘ": 45, "ﺃ": 46, "ﻻ": 47, "|": 0, "<unk>": 48, "<pad>": 49, "<s>": 50, "</s>": 51}