ahmad573's picture
add tokenizer
299425c
{"ذ": 0, "ق": 1, "س": 2, "ع": 3, "ھ": 4, "ب": 5, "ن": 6, "ں": 7, "ت": 8, "ر": 9, "ژ": 10, "ئ": 11, "ء": 12, "ڈ": 13, "ٔ": 14, "غ": 15, "ی": 16, "ے": 17, "ح": 18, "‌": 19, "ً": 20, "ض": 21, "ز": 23, "م": 24, "آ": 25, "و": 26, "پ": 27, "ُ": 28, "د": 29, "ش": 30, "َ": 31, "خ": 32, "ظ": 33, "گ": 34, "ف": 35, "ڑ": 36, "ّ": 37, "ک": 38, "ث": 39, "ٹ": 40, "چ": 41, "ا": 42, "ؤ": 43, "ہ": 44, "ص": 45, "ج": 46, "ط": 47, "ِ": 48, "ل": 49, "|": 22, "[UNK]": 50, "[PAD]": 51}