hassnain's picture
add tokenizer
329a00e
raw
history blame
516 Bytes
{"ا": 0, "د": 1, "ث": 2, "غ": 3, "ز": 4, "ؤ": 5, "ل": 6, "ی": 7, "ِ": 9, "ڑ": 10, "ں": 11, "ج": 12, "ر": 13, "ت": 14, "ّ": 15, "ش": 16, "ٔ": 17, "ک": 18, "س": 19, "ہ": 20, "ع": 21, "ط": 22, "ء": 23, "خ": 24, "ب": 25, "ژ": 26, "گ": 27, "ص": 28, "ے": 29, "ح": 30, "ُ": 31, "ٹ": 32, "و": 33, "م": 34, "ظ": 35, "ئ": 36, "ق": 37, "ذ": 38, "ً": 39, "ڈ": 40, "ف": 41, "ض": 42, "ھ": 43, "چ": 44, "پ": 45, "آ": 46, "‌": 47, "َ": 48, "ن": 49, "|": 8, "[UNK]": 50, "[PAD]": 51}