hassnain commited on
Commit
759f044
1 Parent(s): 16ca6de

add tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +1 -1
vocab.json CHANGED
@@ -1 +1 @@
1
- {"ث": 0, "و": 1, "ج": 2, "ش": 3, "م": 4, "ع": 5, "َ": 6, "ڑ": 7, "ُ": 8, "ا": 9, "ٹ": 10, "": 11, "س": 12, "ص": 13, "ّ": 14, "ن": 15, "غ": 16, "ک": 17, "ر": 18, "ی": 19, "ح": 20, "ئ": 22, "ل": 23, "ڈ": 24, "ٔ": 25, "ز": 26, "ط": 27, "ذ": 28, "خ": 29, "ق": 30, "ف": 31, "آ": 32, "ؤ": 33, "د": 34, "پ": 35, "ھ": 36, "ِ": 37, "ہ": 38, "ژ": 39, "ً": 40, "ں": 41, "ض": 42, "گ": 43, "ء": 44, "ظ": 45, "چ": 46, "ے": 47, "ت": 48, "ب": 49, "|": 21, "[UNK]": 50, "[PAD]": 51}
 
1
+ {"ض": 0, "چ": 1, "ژ": 2, "ط": 3, "و": 4, "خ": 5, "ں": 6, "ث": 7, "پ": 8, "َ": 9, "گ": 10, "ڈ": 11, "آ": 12, "ع": 13, "ی": 14, "ڑ": 15, "ذ": 16, "ر": 17, "ّ": 18, "ھ": 19, "ح": 20, "ف": 21, "‌": 22, "ل": 23, "ئ": 24, "م": 25, "ب": 26, "ؤ": 27, "ِ": 28, "ُ": 29, "ٔ": 30, "ش": 31, "س": 32, "ظ": 33, "ق": 34, "ً": 35, "ہ": 36, "ے": 37, "ا": 38, "ن": 39, "د": 40, "ک": 41, "ج": 42, "ز": 44, "ء": 45, "ت": 46, "غ": 47, "ص": 48, "ٹ": 49, "|": 43, "[UNK]": 50, "[PAD]": 51}