Upload tokenizer

added_tokens.json

@@ -1,4 +1,4 @@
 {
-  "</s>": 90,
-  "<s>": 89
+  "</s>": 146,
+  "<s>": 145
 }

tokenizer_config.json

@@ -1,6 +1,6 @@
 {
   "added_tokens_decoder": {
-    "87": {
+    "143": {
       "content": "[UNK]",
       "lstrip": true,
       "normalized": false,
@@ -8,7 +8,7 @@
       "single_word": false,
       "special": false
     },
-    "88": {
+    "144": {
       "content": "[PAD]",
       "lstrip": true,
       "normalized": false,
@@ -16,7 +16,7 @@
       "single_word": false,
       "special": false
     },
-    "89": {
+    "145": {
       "content": "<s>",
       "lstrip": false,
       "normalized": false,
@@ -24,7 +24,7 @@
       "single_word": false,
       "special": true
     },
-    "90": {
+    "146": {
       "content": "</s>",
       "lstrip": false,
       "normalized": false,

vocab.json

@@ -1 +1,149 @@
-{"ara": {"@": 1, "\\": 2, "a": 3, "b": 4, "c": 5, "d": 6, "e": 7, "f": 8, "g": 9, "h": 10, "i": 11, "j": 12, "k": 13, "l": 14, "m": 15, "n": 16, "o": 17, "p": 18, "q": 19, "r": 20, "s": 21, "t": 22, "u": 23, "v": 24, "w": 25, "x": 26, "y": 27, "z": 28, "\u00b7": 29, "\u00e0": 30, "\u00e5": 31, "\u00e7": 32, "\u00e8": 33, "\u00e9": 34, "\u00ea": 35, "\u00fb": 36, "\u0307": 37, "\u061b": 38, "\u0621": 39, "\u0622": 40, "\u0623": 41, "\u0624": 42, "\u0625": 43, "\u0626": 44, "\u0627": 45, "\u0628": 46, "\u0629": 47, "\u062a": 48, "\u062b": 49, "\u062c": 50, "\u062d": 51, "\u062e": 52, "\u062f": 53, "\u0630": 54, "\u0631": 55, "\u0632": 56, "\u0633": 57, "\u0634": 58, "\u0635": 59, "\u0636": 60, "\u0637": 61, "\u0638": 62, "\u0639": 63, "\u063a": 64, "\u0640": 65, "\u0641": 66, "\u0642": 67, "\u0643": 68, "\u0644": 69, "\u0645": 70, "\u0646": 71, "\u0647": 72, "\u0648": 73, "\u0649": 74, "\u064a": 75, "\u064b": 76, "\u064c": 77, "\u064d": 78, "\u064e": 79, "\u064f": 80, "\u0650": 81, "\u0651": 82, "\u0663": 83, "\u0665": 84, "\u0667": 85, "\u2019": 86, "|": 0, "[UNK]": 87, "[PAD]": 88}}
+{
+  "ara": {
+    "!": 1,
+    "\"": 2,
+    "#": 3,
+    "%": 4,
+    "&": 5,
+    "'": 6,
+    "(": 7,
+    ")": 8,
+    "+": 9,
+    ",": 10,
+    "-": 11,
+    ".": 12,
+    "/": 13,
+    "0": 14,
+    "1": 15,
+    "2": 16,
+    "3": 17,
+    "4": 18,
+    "5": 19,
+    "6": 20,
+    "7": 21,
+    "8": 22,
+    "9": 23,
+    ":": 24,
+    ";": 25,
+    "?": 26,
+    "@": 27,
+    "A": 28,
+    "B": 29,
+    "C": 30,
+    "D": 31,
+    "E": 32,
+    "F": 33,
+    "G": 34,
+    "H": 35,
+    "I": 36,
+    "J": 37,
+    "K": 38,
+    "L": 39,
+    "M": 40,
+    "N": 41,
+    "O": 42,
+    "P": 43,
+    "Q": 44,
+    "R": 45,
+    "S": 46,
+    "T": 47,
+    "U": 48,
+    "V": 49,
+    "W": 50,
+    "X": 51,
+    "Y": 52,
+    "Z": 53,
+    "[PAD]": 144,
+    "[UNK]": 143,
+    "\\": 54,
+    "a": 55,
+    "b": 56,
+    "c": 57,
+    "d": 58,
+    "e": 59,
+    "f": 60,
+    "g": 61,
+    "h": 62,
+    "i": 63,
+    "j": 64,
+    "k": 65,
+    "l": 66,
+    "m": 67,
+    "n": 68,
+    "o": 69,
+    "p": 70,
+    "q": 71,
+    "r": 72,
+    "s": 73,
+    "t": 74,
+    "u": 75,
+    "v": 76,
+    "w": 77,
+    "x": 78,
+    "y": 79,
+    "z": 80,
+    "|": 0,
+    "·": 81,
+    "à": 82,
+    "å": 83,
+    "ç": 84,
+    "è": 85,
+    "é": 86,
+    "ê": 87,
+    "û": 88,
+    "İ": 89,
+    "،": 90,
+    "؛": 91,
+    "؟": 92,
+    "ء": 93,
+    "آ": 94,
+    "أ": 95,
+    "ؤ": 96,
+    "إ": 97,
+    "ئ": 98,
+    "ا": 99,
+    "ب": 100,
+    "ة": 101,
+    "ت": 102,
+    "ث": 103,
+    "ج": 104,
+    "ح": 105,
+    "خ": 106,
+    "د": 107,
+    "ذ": 108,
+    "ر": 109,
+    "ز": 110,
+    "س": 111,
+    "ش": 112,
+    "ص": 113,
+    "ض": 114,
+    "ط": 115,
+    "ظ": 116,
+    "ع": 117,
+    "غ": 118,
+    "ـ": 119,
+    "ف": 120,
+    "ق": 121,
+    "ك": 122,
+    "ل": 123,
+    "م": 124,
+    "ن": 125,
+    "ه": 126,
+    "و": 127,
+    "ى": 128,
+    "ي": 129,
+    "ً": 130,
+    "ٌ": 131,
+    "ٍ": 132,
+    "َ": 133,
+    "ُ": 134,
+    "ِ": 135,
+    "ّ": 136,
+    "٣": 137,
+    "٥": 138,
+    "٧": 139,
+    "’": 140,
+    "“": 141,
+    "”": 142
+  }
+}