Upload tokenizer

tokenizer.json

@@ -229,142 +229,72 @@
       "<|meter_13|>": 18,
       "<|meter_14|>": 19,
       "<|meter_15|>": 20,
-      "\t": 21,
-      " ": 22,
-      "!": 23,
-      "\"": 24,
-      "#": 25,
-      "(": 26,
-      ")": 27,
-      "*": 28,
-      ",": 29,
-      "-": 30,
-      ".": 31,
-      "/": 32,
-      "0": 33,
-      "1": 34,
-      "2": 35,
-      "3": 36,
-      "4": 37,
-      "5": 38,
-      "6": 39,
-      "7": 40,
-      "8": 41,
-      "9": 42,
-      ":": 43,
-      ";": 44,
-      "<": 45,
-      "=": 46,
-      ">": 47,
-      "?": 48,
-      "G": 49,
-      "H": 50,
-      "I": 51,
-      "P": 52,
-      "R": 53,
-      "T": 54,
-      "[": 55,
-      "]": 56,
-      "_": 57,
-      "a": 58,
-      "b": 59,
-      "c": 60,
-      "e": 61,
-      "f": 62,
-      "g": 63,
-      "h": 64,
-      "i": 65,
-      "l": 66,
-      "m": 67,
-      "n": 68,
-      "p": 69,
-      "r": 70,
-      "s": 71,
-      "t": 72,
-      "v": 73,
-      "x": 74,
-      "y": 75,
-      "|": 76,
-      "}": 77,
-      "~": 78,
-      " ": 79,
-      "£": 80,
-      "§": 81,
-      "«": 82,
-      "±": 83,
-      "´": 84,
-      "·": 85,
-      "»": 86,
-      "×": 87,
-      "ˆ": 88,
-      "،": 89,
-      "؛": 90,
-      "؟": 91,
-      "ء": 92,
-      "آ": 93,
-      "أ": 94,
-      "ؤ": 95,
-      "إ": 96,
-      "ئ": 97,
-      "ا": 98,
-      "ب": 99,
-      "ة": 100,
-      "ت": 101,
-      "ث": 102,
-      "ج": 103,
-      "ح": 104,
-      "خ": 105,
-      "د": 106,
-      "ذ": 107,
-      "ر": 108,
-      "ز": 109,
-      "س": 110,
-      "ش": 111,
-      "ص": 112,
-      "ض": 113,
-      "ط": 114,
-      "ظ": 115,
-      "ع": 116,
-      "غ": 117,
-      "ف": 118,
-      "ق": 119,
-      "ك": 120,
-      "ل": 121,
-      "م": 122,
-      "ن": 123,
-      "ه": 124,
-      "و": 125,
-      "ى": 126,
-      "ي": 127,
-      "ً": 128,
-      "ٌ": 129,
-      "ٍ": 130,
-      "َ": 131,
-      "ُ": 132,
-      "ِ": 133,
-      "ّ": 134,
-      "ْ": 135,
-      "ٓ": 136,
-      "٤": 137,
-      "٩": 138,
-      "ٹ": 139,
-      "ک": 140,
-      "ی": 141,
-      "‌": 142,
-      "‍": 143,
-      "‏": 144,
-      "–": 145,
-      "“": 146,
-      "”": 147,
-      "„": 148,
-      "†": 149,
-      "…": 150,
-      "": 151,
-      "": 152,
-      "": 153,
-      "": 154,
-      "ﺑ": 155,
-      "�": 156
+      " ": 21,
+      "0": 22,
+      "1": 23,
+      "2": 24,
+      "3": 25,
+      "4": 26,
+      "5": 27,
+      "6": 28,
+      "7": 29,
+      "8": 30,
+      "9": 31,
+      "<": 32,
+      ">": 33,
+      "_": 34,
+      "b": 35,
+      "e": 36,
+      "m": 37,
+      "p": 38,
+      "r": 39,
+      "s": 40,
+      "t": 41,
+      "v": 42,
+      "|": 43,
+      "~": 44,
+      "ء": 45,
+      "أ": 46,
+      "ؤ": 47,
+      "ئ": 48,
+      "ا": 49,
+      "ب": 50,
+      "ة": 51,
+      "ت": 52,
+      "ث": 53,
+      "ج": 54,
+      "ح": 55,
+      "خ": 56,
+      "د": 57,
+      "ذ": 58,
+      "ر": 59,
+      "ز": 60,
+      "س": 61,
+      "ش": 62,
+      "ص": 63,
+      "ض": 64,
+      "ط": 65,
+      "ظ": 66,
+      "ع": 67,
+      "غ": 68,
+      "ف": 69,
+      "ق": 70,
+      "ك": 71,
+      "ل": 72,
+      "م": 73,
+      "ن": 74,
+      "ه": 75,
+      "و": 76,
+      "ى": 77,
+      "ي": 78,
+      "ً": 79,
+      "ٌ": 80,
+      "ٍ": 81,
+      "َ": 82,
+      "ُ": 83,
+      "ِ": 84,
+      "ّ": 85,
+      "ْ": 86
     },
     "merges": []
   }

vocab.json

@@ -1 +1 @@
-{"<|endoftext|>":0,"<|vsep|>":1,"<|bsep|>":2,"<|pad|>":3,"<|psep|>":4,"<|meter_0|>":5,"<|meter_1|>":6,"<|meter_2|>":7,"<|meter_3|>":8,"<|meter_4|>":9,"<|meter_5|>":10,"<|meter_6|>":11,"<|meter_7|>":12,"<|meter_8|>":13,"<|meter_9|>":14,"<|meter_10|>":15,"<|meter_11|>":16,"<|meter_12|>":17,"<|meter_13|>":18,"<|meter_14|>":19,"<|meter_15|>":20,"\t":21," ":22,"!":23,"\"":24,"#":25,"(":26,")":27,"*":28,",":29,"-":30,".":31,"/":32,"0":33,"1":34,"2":35,"3":36,"4":37,"5":38,"6":39,"7":40,"8":41,"9":42,":":43,";":44,"<":45,"=":46,">":47,"?":48,"G":49,"H":50,"I":51,"P":52,"R":53,"T":54,"[":55,"]":56,"_":57,"a":58,"b":59,"c":60,"e":61,"f":62,"g":63,"h":64,"i":65,"l":66,"m":67,"n":68,"p":69,"r":70,"s":71,"t":72,"v":73,"x":74,"y":75,"|":76,"}":77,"~":78," ":79,"£":80,"§":81,"«":82,"±":83,"´":84,"·":85,"»":86,"×":87,"ˆ":88,"،":89,"؛":90,"؟":91,"ء":92,"آ":93,"أ":94,"ؤ":95,"إ":96,"ئ":97,"ا":98,"ب":99,"ة":100,"ت":101,"ث":102,"ج":103,"ح":104,"خ":105,"د":106,"ذ":107,"ر":108,"ز":109,"س":110,"ش":111,"ص":112,"ض":113,"ط":114,"ظ":115,"ع":116,"غ":117,"ف":118,"ق":119,"ك":120,"ل":121,"م":122,"ن":123,"ه":124,"و":125,"ى":126,"ي":127,"ً":128,"ٌ":129,"ٍ":130,"َ":131,"ُ":132,"ِ":133,"ّ":134,"ْ":135,"ٓ":136,"٤":137,"٩":138,"ٹ":139,"ک":140,"ی":141,"‌":142,"‍":143,"‏":144,"–":145,"“":146,"”":147,"„":148,"†":149,"…":150,"":151,"":152,"":153,"":154,"ﺑ":155,"�":156}
+{"<|endoftext|>":0,"<|vsep|>":1,"<|bsep|>":2,"<|pad|>":3,"<|psep|>":4,"<|meter_0|>":5,"<|meter_1|>":6,"<|meter_2|>":7,"<|meter_3|>":8,"<|meter_4|>":9,"<|meter_5|>":10,"<|meter_6|>":11,"<|meter_7|>":12,"<|meter_8|>":13,"<|meter_9|>":14,"<|meter_10|>":15,"<|meter_11|>":16,"<|meter_12|>":17,"<|meter_13|>":18,"<|meter_14|>":19,"<|meter_15|>":20," ":21,"0":22,"1":23,"2":24,"3":25,"4":26,"5":27,"6":28,"7":29,"8":30,"9":31,"<":32,">":33,"_":34,"b":35,"e":36,"m":37,"p":38,"r":39,"s":40,"t":41,"v":42,"|":43,"~":44,"ء":45,"أ":46,"ؤ":47,"ئ":48,"ا":49,"ب":50,"ة":51,"ت":52,"ث":53,"ج":54,"ح":55,"خ":56,"د":57,"ذ":58,"ر":59,"ز":60,"س":61,"ش":62,"ص":63,"ض":64,"ط":65,"ظ":66,"ع":67,"غ":68,"ف":69,"ق":70,"ك":71,"ل":72,"م":73,"ن":74,"ه":75,"و":76,"ى":77,"ي":78,"ً":79,"ٌ":80,"ٍ":81,"َ":82,"ُ":83,"ِ":84,"ّ":85,"ْ":86}