Zaid
/

ashaar_tokenizer

Model card Files Files and versions Community

Zaid commited on May 25, 2023

Commit

d3f1780

•

1 Parent(s): 17d50a9

Upload tokenizer

Browse files

Files changed (2) hide show

tokenizer.json +39 -40
vocab.json +1 -1

tokenizer.json CHANGED Viewed

@@ -326,46 +326,45 @@
       "ظ": 115,
       "ع": 116,
       "غ": 117,
-      "ـ": 118,
-      "ف": 119,
-      "ق": 120,
-      "ك": 121,
-      "ل": 122,
-      "م": 123,
-      "ن": 124,
-      "ه": 125,
-      "و": 126,
-      "ى": 127,
-      "ي": 128,
-      "ً": 129,
-      "ٌ": 130,
-      "ٍ": 131,
-      "َ": 132,
-      "ُ": 133,
-      "ِ": 134,
-      "ّ": 135,
-      "ْ": 136,
-      "ٓ": 137,
-      "٤": 138,
-      "٩": 139,
-      "ٹ": 140,
-      "ک": 141,
-      "ی": 142,
-      "‌": 143,
-      "‍": 144,
-      "‏": 145,
-      "–": 146,
-      "“": 147,
-      "”": 148,
-      "„": 149,
-      "†": 150,
-      "…": 151,
-      "": 152,
-      "": 153,
-      "": 154,
-      "": 155,
-      "ﺑ": 156,
-      "�": 157
     },
     "merges": []
   }

       "ظ": 115,
       "ع": 116,
       "غ": 117,
+      "ف": 118,
+      "ق": 119,
+      "ك": 120,
+      "ل": 121,
+      "م": 122,
+      "ن": 123,
+      "ه": 124,
+      "و": 125,
+      "ى": 126,
+      "ي": 127,
+      "ً": 128,
+      "ٌ": 129,
+      "ٍ": 130,
+      "َ": 131,
+      "ُ": 132,
+      "ِ": 133,
+      "ّ": 134,
+      "ْ": 135,
+      "ٓ": 136,
+      "٤": 137,
+      "٩": 138,
+      "ٹ": 139,
+      "ک": 140,
+      "ی": 141,
+      "‌": 142,
+      "‍": 143,
+      "‏": 144,
+      "–": 145,
+      "“": 146,
+      "”": 147,
+      "„": 148,
+      "†": 149,
+      "…": 150,
+      "": 151,
+      "": 152,
+      "": 153,
+      "": 154,
+      "ﺑ": 155,
+      "�": 156
     },
     "merges": []
   }

vocab.json CHANGED Viewed

@@ -1 +1 @@

- {"<|endoftext|>":0,"<|vsep|>":1,"<|bsep|>":2,"<|pad|>":3,"<|psep|>":4,"<|meter_0|>":5,"<|meter_1|>":6,"<|meter_2|>":7,"<|meter_3|>":8,"<|meter_4|>":9,"<|meter_5|>":10,"<|meter_6|>":11,"<|meter_7|>":12,"<|meter_8|>":13,"<|meter_9|>":14,"<|meter_10|>":15,"<|meter_11|>":16,"<|meter_12|>":17,"<|meter_13|>":18,"<|meter_14|>":19,"<|meter_15|>":20,"\t":21," ":22,"!":23,"\"":24,"#":25,"(":26,")":27,"*":28,",":29,"-":30,".":31,"/":32,"0":33,"1":34,"2":35,"3":36,"4":37,"5":38,"6":39,"7":40,"8":41,"9":42,":":43,";":44,"<":45,"=":46,">":47,"?":48,"G":49,"H":50,"I":51,"P":52,"R":53,"T":54,"[":55,"]":56,"_":57,"a":58,"b":59,"c":60,"e":61,"f":62,"g":63,"h":64,"i":65,"l":66,"m":67,"n":68,"p":69,"r":70,"s":71,"t":72,"v":73,"x":74,"y":75,"|":76,"}":77,"~":78," ":79,"£":80,"§":81,"«":82,"±":83,"´":84,"·":85,"»":86,"×":87,"ˆ":88,"،":89,"؛":90,"؟":91,"ء":92,"آ":93,"أ":94,"ؤ":95,"إ":96,"ئ":97,"ا":98,"ب":99,"ة":100,"ت":101,"ث":102,"ج":103,"ح":104,"خ":105,"د":106,"ذ":107,"ر":108,"ز":109,"س":110,"ش":111,"ص":112,"ض":113,"ط":114,"ظ":115,"ع":116,"غ":117,"ـ":118,"ف":119,"ق":120,"ك":121,"ل":122,"م":123,"ن":124,"ه":125,"و":126,"ى":127,"ي":128,"ً":129,"ٌ":130,"ٍ":131,"َ":132,"ُ":133,"ِ":134,"ّ":135,"ْ":136,"ٓ":137,"٤":138,"٩":139,"ٹ":140,"ک":141,"ی":142,"‌":143,"‍":144,"‏":145,"–":146,"“":147,"”":148,"„":149,"†":150,"…":151,"":152,"":153,"":154,"":155,"ﺑ":156~~,"�":157~~}

+ {"<|endoftext|>":0,"<|vsep|>":1,"<|bsep|>":2,"<|pad|>":3,"<|psep|>":4,"<|meter_0|>":5,"<|meter_1|>":6,"<|meter_2|>":7,"<|meter_3|>":8,"<|meter_4|>":9,"<|meter_5|>":10,"<|meter_6|>":11,"<|meter_7|>":12,"<|meter_8|>":13,"<|meter_9|>":14,"<|meter_10|>":15,"<|meter_11|>":16,"<|meter_12|>":17,"<|meter_13|>":18,"<|meter_14|>":19,"<|meter_15|>":20,"\t":21," ":22,"!":23,"\"":24,"#":25,"(":26,")":27,"*":28,",":29,"-":30,".":31,"/":32,"0":33,"1":34,"2":35,"3":36,"4":37,"5":38,"6":39,"7":40,"8":41,"9":42,":":43,";":44,"<":45,"=":46,">":47,"?":48,"G":49,"H":50,"I":51,"P":52,"R":53,"T":54,"[":55,"]":56,"_":57,"a":58,"b":59,"c":60,"e":61,"f":62,"g":63,"h":64,"i":65,"l":66,"m":67,"n":68,"p":69,"r":70,"s":71,"t":72,"v":73,"x":74,"y":75,"|":76,"}":77,"~":78," ":79,"£":80,"§":81,"«":82,"±":83,"´":84,"·":85,"»":86,"×":87,"ˆ":88,"،":89,"؛":90,"؟":91,"ء":92,"آ":93,"أ":94,"ؤ":95,"إ":96,"ئ":97,"ا":98,"ب":99,"ة":100,"ت":101,"ث":102,"ج":103,"ح":104,"خ":105,"د":106,"ذ":107,"ر":108,"ز":109,"س":110,"ش":111,"ص":112,"ض":113,"ط":114,"ظ":115,"ع":116,"غ":117,"ف":118,"ق":119,"ك":120,"ل":121,"م":122,"ن":123,"ه":124,"و":125,"ى":126,"ي":127,"ً":128,"ٌ":129,"ٍ":130,"َ":131,"ُ":132,"ِ":133,"ّ":134,"ْ":135,"ٓ":136,"٤":137,"٩":138,"ٹ":139,"ک":140,"ی":141,"‌":142,"‍":143,"‏":144,"–":145,"“":146,"”":147,"„":148,"†":149,"…":150,"":151,"":152,"":153,"":154,"ﺑ":155,"�":156}