ashaar_tokenizer / vocab.json
Zaid's picture
Upload tokenizer
f11724f
raw
history blame
1.03 kB
{"<|endoftext|>":0,"<|psep|>":1,"</|psep|>":2,"<|vsep|>":3,"<|bsep|>":4,"</|bsep|>":5,"<|pad|>":6,"<|meter_0|>":7,"<|meter_1|>":8,"<|meter_2|>":9,"<|meter_3|>":10,"<|meter_4|>":11,"<|meter_5|>":12,"<|meter_6|>":13,"<|meter_7|>":14,"<|meter_8|>":15,"<|meter_9|>":16,"<|meter_10|>":17,"<|meter_11|>":18,"<|meter_12|>":19,"<|meter_13|>":20,"<|meter_14|>":21,"<|meter_15|>":22,"<|res_0|>":23,"<|res_1|>":24,"<|res_2|>":25,"<|res_3|>":26,"<|res_4|>":27,"<|res_5|>":28,"<|res_6|>":29,"<|res_7|>":30,"<|res_8|>":31,"<|res_9|>":32," ":33,"/":34,"0":35,"1":36,"2":37,"3":38,"4":39,"5":40,"6":41,"7":42,"8":43,"9":44,"<":45,">":46,"_":47,"b":48,"e":49,"m":50,"p":51,"r":52,"s":53,"t":54,"v":55,"|":56,"~":57,"ء":58,"أ":59,"ؤ":60,"ئ":61,"ا":62,"ب":63,"ة":64,"ت":65,"ث":66,"ج":67,"ح":68,"خ":69,"د":70,"ذ":71,"ر":72,"ز":73,"س":74,"ش":75,"ص":76,"ض":77,"ط":78,"ظ":79,"ع":80,"غ":81,"ف":82,"ق":83,"ك":84,"ل":85,"م":86,"ن":87,"ه":88,"و":89,"ى":90,"ي":91,"ً":92,"ٌ":93,"ٍ":94,"َ":95,"ُ":96,"ِ":97,"ّ":98,"ْ":99}