Upload tokenizer

merges.txt

@@ -0,0 +1 @@
+#version: 0.2

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

tokenizer.json

@@ -0,0 +1,623 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "<|endoftext|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "<|psep|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "</|psep|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "<|vsep|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 4,
+      "content": "<|bsep|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 5,
+      "content": "</|bsep|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 6,
+      "content": "<|pad|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 7,
+      "content": "<|meter_0|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 8,
+      "content": "<|meter_1|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 9,
+      "content": "<|meter_2|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 10,
+      "content": "<|meter_3|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 11,
+      "content": "<|meter_4|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 12,
+      "content": "<|meter_5|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 13,
+      "content": "<|meter_6|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 14,
+      "content": "<|meter_7|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 15,
+      "content": "<|meter_8|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 16,
+      "content": "<|meter_9|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 17,
+      "content": "<|meter_10|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 18,
+      "content": "<|meter_11|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 19,
+      "content": "<|meter_12|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 20,
+      "content": "<|meter_13|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 21,
+      "content": "<|meter_14|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 22,
+      "content": "<|meter_15|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 23,
+      "content": "<|meter_16|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 24,
+      "content": "<|theme_0|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 25,
+      "content": "<|theme_1|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 26,
+      "content": "<|theme_2|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 27,
+      "content": "<|theme_3|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 28,
+      "content": "<|theme_4|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 29,
+      "content": "<|theme_5|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 30,
+      "content": "<|theme_6|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 31,
+      "content": "<|theme_7|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 32,
+      "content": "<|theme_8|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 33,
+      "content": "<|theme_9|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 34,
+      "content": "<|theme_10|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 35,
+      "content": "<|theme_11|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 36,
+      "content": "<|theme_12|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 37,
+      "content": "<|theme_13|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 38,
+      "content": "<|theme_14|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 39,
+      "content": "<|theme_15|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 40,
+      "content": "<|theme_16|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 41,
+      "content": "<|theme_17|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 42,
+      "content": "<|theme_18|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 43,
+      "content": "<|res_0|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 44,
+      "content": "<|res_1|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 45,
+      "content": "<|res_2|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 46,
+      "content": "<|res_3|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 47,
+      "content": "<|res_4|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 48,
+      "content": "<|res_5|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 49,
+      "content": "<|res_6|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 50,
+      "content": "<|res_7|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 51,
+      "content": "<|res_8|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 52,
+      "content": "<|res_9|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "UnicodeScripts"
+  },
+  "post_processor": null,
+  "decoder": null,
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": null,
+    "continuing_subword_prefix": null,
+    "end_of_word_suffix": null,
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "vocab": {
+      "<|endoftext|>": 0,
+      "<|psep|>": 1,
+      "</|psep|>": 2,
+      "<|vsep|>": 3,
+      "<|bsep|>": 4,
+      "</|bsep|>": 5,
+      "<|pad|>": 6,
+      "<|meter_0|>": 7,
+      "<|meter_1|>": 8,
+      "<|meter_2|>": 9,
+      "<|meter_3|>": 10,
+      "<|meter_4|>": 11,
+      "<|meter_5|>": 12,
+      "<|meter_6|>": 13,
+      "<|meter_7|>": 14,
+      "<|meter_8|>": 15,
+      "<|meter_9|>": 16,
+      "<|meter_10|>": 17,
+      "<|meter_11|>": 18,
+      "<|meter_12|>": 19,
+      "<|meter_13|>": 20,
+      "<|meter_14|>": 21,
+      "<|meter_15|>": 22,
+      "<|meter_16|>": 23,
+      "<|theme_0|>": 24,
+      "<|theme_1|>": 25,
+      "<|theme_2|>": 26,
+      "<|theme_3|>": 27,
+      "<|theme_4|>": 28,
+      "<|theme_5|>": 29,
+      "<|theme_6|>": 30,
+      "<|theme_7|>": 31,
+      "<|theme_8|>": 32,
+      "<|theme_9|>": 33,
+      "<|theme_10|>": 34,
+      "<|theme_11|>": 35,
+      "<|theme_12|>": 36,
+      "<|theme_13|>": 37,
+      "<|theme_14|>": 38,
+      "<|theme_15|>": 39,
+      "<|theme_16|>": 40,
+      "<|theme_17|>": 41,
+      "<|theme_18|>": 42,
+      "<|res_0|>": 43,
+      "<|res_1|>": 44,
+      "<|res_2|>": 45,
+      "<|res_3|>": 46,
+      "<|res_4|>": 47,
+      "<|res_5|>": 48,
+      "<|res_6|>": 49,
+      "<|res_7|>": 50,
+      "<|res_8|>": 51,
+      "<|res_9|>": 52,
+      " ": 53,
+      "/": 54,
+      "0": 55,
+      "1": 56,
+      "2": 57,
+      "3": 58,
+      "4": 59,
+      "5": 60,
+      "6": 61,
+      "7": 62,
+      "8": 63,
+      "9": 64,
+      "<": 65,
+      ">": 66,
+      "_": 67,
+      "b": 68,
+      "e": 69,
+      "h": 70,
+      "m": 71,
+      "p": 72,
+      "r": 73,
+      "s": 74,
+      "t": 75,
+      "v": 76,
+      "|": 77,
+      "~": 78,
+      "ء": 79,
+      "أ": 80,
+      "ؤ": 81,
+      "ئ": 82,
+      "ا": 83,
+      "ب": 84,
+      "ة": 85,
+      "ت": 86,
+      "ث": 87,
+      "ج": 88,
+      "ح": 89,
+      "خ": 90,
+      "د": 91,
+      "ذ": 92,
+      "ر": 93,
+      "ز": 94,
+      "س": 95,
+      "ش": 96,
+      "ص": 97,
+      "ض": 98,
+      "ط": 99,
+      "ظ": 100,
+      "ع": 101,
+      "غ": 102,
+      "ف": 103,
+      "ق": 104,
+      "ك": 105,
+      "ل": 106,
+      "م": 107,
+      "ن": 108,
+      "ه": 109,
+      "و": 110,
+      "ى": 111,
+      "ي": 112,
+      "ً": 113,
+      "ٌ": 114,
+      "ٍ": 115,
+      "َ": 116,
+      "ُ": 117,
+      "ِ": 118,
+      "ّ": 119,
+      "ْ": 120
+    },
+    "merges": []
+  }
+}

tokenizer_config.json

@@ -0,0 +1,9 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

vocab.json

@@ -0,0 +1 @@
+{"<|endoftext|>":0,"<|psep|>":1,"</|psep|>":2,"<|vsep|>":3,"<|bsep|>":4,"</|bsep|>":5,"<|pad|>":6,"<|meter_0|>":7,"<|meter_1|>":8,"<|meter_2|>":9,"<|meter_3|>":10,"<|meter_4|>":11,"<|meter_5|>":12,"<|meter_6|>":13,"<|meter_7|>":14,"<|meter_8|>":15,"<|meter_9|>":16,"<|meter_10|>":17,"<|meter_11|>":18,"<|meter_12|>":19,"<|meter_13|>":20,"<|meter_14|>":21,"<|meter_15|>":22,"<|meter_16|>":23,"<|theme_0|>":24,"<|theme_1|>":25,"<|theme_2|>":26,"<|theme_3|>":27,"<|theme_4|>":28,"<|theme_5|>":29,"<|theme_6|>":30,"<|theme_7|>":31,"<|theme_8|>":32,"<|theme_9|>":33,"<|theme_10|>":34,"<|theme_11|>":35,"<|theme_12|>":36,"<|theme_13|>":37,"<|theme_14|>":38,"<|theme_15|>":39,"<|theme_16|>":40,"<|theme_17|>":41,"<|theme_18|>":42,"<|res_0|>":43,"<|res_1|>":44,"<|res_2|>":45,"<|res_3|>":46,"<|res_4|>":47,"<|res_5|>":48,"<|res_6|>":49,"<|res_7|>":50,"<|res_8|>":51,"<|res_9|>":52," ":53,"/":54,"0":55,"1":56,"2":57,"3":58,"4":59,"5":60,"6":61,"7":62,"8":63,"9":64,"<":65,">":66,"_":67,"b":68,"e":69,"h":70,"m":71,"p":72,"r":73,"s":74,"t":75,"v":76,"|":77,"~":78,"ء":79,"أ":80,"ؤ":81,"ئ":82,"ا":83,"ب":84,"ة":85,"ت":86,"ث":87,"ج":88,"ح":89,"خ":90,"د":91,"ذ":92,"ر":93,"ز":94,"س":95,"ش":96,"ص":97,"ض":98,"ط":99,"ظ":100,"ع":101,"غ":102,"ف":103,"ق":104,"ك":105,"ل":106,"م":107,"ن":108,"ه":109,"و":110,"ى":111,"ي":112,"ً":113,"ٌ":114,"ٍ":115,"َ":116,"ُ":117,"ِ":118,"ّ":119,"ْ":120}