tokenizer.json

{
  "version": "1.0",
  "truncation": null,
  "padding": null,
  "added_tokens": [
    {
      "id": 0,
      "content": "<|endoftext|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 1,
      "content": "<|vsep|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 2,
      "content": "<|bsep|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 3,
      "content": "<|pad|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 4,
      "content": "|psep|",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 5,
      "content": "<|meter_0|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 6,
      "content": "<|meter_1|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 7,
      "content": "<|meter_2|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 8,
      "content": "<|meter_3|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 9,
      "content": "<|meter_4|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 10,
      "content": "<|meter_5|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 11,
      "content": "<|meter_6|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 12,
      "content": "<|meter_7|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 13,
      "content": "<|meter_8|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 14,
      "content": "<|meter_9|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 15,
      "content": "<|meter_10|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 16,
      "content": "<|meter_11|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 17,
      "content": "<|meter_12|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 18,
      "content": "<|meter_13|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    }
  ],
  "normalizer": null,
  "pre_tokenizer": {
    "type": "UnicodeScripts"
  },
  "post_processor": null,
  "decoder": null,
  "model": {
    "type": "BPE",
    "dropout": null,
    "unk_token": null,
    "continuing_subword_prefix": null,
    "end_of_word_suffix": null,
    "fuse_unk": false,
    "byte_fallback": false,
    "vocab": {
      "<|endoftext|>": 0,
      "<|vsep|>": 1,
      "<|bsep|>": 2,
      "<|pad|>": 3,
      "|psep|": 4,
      "<|meter_0|>": 5,
      "<|meter_1|>": 6,
      "<|meter_2|>": 7,
      "<|meter_3|>": 8,
      "<|meter_4|>": 9,
      "<|meter_5|>": 10,
      "<|meter_6|>": 11,
      "<|meter_7|>": 12,
      "<|meter_8|>": 13,
      "<|meter_9|>": 14,
      "<|meter_10|>": 15,
      "<|meter_11|>": 16,
      "<|meter_12|>": 17,
      "<|meter_13|>": 18,
      " ": 19,
      "0": 20,
      "1": 21,
      "2": 22,
      "3": 23,
      "4": 24,
      "5": 25,
      "6": 26,
      "7": 27,
      "8": 28,
      "9": 29,
      ":": 30,
      "<": 31,
      ">": 32,
      "_": 33,
      "b": 34,
      "e": 35,
      "m": 36,
      "p": 37,
      "r": 38,
      "s": 39,
      "t": 40,
      "v": 41,
      "|": 42,
      "ء": 43,
      "آ": 44,
      "أ": 45,
      "ؤ": 46,
      "إ": 47,
      "ئ": 48,
      "ا": 49,
      "ب": 50,
      "ة": 51,
      "ت": 52,
      "ث": 53,
      "ج": 54,
      "ح": 55,
      "خ": 56,
      "د": 57,
      "ذ": 58,
      "ر": 59,
      "ز": 60,
      "س": 61,
      "ش": 62,
      "ص": 63,
      "ض": 64,
      "ط": 65,
      "ظ": 66,
      "ع": 67,
      "غ": 68,
      "ف": 69,
      "ق": 70,
      "ك": 71,
      "ل": 72,
      "م": 73,
      "ن": 74,
      "ه": 75,
      "و": 76,
      "ى": 77,
      "ي": 78,
      "ً": 79,
      "ٌ": 80,
      "ٍ": 81,
      "َ": 82,
      "ُ": 83,
      "ِ": 84,
      "ّ": 85,
      "ْ": 86
    },
    "merges": []
  }
}