{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "<|vsep|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "<|bsep|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "<|pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 4, "content": "|psep|", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 5, "content": "<|meter_0|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 6, "content": "<|meter_1|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 7, "content": "<|meter_2|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 8, "content": "<|meter_3|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 9, "content": "<|meter_4|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 10, "content": "<|meter_5|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 11, "content": "<|meter_6|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 12, "content": "<|meter_7|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 13, "content": "<|meter_8|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 14, "content": "<|meter_9|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 15, "content": "<|meter_10|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 16, "content": "<|meter_11|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 17, "content": "<|meter_12|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 18, "content": "<|meter_13|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "UnicodeScripts" }, "post_processor": null, "decoder": null, "model": { "type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "vocab": { "<|endoftext|>": 0, "<|vsep|>": 1, "<|bsep|>": 2, "<|pad|>": 3, "|psep|": 4, "<|meter_0|>": 5, "<|meter_1|>": 6, "<|meter_2|>": 7, "<|meter_3|>": 8, "<|meter_4|>": 9, "<|meter_5|>": 10, "<|meter_6|>": 11, "<|meter_7|>": 12, "<|meter_8|>": 13, "<|meter_9|>": 14, "<|meter_10|>": 15, "<|meter_11|>": 16, "<|meter_12|>": 17, "<|meter_13|>": 18, " ": 19, "0": 20, "1": 21, "2": 22, "3": 23, "4": 24, "5": 25, "6": 26, "7": 27, "8": 28, "9": 29, ":": 30, "<": 31, ">": 32, "_": 33, "b": 34, "e": 35, "m": 36, "p": 37, "r": 38, "s": 39, "t": 40, "v": 41, "|": 42, "ء": 43, "آ": 44, "أ": 45, "ؤ": 46, "إ": 47, "ئ": 48, "ا": 49, "ب": 50, "ة": 51, "ت": 52, "ث": 53, "ج": 54, "ح": 55, "خ": 56, "د": 57, "ذ": 58, "ر": 59, "ز": 60, "س": 61, "ش": 62, "ص": 63, "ض": 64, "ط": 65, "ظ": 66, "ع": 67, "غ": 68, "ف": 69, "ق": 70, "ك": 71, "ل": 72, "م": 73, "ن": 74, "ه": 75, "و": 76, "ى": 77, "ي": 78, "ً": 79, "ٌ": 80, "ٍ": 81, "َ": 82, "ُ": 83, "ِ": 84, "ّ": 85, "ْ": 86 }, "merges": [] } }