| { | |
| "version": "1.0", | |
| "truncation": null, | |
| "padding": null, | |
| "added_tokens": [ | |
| { | |
| "id": 0, | |
| "content": "<|endoftext|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 1, | |
| "content": "<|psep|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 2, | |
| "content": "</|psep|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 3, | |
| "content": "<|vsep|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 4, | |
| "content": "<|bsep|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 5, | |
| "content": "</|bsep|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 6, | |
| "content": "<|pad|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 7, | |
| "content": "<|meter_0|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 8, | |
| "content": "<|meter_1|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 9, | |
| "content": "<|meter_2|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 10, | |
| "content": "<|meter_3|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 11, | |
| "content": "<|meter_4|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 12, | |
| "content": "<|meter_5|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 13, | |
| "content": "<|meter_6|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 14, | |
| "content": "<|meter_7|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 15, | |
| "content": "<|meter_8|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 16, | |
| "content": "<|meter_9|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 17, | |
| "content": "<|meter_10|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 18, | |
| "content": "<|meter_11|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 19, | |
| "content": "<|meter_12|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 20, | |
| "content": "<|meter_13|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 21, | |
| "content": "<|meter_14|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 22, | |
| "content": "<|meter_15|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 23, | |
| "content": "<|meter_16|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 24, | |
| "content": "<|theme_0|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 25, | |
| "content": "<|theme_1|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 26, | |
| "content": "<|theme_2|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 27, | |
| "content": "<|theme_3|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 28, | |
| "content": "<|theme_4|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 29, | |
| "content": "<|theme_5|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 30, | |
| "content": "<|theme_6|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 31, | |
| "content": "<|theme_7|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 32, | |
| "content": "<|theme_8|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 33, | |
| "content": "<|theme_9|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 34, | |
| "content": "<|theme_10|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 35, | |
| "content": "<|theme_11|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 36, | |
| "content": "<|theme_12|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 37, | |
| "content": "<|theme_13|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 38, | |
| "content": "<|theme_14|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 39, | |
| "content": "<|theme_15|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 40, | |
| "content": "<|theme_16|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 41, | |
| "content": "<|theme_17|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 42, | |
| "content": "<|theme_18|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 43, | |
| "content": "<|res_0|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 44, | |
| "content": "<|res_1|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 45, | |
| "content": "<|res_2|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 46, | |
| "content": "<|res_3|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 47, | |
| "content": "<|res_4|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 48, | |
| "content": "<|res_5|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 49, | |
| "content": "<|res_6|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 50, | |
| "content": "<|res_7|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 51, | |
| "content": "<|res_8|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 52, | |
| "content": "<|res_9|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| } | |
| ], | |
| "normalizer": null, | |
| "pre_tokenizer": { | |
| "type": "UnicodeScripts" | |
| }, | |
| "post_processor": null, | |
| "decoder": null, | |
| "model": { | |
| "type": "BPE", | |
| "dropout": null, | |
| "unk_token": null, | |
| "continuing_subword_prefix": null, | |
| "end_of_word_suffix": null, | |
| "fuse_unk": false, | |
| "byte_fallback": false, | |
| "vocab": { | |
| "<|endoftext|>": 0, | |
| "<|psep|>": 1, | |
| "</|psep|>": 2, | |
| "<|vsep|>": 3, | |
| "<|bsep|>": 4, | |
| "</|bsep|>": 5, | |
| "<|pad|>": 6, | |
| "<|meter_0|>": 7, | |
| "<|meter_1|>": 8, | |
| "<|meter_2|>": 9, | |
| "<|meter_3|>": 10, | |
| "<|meter_4|>": 11, | |
| "<|meter_5|>": 12, | |
| "<|meter_6|>": 13, | |
| "<|meter_7|>": 14, | |
| "<|meter_8|>": 15, | |
| "<|meter_9|>": 16, | |
| "<|meter_10|>": 17, | |
| "<|meter_11|>": 18, | |
| "<|meter_12|>": 19, | |
| "<|meter_13|>": 20, | |
| "<|meter_14|>": 21, | |
| "<|meter_15|>": 22, | |
| "<|meter_16|>": 23, | |
| "<|theme_0|>": 24, | |
| "<|theme_1|>": 25, | |
| "<|theme_2|>": 26, | |
| "<|theme_3|>": 27, | |
| "<|theme_4|>": 28, | |
| "<|theme_5|>": 29, | |
| "<|theme_6|>": 30, | |
| "<|theme_7|>": 31, | |
| "<|theme_8|>": 32, | |
| "<|theme_9|>": 33, | |
| "<|theme_10|>": 34, | |
| "<|theme_11|>": 35, | |
| "<|theme_12|>": 36, | |
| "<|theme_13|>": 37, | |
| "<|theme_14|>": 38, | |
| "<|theme_15|>": 39, | |
| "<|theme_16|>": 40, | |
| "<|theme_17|>": 41, | |
| "<|theme_18|>": 42, | |
| "<|res_0|>": 43, | |
| "<|res_1|>": 44, | |
| "<|res_2|>": 45, | |
| "<|res_3|>": 46, | |
| "<|res_4|>": 47, | |
| "<|res_5|>": 48, | |
| "<|res_6|>": 49, | |
| "<|res_7|>": 50, | |
| "<|res_8|>": 51, | |
| "<|res_9|>": 52, | |
| " ": 53, | |
| "/": 54, | |
| "0": 55, | |
| "1": 56, | |
| "2": 57, | |
| "3": 58, | |
| "4": 59, | |
| "5": 60, | |
| "6": 61, | |
| "7": 62, | |
| "8": 63, | |
| "9": 64, | |
| "<": 65, | |
| ">": 66, | |
| "_": 67, | |
| "b": 68, | |
| "e": 69, | |
| "h": 70, | |
| "m": 71, | |
| "p": 72, | |
| "r": 73, | |
| "s": 74, | |
| "t": 75, | |
| "v": 76, | |
| "|": 77, | |
| "~": 78, | |
| "ء": 79, | |
| "أ": 80, | |
| "ؤ": 81, | |
| "ئ": 82, | |
| "ا": 83, | |
| "ب": 84, | |
| "ة": 85, | |
| "ت": 86, | |
| "ث": 87, | |
| "ج": 88, | |
| "ح": 89, | |
| "خ": 90, | |
| "د": 91, | |
| "ذ": 92, | |
| "ر": 93, | |
| "ز": 94, | |
| "س": 95, | |
| "ش": 96, | |
| "ص": 97, | |
| "ض": 98, | |
| "ط": 99, | |
| "ظ": 100, | |
| "ع": 101, | |
| "غ": 102, | |
| "ف": 103, | |
| "ق": 104, | |
| "ك": 105, | |
| "ل": 106, | |
| "م": 107, | |
| "ن": 108, | |
| "ه": 109, | |
| "و": 110, | |
| "ى": 111, | |
| "ي": 112, | |
| "ً": 113, | |
| "ٌ": 114, | |
| "ٍ": 115, | |
| "َ": 116, | |
| "ُ": 117, | |
| "ِ": 118, | |
| "ّ": 119, | |
| "ْ": 120 | |
| }, | |
| "merges": [] | |
| } | |
| } |