{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "<|pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "<|bos|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "<|eos|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "<|unk|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 4, "content": "<|system|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 5, "content": "<|user|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 6, "content": "<|assistant|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 7, "content": "<|plan|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 8, "content": "<|tool_call|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 9, "content": "<|tool_result|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 10, "content": "<|code|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 11, "content": "<|/code|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 12, "content": "<|file|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 13, "content": "<|diff|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 14, "content": "<|error|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 15, "content": "<|fix|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 16, "content": "<|python|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 17, "content": "<|javascript|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 18, "content": "<|typescript|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 19, "content": "<|cpp|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 20, "content": "<|java|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 21, "content": "<|go|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 22, "content": "<|rust|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 23, "content": "<|end_tool|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": true }, "post_processor": { "type": "TemplateProcessing", "single": [ { "Sequence": { "id": "A", "type_id": 0 } } ], "pair": [ { "Sequence": { "id": "A", "type_id": 0 } }, { "Sequence": { "id": "B", "type_id": 1 } } ], "special_tokens": {} }, "decoder": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "model": { "type": "BPE", "dropout": null, "unk_token": "<|unk|>", "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "ignore_merges": false, "vocab": { "<|pad|>": 0, "<|bos|>": 1, "<|eos|>": 2, "<|unk|>": 3, "<|system|>": 4, "<|user|>": 5, "<|assistant|>": 6, "<|plan|>": 7, "<|tool_call|>": 8, "<|tool_result|>": 9, "<|code|>": 10, "<|/code|>": 11, "<|file|>": 12, "<|diff|>": 13, "<|error|>": 14, "<|fix|>": 15, "<|python|>": 16, "<|javascript|>": 17, "<|typescript|>": 18, "<|cpp|>": 19, "<|java|>": 20, "<|go|>": 21, "<|rust|>": 22, "<|end_tool|>": 23, "!": 24, "\"": 25, "#": 26, "$": 27, "%": 28, "&": 29, "'": 30, "(": 31, ")": 32, "*": 33, "+": 34, ",": 35, "-": 36, ".": 37, "/": 38, "0": 39, "1": 40, "2": 41, "3": 42, "4": 43, "5": 44, "6": 45, "7": 46, "8": 47, "9": 48, ":": 49, ";": 50, "<": 51, "=": 52, ">": 53, "?": 54, "@": 55, "A": 56, "B": 57, "C": 58, "D": 59, "E": 60, "F": 61, "G": 62, "H": 63, "I": 64, "J": 65, "K": 66, "L": 67, "M": 68, "N": 69, "O": 70, "P": 71, "Q": 72, "R": 73, "S": 74, "T": 75, "U": 76, "V": 77, "W": 78, "X": 79, "Y": 80, "Z": 81, "[": 82, "\\": 83, "]": 84, "^": 85, "_": 86, "`": 87, "a": 88, "b": 89, "c": 90, "d": 91, "e": 92, "f": 93, "g": 94, "h": 95, "i": 96, "j": 97, "k": 98, "l": 99, "m": 100, "n": 101, "o": 102, "p": 103, "q": 104, "r": 105, "s": 106, "t": 107, "u": 108, "v": 109, "w": 110, "x": 111, "y": 112, "z": 113, "{": 114, "|": 115, "}": 116, "~": 117, "¡": 118, "¢": 119, "£": 120, "¤": 121, "¥": 122, "¦": 123, "§": 124, "¨": 125, "©": 126, "ª": 127, "«": 128, "¬": 129, "®": 130, "¯": 131, "°": 132, "±": 133, "²": 134, "³": 135, "´": 136, "µ": 137, "¶": 138, "·": 139, "¸": 140, "¹": 141, "º": 142, "»": 143, "¼": 144, "½": 145, "¾": 146, "¿": 147, "À": 148, "Á": 149, "Â": 150, "Ã": 151, "Ä": 152, "Å": 153, "Æ": 154, "Ç": 155, "È": 156, "É": 157, "Ê": 158, "Ë": 159, "Ì": 160, "Í": 161, "Î": 162, "Ï": 163, "Ð": 164, "Ñ": 165, "Ò": 166, "Ó": 167, "Ô": 168, "Õ": 169, "Ö": 170, "×": 171, "Ø": 172, "Ù": 173, "Ú": 174, "Û": 175, "Ü": 176, "Ý": 177, "Þ": 178, "ß": 179, "à": 180, "á": 181, "â": 182, "ã": 183, "ä": 184, "å": 185, "æ": 186, "ç": 187, "è": 188, "é": 189, "ê": 190, "ë": 191, "ì": 192, "í": 193, "î": 194, "ï": 195, "ð": 196, "ñ": 197, "ò": 198, "ó": 199, "ô": 200, "õ": 201, "ö": 202, "÷": 203, "ø": 204, "ù": 205, "ú": 206, "û": 207, "ü": 208, "ý": 209, "þ": 210, "ÿ": 211, "Ā": 212, "ā": 213, "Ă": 214, "ă": 215, "Ą": 216, "ą": 217, "Ć": 218, "ć": 219, "Ĉ": 220, "ĉ": 221, "Ċ": 222, "ċ": 223, "Č": 224, "č": 225, "Ď": 226, "ď": 227, "Đ": 228, "đ": 229, "Ē": 230, "ē": 231, "Ĕ": 232, "ĕ": 233, "Ė": 234, "ė": 235, "Ę": 236, "ę": 237, "Ě": 238, "ě": 239, "Ĝ": 240, "ĝ": 241, "Ğ": 242, "ğ": 243, "Ġ": 244, "ġ": 245, "Ģ": 246, "ģ": 247, "Ĥ": 248, "ĥ": 249, "Ħ": 250, "ħ": 251, "Ĩ": 252, "ĩ": 253, "Ī": 254, "ī": 255, "Ĭ": 256, "ĭ": 257, "Į": 258, "į": 259, "İ": 260, "ı": 261, "IJ": 262, "ij": 263, "Ĵ": 264, "ĵ": 265, "Ķ": 266, "ķ": 267, "ĸ": 268, "Ĺ": 269, "ĺ": 270, "Ļ": 271, "ļ": 272, "Ľ": 273, "ľ": 274, "Ŀ": 275, "ŀ": 276, "Ł": 277, "ł": 278, "Ń": 279 }, "merges": [] } }