{ "added_tokens_decoder": { "0": { "content": "<pad>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": true }, "1": { "content": "<bos>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": true }, "2": { "content": "<eos>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": true }, "3": { "content": "<unk>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": true }, "4": { "content": "<sep>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": true }, "5": { "content": "<repo_name>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "6": { "content": "<file_sep>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "7": { "content": "<t2v_token>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "8": { "content": "<v2t_token>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "9": { "content": "<|start_header_id|>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "10": { "content": "<|end_header_id|>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "11": { "content": "<|eot_id|>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "12": { "content": "<extra_id_12>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "13": { "content": "<extra_id_13>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "14": { "content": "<extra_id_14>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "15": { "content": "<extra_id_15>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "16": { "content": "<extra_id_16>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "17": { "content": "<extra_id_17>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "18": { "content": "<extra_id_18>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "19": { "content": "<extra_id_19>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "20": { "content": "<extra_id_20>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "21": { "content": "<extra_id_21>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "22": { "content": "<extra_id_22>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "23": { "content": "<extra_id_23>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "24": { "content": "<extra_id_24>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "25": { "content": "<extra_id_25>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "26": { "content": "<extra_id_26>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "27": { "content": "<extra_id_27>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "28": { "content": "<extra_id_28>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "29": { "content": "<extra_id_29>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "30": { "content": "<extra_id_30>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "31": { "content": "<extra_id_31>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "32": { "content": "<extra_id_32>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "33": { "content": "<extra_id_33>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "34": { "content": "<extra_id_34>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "35": { "content": "<extra_id_35>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "36": { "content": "<extra_id_36>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "37": { "content": "<extra_id_37>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "38": { "content": "<extra_id_38>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "39": { "content": "<extra_id_39>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "40": { "content": "<extra_id_40>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "41": { "content": "<extra_id_41>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "42": { "content": "<extra_id_42>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "43": { "content": "<extra_id_43>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "44": { "content": "<extra_id_44>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "45": { "content": "<extra_id_45>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "46": { "content": "<extra_id_46>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "47": { "content": "<extra_id_47>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "48": { "content": "<extra_id_48>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "49": { "content": "<extra_id_49>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50": { "content": "<extra_id_50>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "51": { "content": "<extra_id_51>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "52": { "content": "<extra_id_52>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "53": { "content": "<extra_id_53>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "54": { "content": "<extra_id_54>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "55": { "content": "<extra_id_55>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "56": { "content": "<extra_id_56>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "57": { "content": "<extra_id_57>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "58": { "content": "<extra_id_58>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "59": { "content": "<extra_id_59>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "60": { "content": "<extra_id_60>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "61": { "content": "<extra_id_61>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "62": { "content": "<extra_id_62>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "63": { "content": "<extra_id_63>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false } }, "additional_special_tokens": [ "<repo_name>", "<file_sep>", "<t2v_token>", "<v2t_token>", "<|start_header_id|>", "<|end_header_id|>", "<|eot_id|>", "<extra_id_12>", "<extra_id_13>", "<extra_id_14>", "<extra_id_15>", "<extra_id_16>", "<extra_id_17>", "<extra_id_18>", "<extra_id_19>", "<extra_id_20>", "<extra_id_21>", "<extra_id_22>", "<extra_id_23>", "<extra_id_24>", "<extra_id_25>", "<extra_id_26>", "<extra_id_27>", "<extra_id_28>", "<extra_id_29>", "<extra_id_30>", "<extra_id_31>", "<extra_id_32>", "<extra_id_33>", "<extra_id_34>", "<extra_id_35>", "<extra_id_36>", "<extra_id_37>", "<extra_id_38>", "<extra_id_39>", "<extra_id_40>", "<extra_id_41>", "<extra_id_42>", "<extra_id_43>", "<extra_id_44>", "<extra_id_45>", "<extra_id_46>", "<extra_id_47>", "<extra_id_48>", "<extra_id_49>", "<extra_id_50>", "<extra_id_51>", "<extra_id_52>", "<extra_id_53>", "<extra_id_54>", "<extra_id_55>", "<extra_id_56>", "<extra_id_57>", "<extra_id_58>", "<extra_id_59>", "<extra_id_60>", "<extra_id_61>", "<extra_id_62>", "<extra_id_63>" ], "auto_map": { "AutoProcessor": "processing_evabyte.EvaByteProcessor", "AutoTokenizer": [ "tokenization_evabyte.EvaByteTokenizer", null ] }, "bos_token": "<bos>", "chat_template": "\n{{- bos_token }}\n{%- if messages[0]['role'] == 'system' %}\n {%- set system_message = messages[0]['content'] %}\n {%- set messages = messages[1:] %}\n{%- else %}\n {%- set system_message = \"\" %}\n{%- endif %}\n\n{{- '<|start_header_id|>system<|end_header_id|>\n\n' + system_message + '<|eot_id|>'}}\n\n{%- for message in messages %}\n {%- if (message['role'] != 'user') and (message['role'] != 'assistant') %}\n {{- raise_exception('Conversation roles must be user or assistant') }}\n {%- endif %}\n\n {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] + '<|eot_id|>' }}\n{%- endfor %}\n\n{%- if add_generation_prompt %}\n {{- '<|start_header_id|>' + 'assistant' + '<|end_header_id|>\n\n' }}\n{%- endif %}\n", "clean_up_tokenization_spaces": false, "eos_token": "<|eot_id|>", "extra_ids": 0, "extra_special_tokens": {}, "model_max_length": 1000000000000000019884624838656, "pad_token": "<pad>", "processor_class": "EvaByteProcessor", "sep_token": "<eos>", "tokenizer_class": "EvaByteTokenizer", "unk_token": "<unk>" }