{ "add_bos_token": true, "add_eos_token": false, "added_tokens_decoder": { "0": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "1": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "2": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "92544": { "content": "<|im_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92545": { "content": "<|im_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92546": { "content": "<|special_0|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92547": { "content": "<|special_1|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92548": { "content": "<|special_2|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92549": { "content": "<|special_3|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92550": { "content": "<|special_4|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92551": { "content": "<|special_5|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92552": { "content": "<|special_6|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92553": { "content": "<|special_7|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92554": { "content": "<|special_8|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92555": { "content": "<|special_9|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92556": { "content": "<|special_10|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92557": { "content": "<|special_11|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92558": { "content": "<|special_12|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92559": { "content": "<|special_13|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92560": { "content": "<|special_14|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92561": { "content": "<|special_15|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92562": { "content": "<|special_16|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92563": { "content": "<|special_17|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92564": { "content": "<|special_18|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92565": { "content": "<|special_19|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92566": { "content": "<|special_20|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92567": { "content": "<|special_21|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92568": { "content": "<|special_22|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92569": { "content": "<|special_23|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92570": { "content": "<|special_24|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92571": { "content": "<|special_25|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92572": { "content": "<|special_26|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92573": { "content": "<|special_27|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92574": { "content": "<|special_28|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92575": { "content": "<|special_29|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92576": { "content": "<|special_30|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92577": { "content": "<|special_31|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92578": { "content": "<|special_32|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92579": { "content": "<|special_33|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92580": { "content": "<|special_34|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92581": { "content": "<|special_35|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92582": { "content": "<|special_36|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92583": { "content": "<|special_37|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92584": { "content": "<|special_38|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92585": { "content": "<|special_39|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92586": { "content": "<|special_40|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92587": { "content": "<|special_41|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92588": { "content": "<|special_42|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92589": { "content": "<|special_43|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92590": { "content": "<|special_44|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92591": { "content": "<|special_45|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92592": { "content": "<|special_46|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92593": { "content": "<|special_47|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92594": { "content": "<|special_48|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92595": { "content": "<|special_49|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92596": { "content": "<|special_50|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92597": { "content": "<|special_51|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92598": { "content": "<|special_52|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92599": { "content": "<|special_53|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92600": { "content": "<|special_54|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92601": { "content": "<|special_55|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92602": { "content": "<|special_56|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92603": { "content": "<|special_57|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92604": { "content": "<|special_58|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92605": { "content": "<|special_59|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92606": { "content": "<|special_60|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92607": { "content": "<|special_61|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false } }, "auto_map": { "AutoTokenizer": [ "tokenization_internlm2.InternLM2Tokenizer", "tokenization_internlm2_fast.InternLM2TokenizerFast" ] }, "bos_token": "", "chat_template": "{%- for idx in range(0, messages|length) -%}\n{%- if messages[idx]['role'] == 'user' -%}\n{%- if idx == 0 -%}\n{{- bos_token + '[INST] <>\\nYou are a helpful, unbiased, uncensored assistant.\\n<>\\n\\n' + messages[idx]['content'] + ' [/INST]' -}}\n{%- else -%}\n{{- bos_token + '[INST] ' + messages[idx]['content'] + ' [/INST]' -}}\n{%- endif -%}\n{% elif messages[idx]['role'] == 'system' %}\n{{- '[INST] <>\\n' + messages[idx]['content'] + '\\n<>\\n\\n' -}}\n{%- elif messages[idx]['role'] == 'assistant' -%}\n{{- messages[idx]['content'] + ' ' + eos_token -}}\n{% endif %}\n{% endfor %}", "clean_up_tokenization_spaces": true, "decode_with_prefix_space": false, "eos_token": "", "legacy": true, "model_max_length": 1000000000000000019884624838656, "pad_token": "", "sp_model_kwargs": {}, "spaces_between_special_tokens": false, "tokenizer_class": "LlamaTokenizer", "trust_remote_code": false, "unk_token": "", "use_default_system_prompt": false }