gorilla-openfunctions-v2 / tokenizer_config.json
CharlieJi's picture
Update tokenizer_config.json, correcting DeepSeek to Gorilla LLM
1f6ac3b verified
{
"added_tokens_decoder": {
"100000": {
"content": "<|begin▁of▁sentence|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
},
"100001": {
"content": "<|end▁of▁sentence|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
},
"100002": {
"content": "ø",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"100003": {
"content": "ö",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"100004": {
"content": "ú",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"100005": {
"content": "ÿ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"100006": {
"content": "õ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"100007": {
"content": "÷",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"100008": {
"content": "û",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"100009": {
"content": "ý",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"100010": {
"content": "À",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"100011": {
"content": "ù",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"100012": {
"content": "Á",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"100013": {
"content": "þ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"100014": {
"content": "ü",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"100015": {
"content": "<|EOT|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
}
},
"bos_token": "<|begin▁of▁sentence|>",
"chat_template": "{% if not add_generation_prompt is defined %}\n{% set add_generation_prompt = false %}\n{% endif %}\n{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set ns.found = true -%}\n {%- endif -%}\n{%- endfor -%}\n{{bos_token}}{%- if not ns.found -%}\n{{'You are an AI programming assistant, utilizing the Gorilla LLM model, developed by Gorilla LLM, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\\n'}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' %}\n{{ message['content'] }}\n {%- else %}\n {%- if message['role'] == 'user' %}\n{{'### Instruction:\\n' + message['content'] + '\\n'}}\n {%- else %}\n{{'### Response:\\n' + message['content'] + '\\n<|EOT|>\\n'}}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{% if add_generation_prompt %}\n{{'### Response:'}}\n{% endif %}",
"clean_up_tokenization_spaces": false,
"eos_token": "<|EOT|>",
"legacy": true,
"model_max_length": 4096,
"pad_token": "<|end▁of▁sentence|>",
"padding_side": "right",
"sp_model_kwargs": {},
"tokenizer_class": "LlamaTokenizer",
"unk_token": null,
"use_default_system_prompt": true
}