Update tokenizer_config.json

tokenizer_config.json

@@ -2050,7 +2050,7 @@
     }
   },
   "bos_token": "<|begin_of_text|>",
-  "chat_template": "{# version=v3-llama3.1 #}{%- if not tools is defined -%}\n    {%- set tools = none -%}\n{%- endif -%}\n\n{%- set has_code_interpreter = tools | selectattr(\"type\", \"equalto\", \"code_interpreter\") | list | length > 0 -%}\n{%- if has_code_interpreter -%}\n    {%- set tools = tools | rejectattr(\"type\", \"equalto\", \"code_interpreter\") | list -%}\n{%- endif -%}\n\n{#- System message + builtin tools #}\n{{- bos_token + \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if has_code_interpreter %}\n    {{- \"Environment: ipython\\n\\n\" }}\n{%- else -%}\n    {{ \"\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\\n\" }}\n{%- if tools %}\n    {{- \"\\nYou have access to the following functions:\\n\\n\" }}\n    {%- for t in tools %}\n        {%- if \"type\" in t -%}\n            {{ \"Use the function '\" + t[\"function\"][\"name\"] + \"' to '\" + t[\"function\"][\"description\"] + \"'\\n\" + t[\"function\"] | tojson() }}\n        {%- else -%}\n            {{ \"Use the function '\" + t[\"name\"] + \"' to '\" + t[\"description\"] + \"'\\n\" + t | tojson() }}\n        {%- endif -%}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- '\\nThink very carefully before calling functions.\\nIf a you choose to call a function ONLY reply in the following format:\\n<{start_tag}={function_name}>{parameters}{end_tag}\\nwhere\\n\\nstart_tag => `<function`\\nparameters => a JSON dict with the function argument name as key and function argument value as value.\\nend_tag => `</function>`\\n\\nHere is an example,\\n<function=example_function_name>{\"example_name\": \"example_value\"}</function>\\n\\nReminder:\\n- If looking for real time information use relevant functions before falling back to brave_search\\n- Function calls MUST follow the specified format, start with <function= and end with </function>\\n- Required parameters MUST be specified\\n- Only call one function at a time\\n- Put the entire function call reply on one line\\n\\n' -}}\n{%- endif %}\n{{- \"<|eot_id|>\" -}}\n\n{%- for message in messages -%}\n    {%- if message['role'] == 'user' or message['role'] == 'system' -%}\n        {{ '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n' + message['content'] + '<|eot_id|>' }}\n    {%- elif message['role'] == 'tool' -%}\n        {{ '<|start_header_id|>ipython<|end_header_id|>\\n\\n' + message['content'] + '<|eot_id|>' }}\n    {%- else -%}\n        {{ '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'}}\n        {%- if message['content'] is not none -%}\n            {{ message['content'] }}\n        {%- endif -%}\n        {%- if 'tool_calls' in message and message['tool_calls'] is not none -%}\n            {%- for tool_call in message['tool_calls'] -%}\n                {{ '<function=' + tool_call['function']['name'] + '>' + tool_call['function']['arguments'] + '</function>' }}\n            {%- endfor -%}\n            {{ '<|eom_id|>' }}\n        {%- else -%}\n            {{ '<|eot_id|>' }}\n        {%- endif -%}\n    {%- endif -%}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n    {{ '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif -%}",
+  "chat_template": "{# version=v3-llama3.1 #}{%- if not tools is defined -%}\n    {%- set tools = none -%}\n{%- endif -%}\n\n{%- set has_code_interpreter = tools | selectattr(\"type\", \"equalto\", \"code_interpreter\") | list | length > 0 -%}\n{%- if has_code_interpreter -%}\n    {%- set tools = tools | rejectattr(\"type\", \"equalto\", \"code_interpreter\") | list -%}\n{%- endif -%}\n\n{#- System message + builtin tools #}\n{{- bos_token + \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if has_code_interpreter %}\n    {{- \"Environment: ipython\\n\\n\" }}\n{%- else -%}\n    {{ \"\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\\n\" }}\n{%- if tools %}\n    {{- \"\\nYou have access to the following functions:\\n\\n\" }}\n    {%- for t in tools %}\n        {%- if \"type\" in t -%}\n            {{ \"Use the function '\"|safe + t[\"function\"][\"name\"] + \"' to '\"|safe + t[\"function\"][\"description\"] + \"'\\n\"|safe + t[\"function\"] | tojson() }}\n        {%- else -%}\n            {{ \"Use the function '\"|safe + t[\"name\"] + \"' to '\"|safe + t[\"description\"] + \"'\\n\"|safe + t | tojson() }}\n        {%- endif -%}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- '\\nThink very carefully before calling functions.\\nIf a you choose to call a function ONLY reply in the following format:\\n<{start_tag}={function_name}>{parameters}{end_tag}\\nwhere\\n\\nstart_tag => `<function`\\nparameters => a JSON dict with the function argument name as key and function argument value as value.\\nend_tag => `</function>`\\n\\nHere is an example,\\n<function=example_function_name>{\"example_name\": \"example_value\"}</function>\\n\\nReminder:\\n- If looking for real time information use relevant functions before falling back to brave_search\\n- Function calls MUST follow the specified format, start with <function= and end with </function>\\n- Required parameters MUST be specified\\n- Only call one function at a time\\n- Put the entire function call reply on one line\\n\\n' -}}\n{%- endif %}\n{{- \"<|eot_id|>\" -}}\n\n{%- for message in messages -%}\n    {%- if message['role'] == 'user' or message['role'] == 'system' -%}\n        {{ '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n' + message['content'] + '<|eot_id|>' }}\n    {%- elif message['role'] == 'tool' -%}\n        {{ '<|start_header_id|>ipython<|end_header_id|>\\n\\n' + message['content'] + '<|eot_id|>' }}\n    {%- else -%}\n        {{ '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'}}\n        {%- if message['content'] -%}\n            {{ message['content'] }}\n        {%- endif -%}\n        {%- if 'tool_calls' in message and message['tool_calls'] -%}\n            {%- for tool_call in message['tool_calls'] -%}\n                {%- if tool_call[\"function\"][\"name\"] == \"python\" -%}\n                    {{ '<|python_tag|>' + tool_call['function']['arguments'] }}\n                {%- else -%}\n                    {{ '<function=' + tool_call['function']['name'] + '>' + tool_call['function']['arguments'] + '</function>' }}\n                {%- endif -%}\n            {%- endfor -%}\n            {{ '<|eom_id|>' }}\n        {%- else -%}\n            {{ '<|eot_id|>' }}\n        {%- endif -%}\n    {%- endif -%}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n    {{ '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif -%}",
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|eot_id|>",
   "model_input_names": [