pgarbacki commited on
Commit
e637341
1 Parent(s): 487cbc0

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -2050,7 +2050,7 @@
2050
  }
2051
  },
2052
  "bos_token": "<|begin_of_text|>",
2053
- "chat_template": "{%- set loop_messages = messages -%}\n{%- set message_roles = ['system', 'user', 'assistant', 'tool'] -%}\n{%- set system_prompt_suffix -%}\n{%- filter trim -%}\nIn addition to plain text responses, you can chose to call one or more of the provided functions.\n\nUse the following rule to decide when to call a function:\n * if the response can be generated from your internal knowledge (e.g., as in the case of queries like \"What is the capital of Poland?\"), do so\n * if you need external information that can be obtained by calling one or more of the provided functions, generate a function calls\n\nIf you decide to call functions:\n * prefix function calls with functools marker (no closing marker required)\n * all function calls should be generated in a single JSON list formatted as functools[{\"name\": [function name], \"arguments\": [function arguments as JSON]}, ...]\n * follow the provided JSON schema. Do not hallucinate arguments or values. Do to blindly copy values from the provided samples\n * respect the argument type formatting. E.g., if the type if number and format is float, write value 7 as 7.0\n * make sure you pick the right functions that match the user intent\n\nAvailable functions as JSON spec:\n{%- endfilter -%}\n{%- endset -%}\n{%- set system_prompt_suffix = system_prompt_suffix + \"\\n\" + functions -%}\n{%- set system_prompt_suffix = system_prompt_suffix + '\\nToday is ' + datetime + '.' -%}\n{%- set ns = namespace(role='', content='') -%}\n{#- Basic consistency checks -#}\n{%- if not loop_messages -%}\n {{ raise_exception('Expected non-empty messages') }}\n{%- endif -%}\n{%- for message in loop_messages -%}\n {%- set ns.role = message['role'] | lower -%}\n {%- if ns.role not in message_roles -%}\n {{ raise_exception('Invalid role ' + message['role'] + '. Only ' + message_roles + ' are supported.') }}\n {%- endif -%}\n {%- set msg_content = message['content'] | default('', true) | trim -%}\n {%- if loop.index0 == 0 -%}\n {%- if ns.role == 'system' -%}\n {%- set system_prompt = '<|start_header_id|>' + 'system' + '<|end_header_id|>\\n\\n' + message['content'] | trim + '\\n' + system_prompt_suffix + '<|eot_id|>' -%}\n {%- else -%}\n {%- set system_prompt = '<|start_header_id|>' + 'system' + '<|end_header_id|>\\n\\nYou are a helpful assistant with access to functions.\\n' + system_prompt_suffix + '<|eot_id|>' -%}\n {%- endif -%}\n {%- set ns.content = bos_token + system_prompt -%}\n {{- ns.content -}}\n {%- endif -%}\n {%- if loop.index0 > 0 or ns.role != 'system' -%}\n {%- set ns.content = '<|start_header_id|>' + ns.role + '<|end_header_id|>\\n\\n' + msg_content -%}\n {%- if 'tool_calls' in message and message['tool_calls'] -%}\n {%- set tool = namespace(calls=[]) -%}\n {%- for call in message['tool_calls'] -%}\n {%- set tool.calls = tool.calls + ['{\"name\": \"' + call['function']['name'] + '\", \"arguments\": ' + call['function']['arguments'] + '}'] -%}\n {%- endfor -%}\n {%- set ns.content = ns.content + ' functools[' + tool.calls | join(', ') + ']' -%}\n {%- endif -%}\n {%- set ns.content = ns.content + '<|eot_id|>' -%}\n {{- ns.content -}}\n {%- endif -%}\n{%- endfor -%}\n{{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n",
2054
  "clean_up_tokenization_spaces": true,
2055
  "eos_token": "<|end_of_text|>",
2056
  "legacy": false,
 
2050
  }
2051
  },
2052
  "bos_token": "<|begin_of_text|>",
2053
+ "chat_template": "{%- set loop_messages = messages -%}\n{%- set message_roles = ['system', 'user', 'assistant', 'tool'] -%}\n{%- set system_prompt_suffix -%}\n{%- filter trim -%}\nIn addition to plain text responses, you can chose to call one or more of the provided functions.\n\nUse the following rule to decide when to call a function:\n * if the response can be generated from your internal knowledge (e.g., as in the case of queries like \"What is the capital of Poland?\"), do so\n * if you need external information that can be obtained by calling one or more of the provided functions, generate a function calls\n\nIf you decide to call functions:\n * prefix function calls with functools marker (no closing marker required)\n * all function calls should be generated in a single JSON list formatted as functools[{\"name\": [function name], \"arguments\": [function arguments as JSON]}, ...]\n * follow the provided JSON schema. Do not hallucinate arguments or values. Do to blindly copy values from the provided samples\n * respect the argument type formatting. E.g., if the type if number and format is float, write value 7 as 7.0\n * make sure you pick the right functions that match the user intent\n\nAvailable functions as JSON spec:\n{%- endfilter -%}\n{%- endset -%}\n{%- set system_prompt_suffix = system_prompt_suffix + \"\\n\" + functions -%}\n{%- set system_prompt_suffix = system_prompt_suffix + '\\nToday is ' + datetime + '.' -%}\n{%- set ns = namespace(role='', content='') -%}\n{#- Basic consistency checks -#}\n{%- if not loop_messages -%}\n {{ raise_exception('Expected non-empty messages') }}\n{%- endif -%}\n{%- for message in loop_messages -%}\n {%- set ns.role = message['role'] | lower -%}\n {%- if ns.role not in message_roles -%}\n {%- set message_roles_string = message_roles | join(', ') -%}\n {{ raise_exception('Invalid role ' + message['role'] + '. Only ' + message_roles_string + ' are supported.') }}\n {%- endif -%}\n {%- set msg_content = message['content'] | default('', true) | trim -%}\n {%- if loop.index0 == 0 -%}\n {%- if ns.role == 'system' -%}\n {%- set system_prompt = '<|start_header_id|>' + 'system' + '<|end_header_id|>\\n\\n' + message['content'] | trim + '\\n' + system_prompt_suffix + '<|eot_id|>' -%}\n {%- else -%}\n {%- set system_prompt = '<|start_header_id|>' + 'system' + '<|end_header_id|>\\n\\nYou are a helpful assistant with access to functions.\\n' + system_prompt_suffix + '<|eot_id|>' -%}\n {%- endif -%}\n {%- set ns.content = bos_token + system_prompt -%}\n {{- ns.content -}}\n {%- endif -%}\n {%- if loop.index0 > 0 or ns.role != 'system' -%}\n {%- set ns.content = '<|start_header_id|>' + ns.role + '<|end_header_id|>\\n\\n' + msg_content -%}\n {%- if 'tool_calls' in message and message['tool_calls'] -%}\n {%- set tool = namespace(calls=[]) -%}\n {%- for call in message['tool_calls'] -%}\n {%- set tool.calls = tool.calls + ['{\"name\": \"' + call['function']['name'] + '\", \"arguments\": ' + call['function']['arguments'] + '}'] -%}\n {%- endfor -%}\n {%- set ns.content = ns.content + ' functools[' + tool.calls | join(', ') + ']' -%}\n {%- endif -%}\n {%- set ns.content = ns.content + '<|eot_id|>' -%}\n {{- ns.content -}}\n {%- endif -%}\n{%- endfor -%}\n{{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n",
2054
  "clean_up_tokenization_spaces": true,
2055
  "eos_token": "<|end_of_text|>",
2056
  "legacy": false,