Bohanlu commited on
Commit
90f0c55
1 Parent(s): 07d73d0

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +0 -1
tokenizer_config.json CHANGED
@@ -37,7 +37,6 @@
37
  }
38
  },
39
  "bos_token": "<s>",
40
- "chat_template": "{% if messages[0]['role'] == 'system' %}\n {% set loop_messages = messages[1:] %}\n {% set system_message = messages[0]['content'] %}\n{% else %}\n {% set loop_messages = messages %}\n {% set system_message = false %}\n{% endif %}\n{% if system_message != false and (loop_messages|length == 0 or loop_messages|length == 1 and loop_messages[0]['role'] != 'user') %}\n {{ raise_exception('The system prompt must be passed along with a user prompt.') }}\n{% endif %}\n{% for message in loop_messages %}\n {% if loop_messages|length > 1 and (message['role'] == 'user') != (loop.index0 % 2 == 0) %}\n {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}\n {% endif %}\n {% if loop.index0 == 0 and system_message != false %}\n {% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}\n {% else %}\n {% set content = message['content'] %}\n {% endif %}\n {% if message['role'] == 'user' %}\n {{- bos_token + '[TRANS]\\n' + content + '\\n[/TRANS]\\n' -}}\n {% elif message['role'] == 'system' %}\n {{- '<<SYS>>\\n' + content + '\\n<</SYS>>\\n\\n' -}}\n {% elif message['role'] == 'assistant' %}\n {{- content + eos_token -}}\n {% endif %}\n{% endfor %}\n",
41
  "clean_up_tokenization_spaces": false,
42
  "eos_token": "</s>",
43
  "legacy": false,
 
37
  }
38
  },
39
  "bos_token": "<s>",
 
40
  "clean_up_tokenization_spaces": false,
41
  "eos_token": "</s>",
42
  "legacy": false,