mgoin commited on
Commit
7ed3fbd
·
verified ·
1 Parent(s): e62c760

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -0
tokenizer_config.json CHANGED
@@ -6747,6 +6747,7 @@
6747
  "[control_746]"
6748
  ],
6749
  "bos_token": "<s>",
 
6750
  "clean_up_tokenization_spaces": false,
6751
  "eos_token": "</s>",
6752
  "extra_special_tokens": {},
 
6747
  "[control_746]"
6748
  ],
6749
  "bos_token": "<s>",
6750
+ "chat_template": "{%- if messages[0][\"role\"] == \"system\" %}{%- set system_message = messages[0][\"content\"] %}{%- set loop_messages = messages[1:] %}\n{%- else %}{%- set loop_messages = messages %}{%- endif %}{{- bos_token }}{%- for message in loop_messages %}{%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{- raise_exception('After the optional system message, conversation roles must alternate user/assistant/user/assistant/...') }}{%- endif %}{%- if message[\"role\"] == \"user\" %}{%- if loop.last and system_message is defined %}{{- \"[INST]\" + system_message + \"\n\n\" }}{%- else %}{{ \"[INST]\" }}{%- endif %}{%- endif %}{%- if message[\"content\"] is not string %}{%- for chunk in message[\"content\"] %}{%- if chunk[\"type\"] == \"text\" %}{%- if \"content\" in chunk %}{{- chunk[\"content\"] }}{%- elif \"text\" in chunk %}{{- chunk[\"text\"] }}{%- endif %}{%- elif chunk[\"type\"] == \"image\" %}{{- \"[IMG]\" }}{%- else %}{{- raise_exception(\"Unrecognized content type!\") }}{%- endif %}{%- endfor %}{%- else %}{{- message[\"content\"] }}{%- endif %}{%- if message[\"role\"] == \"user\" %}{{- \"[/INST]\" }}{%- elif message[\"role\"] == \"assistant\" %}{{- eos_token}}{%- else %}{{- raise_exception(\"Only user and assistant roles are supported, with the exception of an initial optional system message!\") }}{%- endif %}{%- endfor %}",
6751
  "clean_up_tokenization_spaces": false,
6752
  "eos_token": "</s>",
6753
  "extra_special_tokens": {},