Update tokenizer_config.json
Browse files- tokenizer_config.json +1 -23
tokenizer_config.json
CHANGED
@@ -34,29 +34,7 @@
|
|
34 |
"</s>"
|
35 |
],
|
36 |
"bos_token": "<s>",
|
37 |
-
"chat_template": "{
|
38 |
-
{%- for message in messages -%}
|
39 |
-
{%- if message['role'] == 'system' -%}
|
40 |
-
{%- set ns.found = true -%}
|
41 |
-
{%- endif -%}
|
42 |
-
{%- endfor -%}
|
43 |
-
{%- if not ns.found -%}
|
44 |
-
{{- '' + 'Below is an instruction that describes a task. Write a response that appropriately completes the request.' + '\n\n' -}}
|
45 |
-
{%- endif %}
|
46 |
-
{%- for message in messages %}
|
47 |
-
{%- if message['role'] == 'system' -%}
|
48 |
-
{{- '' + message['content'] + '\n\n' -}}
|
49 |
-
{%- else -%}
|
50 |
-
{%- if message['role'] == 'user' -%}
|
51 |
-
{{-'### Instruction:\n' + message['content'] + '\n\n'-}}
|
52 |
-
{%- else -%}
|
53 |
-
{{-'### Response:\n' + message['content'] + '\n\n' -}}
|
54 |
-
{%- endif -%}
|
55 |
-
{%- endif -%}
|
56 |
-
{%- endfor -%}
|
57 |
-
{%- if add_generation_prompt -%}
|
58 |
-
{{-'### Response:\n'-}}
|
59 |
-
{%- endif -%}",
|
60 |
"clean_up_tokenization_spaces": false,
|
61 |
"eos_token": "</s>",
|
62 |
"legacy": true,
|
|
|
34 |
"</s>"
|
35 |
],
|
36 |
"bos_token": "<s>",
|
37 |
+
"chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '### Instruction:\n' + message['content'] }}\n{% elif message['role'] == 'system' %}\n{{ '### System:\n' + message['content'] }}\n{% elif message['role'] == 'assistant' %}\n{{ '### Response:\n' + message['content'] }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '### Response:' }}\n{% endif %}\n{% endfor %}",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
38 |
"clean_up_tokenization_spaces": false,
|
39 |
"eos_token": "</s>",
|
40 |
"legacy": true,
|