Update tokenizer_config.json
Browse files- tokenizer_config.json +23 -1
tokenizer_config.json
CHANGED
@@ -34,7 +34,29 @@
|
|
34 |
"</s>"
|
35 |
],
|
36 |
"bos_token": "<s>",
|
37 |
-
"chat_template": "{%-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
38 |
"clean_up_tokenization_spaces": false,
|
39 |
"eos_token": "</s>",
|
40 |
"legacy": true,
|
|
|
34 |
"</s>"
|
35 |
],
|
36 |
"bos_token": "<s>",
|
37 |
+
"chat_template": "{%- set ns = namespace(found=false) -%}
|
38 |
+
{%- for message in messages -%}
|
39 |
+
{%- if message['role'] == 'system' -%}
|
40 |
+
{%- set ns.found = true -%}
|
41 |
+
{%- endif -%}
|
42 |
+
{%- endfor -%}
|
43 |
+
{%- if not ns.found -%}
|
44 |
+
{{- '' + 'Below is an instruction that describes a task. Write a response that appropriately completes the request.' + '\n\n' -}}
|
45 |
+
{%- endif %}
|
46 |
+
{%- for message in messages %}
|
47 |
+
{%- if message['role'] == 'system' -%}
|
48 |
+
{{- '' + message['content'] + '\n\n' -}}
|
49 |
+
{%- else -%}
|
50 |
+
{%- if message['role'] == 'user' -%}
|
51 |
+
{{-'### Instruction:\n' + message['content'] + '\n\n'-}}
|
52 |
+
{%- else -%}
|
53 |
+
{{-'### Response:\n' + message['content'] + '\n\n' -}}
|
54 |
+
{%- endif -%}
|
55 |
+
{%- endif -%}
|
56 |
+
{%- endfor -%}
|
57 |
+
{%- if add_generation_prompt -%}
|
58 |
+
{{-'### Response:\n'-}}
|
59 |
+
{%- endif -%}",
|
60 |
"clean_up_tokenization_spaces": false,
|
61 |
"eos_token": "</s>",
|
62 |
"legacy": true,
|