zhengr commited on
Commit
05dc72d
1 Parent(s): 7190665

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -23
tokenizer_config.json CHANGED
@@ -34,29 +34,7 @@
34
  "</s>"
35
  ],
36
  "bos_token": "<s>",
37
- "chat_template": "{%- set ns = namespace(found=false) -%}
38
- {%- for message in messages -%}
39
- {%- if message['role'] == 'system' -%}
40
- {%- set ns.found = true -%}
41
- {%- endif -%}
42
- {%- endfor -%}
43
- {%- if not ns.found -%}
44
- {{- '' + 'Below is an instruction that describes a task. Write a response that appropriately completes the request.' + '\n\n' -}}
45
- {%- endif %}
46
- {%- for message in messages %}
47
- {%- if message['role'] == 'system' -%}
48
- {{- '' + message['content'] + '\n\n' -}}
49
- {%- else -%}
50
- {%- if message['role'] == 'user' -%}
51
- {{-'### Instruction:\n' + message['content'] + '\n\n'-}}
52
- {%- else -%}
53
- {{-'### Response:\n' + message['content'] + '\n\n' -}}
54
- {%- endif -%}
55
- {%- endif -%}
56
- {%- endfor -%}
57
- {%- if add_generation_prompt -%}
58
- {{-'### Response:\n'-}}
59
- {%- endif -%}",
60
  "clean_up_tokenization_spaces": false,
61
  "eos_token": "</s>",
62
  "legacy": true,
 
34
  "</s>"
35
  ],
36
  "bos_token": "<s>",
37
+ "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '### Instruction:\n' + message['content'] }}\n{% elif message['role'] == 'system' %}\n{{ '### System:\n' + message['content'] }}\n{% elif message['role'] == 'assistant' %}\n{{ '### Response:\n' + message['content'] }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '### Response:' }}\n{% endif %}\n{% endfor %}",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
38
  "clean_up_tokenization_spaces": false,
39
  "eos_token": "</s>",
40
  "legacy": true,