zhengr commited on
Commit
7190665
1 Parent(s): 08ee3be

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +23 -1
tokenizer_config.json CHANGED
@@ -34,7 +34,29 @@
34
  "</s>"
35
  ],
36
  "bos_token": "<s>",
37
- "chat_template": "{%- if message['role'] == 'system' -%}{{- '' + message['content'] + '\n\n' -}}{%- else -%}{%- if message['role'] == 'user' -%}{{-'### Instruction:\n' + message['content'] + '\n\n'-}}{%- else -%}{{-'### Response:\n' + message['content'] + '\n\n' -}}{%- endif -%}{%- endif -%}",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
38
  "clean_up_tokenization_spaces": false,
39
  "eos_token": "</s>",
40
  "legacy": true,
 
34
  "</s>"
35
  ],
36
  "bos_token": "<s>",
37
+ "chat_template": "{%- set ns = namespace(found=false) -%}
38
+ {%- for message in messages -%}
39
+ {%- if message['role'] == 'system' -%}
40
+ {%- set ns.found = true -%}
41
+ {%- endif -%}
42
+ {%- endfor -%}
43
+ {%- if not ns.found -%}
44
+ {{- '' + 'Below is an instruction that describes a task. Write a response that appropriately completes the request.' + '\n\n' -}}
45
+ {%- endif %}
46
+ {%- for message in messages %}
47
+ {%- if message['role'] == 'system' -%}
48
+ {{- '' + message['content'] + '\n\n' -}}
49
+ {%- else -%}
50
+ {%- if message['role'] == 'user' -%}
51
+ {{-'### Instruction:\n' + message['content'] + '\n\n'-}}
52
+ {%- else -%}
53
+ {{-'### Response:\n' + message['content'] + '\n\n' -}}
54
+ {%- endif -%}
55
+ {%- endif -%}
56
+ {%- endfor -%}
57
+ {%- if add_generation_prompt -%}
58
+ {{-'### Response:\n'-}}
59
+ {%- endif -%}",
60
  "clean_up_tokenization_spaces": false,
61
  "eos_token": "</s>",
62
  "legacy": true,