zqh11 commited on
Commit
623a105
1 Parent(s): 83bb1ca

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +2 -2
tokenizer_config.json CHANGED
@@ -12,7 +12,7 @@
12
  "clean_up_tokenization_spaces": false,
13
  "eos_token": {
14
  "__type": "AddedToken",
15
- "content": "<|EOT|>",
16
  "lstrip": false,
17
  "normalized": true,
18
  "rstrip": false,
@@ -31,5 +31,5 @@
31
  "sp_model_kwargs": {},
32
  "unk_token": null,
33
  "tokenizer_class": "LlamaTokenizerFast",
34
- "chat_template": "{%- set found_item = false -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set found_item = true -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if not found_item -%}\n{{'You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer.\\n'}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' %}\n{{ message['content'] }}\n {%- else %}\n {%- if message['role'] == 'user' %}\n{{'### Instruction:\\n' + message['content'] + '\\n'}}\n {%- else %}\n{{'### Response:\\n' + message['content'] + '\\n<|EOT|>\\n'}}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{{'### Response:\\n'}}\n"
35
  }
 
12
  "clean_up_tokenization_spaces": false,
13
  "eos_token": {
14
  "__type": "AddedToken",
15
+ "content": "<|end▁of▁sentence|>",
16
  "lstrip": false,
17
  "normalized": true,
18
  "rstrip": false,
 
31
  "sp_model_kwargs": {},
32
  "unk_token": null,
33
  "tokenizer_class": "LlamaTokenizerFast",
34
+ "chat_template": "{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set ns.found = true -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if not ns.found -%}\n{{'You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\\n'}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' %}\n{{ message['content'] }}\n {%- else %}\n {%- if message['role'] == 'user' %}\n{{'### Instruction:\\n' + message['content'] + '\\n'}}\n {%- else %}\n{{'### Response:\\n' + message['content'] + '\\n<|EOT|>\\n'}}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{{'### Response:\\n'}}\n"
35
  }