QagentS commited on
Commit
d1c77a7
1 Parent(s): 9d67e56

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +1 -0
  2. tokenizer_config.json +1 -1
tokenizer.json CHANGED
@@ -321,6 +321,7 @@
321
  "end_of_word_suffix": null,
322
  "fuse_unk": false,
323
  "byte_fallback": false,
 
324
  "vocab": {
325
  "!": 0,
326
  "\"": 1,
 
321
  "end_of_word_suffix": null,
322
  "fuse_unk": false,
323
  "byte_fallback": false,
324
+ "ignore_merges": false,
325
  "vocab": {
326
  "!": 0,
327
  "\"": 1,
tokenizer_config.json CHANGED
@@ -180,7 +180,7 @@
180
  }
181
  },
182
  "bos_token": "<|begin▁of▁sentence|>",
183
- "chat_template": "{% if not add_generation_prompt is defined %}\n{% set add_generation_prompt = false %}\n{% endif %}\n{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set ns.found = true -%}\n {%- endif -%}\n{%- endfor -%}\n{{bos_token}}{%- if not ns.found -%}\n{{'You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\\n'}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' %}\n{{ message['content'] }}\n {%- else %}\n {%- if message['role'] == 'user' %}\n{{'### Instruction:\\n' + message['content'] + '\\n'}}\n {%- else %}\n{{'### Response:\\n' + message['content'] + '\\n<|EOT|>\\n'}}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{% if add_generation_prompt %}\n{{'### Response:'}}\n{% endif %}",
184
  "clean_up_tokenization_spaces": false,
185
  "eos_token": "<|EOT|>",
186
  "legacy": true,
 
180
  }
181
  },
182
  "bos_token": "<|begin▁of▁sentence|>",
183
+ "chat_template": "{% if not add_generation_prompt is defined %}\n{% set add_generation_prompt = false %}\n{% endif %}\n{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set ns.found = true -%}\n {%- endif -%}\n{%- endfor -%}\n{{bos_token}}{%- if not ns.found -%}\n{{'You are a code and maths expert working at Pipable AI. Give good and precise answer to the questions asked to you.\\n'}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' %}\n{{ message['content'] }}\n {%- else %}\n {%- if message['role'] == 'user' %}\n{{'### Instruction:\\n' + message['content'] + '\\n'}}\n {%- else %}\n{{'### Response:\\n' + message['content'] + '\\n<|EOT|>\\n'}}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{% if add_generation_prompt %}\n{{'### Response:'}}\n{% endif %}",
184
  "clean_up_tokenization_spaces": false,
185
  "eos_token": "<|EOT|>",
186
  "legacy": true,