prince-canuma commited on
Commit
a2c307b
1 Parent(s): 0c37b14

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +2 -2
tokenizer_config.json CHANGED
@@ -400,10 +400,10 @@
400
  "bos_token": "<s>",
401
  "chat_template": "{% for message in messages %}{{'<|' + message['role'] + '|>' + '\n' + message['content'] + '<|end|>\n' }}{% endfor %}{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{- '<|assistant|>\n' -}}{% endif %}",
402
  "clean_up_tokenization_spaces": false,
403
- "eos_token": "<|end|>",
404
  "legacy": false,
405
  "model_max_length": 131072,
406
- "pad_token": "<|end|>",
407
  "padding_side": "right",
408
  "processor_class": "Phi3VProcessor",
409
  "sp_model_kwargs": {},
 
400
  "bos_token": "<s>",
401
  "chat_template": "{% for message in messages %}{{'<|' + message['role'] + '|>' + '\n' + message['content'] + '<|end|>\n' }}{% endfor %}{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{- '<|assistant|>\n' -}}{% endif %}",
402
  "clean_up_tokenization_spaces": false,
403
+ "eos_token": "<|endoftext|>",
404
  "legacy": false,
405
  "model_max_length": 131072,
406
+ "pad_token": "<|endoftext|>",
407
  "padding_side": "right",
408
  "processor_class": "Phi3VProcessor",
409
  "sp_model_kwargs": {},