ZhankuiHe commited on
Commit
77098c4
1 Parent(s): 9982009

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -27,7 +27,7 @@
27
  }
28
  },
29
  "bos_token": "<|endoftext|>",
30
- "chat_template": "{% for message in messages %}{{ message.role + '\n' + message.content + '\n' | replace('_', ' ') }}{{ eos_token }}{% endfor %}{{ 'assistant\n' }}",
31
  "clean_up_tokenization_spaces": true,
32
  "eos_token": "<|endoftext|>",
33
  "model_max_length": 1024,
 
27
  }
28
  },
29
  "bos_token": "<|endoftext|>",
30
+ "chat_template": "{% for message in messages %}{{ message.role + '\n' + message.content | replace('_', ' ') }}{{ eos_token + '\n' }}{% endfor %}{{ 'assistant\n' }}",
31
  "clean_up_tokenization_spaces": true,
32
  "eos_token": "<|endoftext|>",
33
  "model_max_length": 1024,