imone commited on
Commit
df2ce8f
1 Parent(s): e2059b8

add </s> as additional special tokens

Browse files
Files changed (1) hide show
  1. special_tokens_map.json +1 -1
special_tokens_map.json CHANGED
@@ -1 +1 @@
1
- {"additional_special_tokens": ["<|end_of_turn|>", "<|PAD|>"], "bos_token": {"content": "<s>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false}, "eos_token": "<|end_of_turn|>", "unk_token": {"content": "<unk>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false}}
 
1
+ {"additional_special_tokens": ["<|end_of_turn|>", "<|PAD|>", "</s>"], "bos_token": {"content": "<s>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false}, "eos_token": "<|end_of_turn|>", "unk_token": {"content": "<unk>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false}}