Tomy Hsieh commited on
Commit
016776f
1 Parent(s): b8c68d0

🛠 fix: Add special token

Browse files
Files changed (3) hide show
  1. added_tokens.json +1 -0
  2. special_tokens_map.json +1 -1
  3. tokenizer.json +0 -0
added_tokens.json ADDED
@@ -0,0 +1 @@
 
1
+ {"<|EOS|>": 50258, "<|PAD|>": 50260, "<|SEP|>": 50261, "<|BOS|>": 50257, "<|UNK|>": 50259}
special_tokens_map.json CHANGED
@@ -1 +1 @@
1
- {"bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "unk_token": "<|endoftext|>"}
1
+ {"bos_token": "<|BOS|>", "eos_token": "<|EOS|>", "unk_token": "<|UNK|>", "sep_token": "<|SEP|>", "pad_token": "<|PAD|>"}
tokenizer.json CHANGED
The diff for this file is too large to render. See raw diff