gxy commited on
Commit
a078ea5
1 Parent(s): 8ee0562

Update tokenizer/special_tokens_map.json

Browse files
Files changed (1) hide show
  1. tokenizer/special_tokens_map.json +8 -6
tokenizer/special_tokens_map.json CHANGED
@@ -1,7 +1,9 @@
1
  {
2
- "cls_token": "[CLS]",
3
- "mask_token": "[MASK]",
4
- "pad_token": "[PAD]",
5
- "sep_token": "[SEP]",
6
- "unk_token": "[UNK]"
7
- }
 
 
 
1
  {
2
+ "cls_token": "[CLS]",
3
+ "bos_token": "[CLS]",
4
+ "mask_token": "[MASK]",
5
+ "pad_token": "[PAD]",
6
+ "sep_token": "[SEP]",
7
+ "eos_token": "[SEP]",
8
+ "unk_token": "[UNK]"
9
+ }