FLMR / query_tokenizer /special_tokens_map.json
LinWeizheDragon's picture
Upload folder using huggingface_hub
3704bcf verified
{
"additional_special_tokens": [
"<BOV>",
"<SOV>",
"<EOV>",
"<BOQ>",
"<EOQ>",
"<BOC>",
"<EOC>",
"<BOK>",
"<EOK>"
],
"cls_token": {
"content": "[CLS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"mask_token": {
"content": "[MASK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"pad_token": {
"content": "[PAD]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"sep_token": {
"content": "[SEP]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"unk_token": {
"content": "[UNK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
}
}