|
{ |
|
"bos_token": { |
|
"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true |
|
}, |
|
"eos_token": { |
|
"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true |
|
}, |
|
"unk_token": { |
|
"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true |
|
}, |
|
"sep_token": { |
|
"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true |
|
}, |
|
"pad_token": { |
|
"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true |
|
}, |
|
"cls_token": { |
|
"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true |
|
}, |
|
"mask_token": { |
|
"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true |
|
} |
|
} |