{ "additional_special_tokens": [ { "content": "len_1", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "len_2", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "len_3", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "len_4", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "len_5", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "len_6", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "len_7", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "len_8", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "len_9", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "len_10", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "len_11", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "len_12", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "len_13", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "len_14", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "len_15", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "len_16", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "len_17", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "len_18", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "len_19", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "len_20", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "
",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
}
],
"bos_token": {
"content": "",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"cls_token": {
"content": "",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"eos_token": "
",
"mask_token": {
"content": " ",
"sep_token": {
"content": "",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"unk_token": {
"content": "