bart-finetuned-lyrlen-128-special_tokens / special_tokens_map.json
adamjweintraut's picture
Upload tokenizer
8d16aed verified
{
"additional_special_tokens": [
{
"content": "len_1",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "len_2",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "len_3",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "len_4",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "len_5",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "len_6",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "len_7",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "len_8",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "len_9",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "len_10",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "len_11",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "len_12",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "len_13",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "len_14",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "len_15",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "len_16",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "len_17",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "len_18",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "len_19",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "len_20",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
{
"content": "<P>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
}
],
"bos_token": {
"content": "<s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"cls_token": {
"content": "<s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"eos_token": "<P>",
"mask_token": {
"content": "<mask>",
"lstrip": true,
"normalized": true,
"rstrip": false,
"single_word": false
},
"pad_token": "<P>",
"sep_token": {
"content": "</s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"unk_token": {
"content": "<unk>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}