roberta-base-bne / tokenizer_config.json
asier-gutierrez's picture
Initial model V1
9879d2c
raw
history blame
1.46 kB
{
"unk_token": {
"content": "<unk>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": true,
"__type": "AddedToken"
},
"bos_token": {
"content": "<s>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": true,
"__type": "AddedToken"
},
"eos_token": {
"content": "</s>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": true,
"__type": "AddedToken"
},
"add_prefix_space": false,
"errors": "replace",
"sep_token": {
"content": "</s>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": true,
"__type": "AddedToken"
},
"cls_token": {
"content": "<s>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": true,
"__type": "AddedToken"
},
"pad_token": {
"content": "<pad>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": true,
"__type": "AddedToken"
},
"mask_token": {
"content": "<mask>",
"single_word": false,
"lstrip": true,
"rstrip": false,
"normalized": true,
"__type": "AddedToken"
},
"max_len": 512,
"special_tokens_map_file": null,
"name_or_path": "/gpfs/projects/bsc88/tools/corpus-utils-lm/17-06-2021-python/output/bne_es_output/roberta-2021-06-17-1849-3a6a-9c4f/train_tokenizer_output/train-tokenizer-2021-06-17-2216-3a6a-3cd6"
}