File size: 1,412 Bytes
a1e8b97 00cb73f a1e8b97 ac67afa 00cb73f a1e8b97 ac67afa 00cb73f a1e8b97 ac67afa 00cb73f 529c470 a1e8b97 ac67afa 00cb73f ac67afa a1e8b97 00cb73f a1e8b97 00cb73f 529c470 00cb73f a1e8b97 ac67afa a1e8b97 00cb73f a1e8b97 00cb73f a1e8b97 00cb73f a1e8b97 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 |
{
"added_tokens_decoder": {
"0": {
"content": "<pad>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "<unk>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "[CLS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"3": {
"content": "[SEP]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"4": {
"content": "[MASK]",
"lstrip": true,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
}
},
"bos_token": "[CLS]",
"clean_up_tokenization_spaces": true,
"cls_token": "[CLS]",
"do_lower_case": true,
"eos_token": "[SEP]",
"keep_accents": false,
"mask_token": "[MASK]",
"max_length": 100,
"model_max_length": 512,
"pad_to_multiple_of": null,
"pad_token": "<pad>",
"pad_token_type_id": 0,
"padding_side": "right",
"remove_space": true,
"sep_token": "[SEP]",
"stride": 0,
"tokenizer_class": "AlbertTokenizer",
"truncation_side": "right",
"truncation_strategy": "longest_first",
"unk_token": "<unk>"
}
|