esm2_t6_8M_UR50D_pep2rec_cppp / tokenizer_config.json

Upload folder using huggingface_hub

8b5cc63 verified 2 months ago

No virus

1.1 kB

	{
	"added_tokens_decoder": {
	"0": {
	"content": "<cls>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	},
	"1": {
	"content": "<pad>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	},
	"2": {
	"content": "<eos>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	},
	"3": {
	"content": "<unk>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	},
	"32": {
	"content": "<mask>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	}
	},
	"clean_up_tokenization_spaces": true,
	"cls_token": "<cls>",
	"eos_token": "<eos>",
	"mask_token": "<mask>",
	"model_max_length": 1024,
	"pad_token": "<pad>",
	"tokenizer_class": "EsmTokenizer",
	"unk_token": "<unk>"
	}