deepmind
/

language-perceiver

Model card Files Files and versions Community

language-perceiver / tokenizer_config.json

nielsr's picture

nielsr HF staff

Add tokenizer config

83473af about 3 years ago

823 Bytes

	{"eos_token":
	{"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"},
	"bos_token":
	{"content": "<bos>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"},
	"pad_token":
	{"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"},
	"mask_token":
	{"content": "<mask>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"},
	"cls_token":
	{"content": "<cls>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"},
	"sep_token":
	{"content": "<sep>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}
	}