orgcatorg
/

EntityCS-39-PEP_MS_MLM-xlmr-base

Token Classification

Inference Endpoints

Model card Files Files and versions Community

EntityCS-39-PEP_MS_MLM-xlmr-base / special_tokens_map.json

Xmm's picture

Xmm

Upload 12 files

9ac8c21 verified 3 months ago

raw history blame contribute delete

No virus

1.98 kB

	{
	"additional_special_tokens": [
	"<ar>",
	"<he>",
	"<vi>",
	"<id>",
	"<jv>",
	"<ms>",
	"<tl>",
	"<eu>",
	"<ml>",
	"<ta>",
	"<te>",
	"<af>",
	"<nl>",
	"<en>",
	"<de>",
	"<el>",
	"<bn>",
	"<hi>",
	"<mr>",
	"<ur>",
	"<fa>",
	"<fr>",
	"<it>",
	"<pt>",
	"<es>",
	"<bg>",
	"<ru>",
	"<ja>",
	"<ka>",
	"<ko>",
	"<th>",
	"<sw>",
	"<my>",
	"<zh>",
	"<kk>",
	"<tr>",
	"<et>",
	"<fi>",
	"<hu>",
	"</ar>",
	"</he>",
	"</vi>",
	"</id>",
	"</jv>",
	"</ms>",
	"</tl>",
	"</eu>",
	"</ml>",
	"</ta>",
	"</te>",
	"</af>",
	"</nl>",
	"</en>",
	"</de>",
	"</el>",
	"</bn>",
	"</hi>",
	"</mr>",
	"</ur>",
	"</fa>",
	"</fr>",
	"</it>",
	"</pt>",
	"</es>",
	"</bg>",
	"</ru>",
	"</ja>",
	"</ka>",
	"</ko>",
	"</th>",
	"</sw>",
	"</my>",
	"</zh>",
	"</kk>",
	"</tr>",
	"</et>",
	"</fi>",
	"</hu>"
	],
	"bos_token": {
	"content": "<s>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false
	},
	"cls_token": {
	"content": "<s>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false
	},
	"eos_token": {
	"content": "</s>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false
	},
	"mask_token": {
	"content": "<mask>",
	"lstrip": true,
	"normalized": false,
	"rstrip": false,
	"single_word": false
	},
	"pad_token": {
	"content": "<pad>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false
	},
	"sep_token": {
	"content": "</s>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false
	},
	"unk_token": {
	"content": "<unk>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false
	}
	}