game-content-safety / tokenizer_config.json
BrightXiaoHan's picture
Upload tokenizer
5861b33 verified
raw
history blame contribute delete
No virus
2.07 kB
{
"added_tokens_decoder": {
"0": {
"content": "<s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "<pad>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "</s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"3": {
"content": "<unk>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"250001": {
"content": "<mask>",
"lstrip": true,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
}
},
"bos_token": "<s>",
"clean_up_tokenization_spaces": true,
"cls_token": "<s>",
"eos_token": "</s>",
"id2label": {
"0": "compliant",
"1": "pornography",
"10": "non-official recharging",
"11": "diverting players to other games",
"2": "violence",
"3": "toxicity",
"4": "politics",
"5": "advertising",
"6": "boycotting games",
"7": "cheating",
"8": "buying and selling resources",
"9": "buying and selling account"
},
"label2id": {
"advertising": 5,
"boycotting games": 6,
"buying and selling account": 9,
"buying and selling resources": 8,
"cheating": 7,
"compliant": 0,
"diverting players to other games": 11,
"non-official recharging": 10,
"politics": 4,
"pornography": 1,
"toxicity": 3,
"violence": 2
},
"mask_token": "<mask>",
"max_length": 512,
"model_max_length": 1000000000000000019884624838656,
"pad_to_multiple_of": null,
"pad_token": "<pad>",
"pad_token_type_id": 0,
"padding_side": "right",
"sep_token": "</s>",
"stride": 0,
"tokenizer_class": "XLMRobertaTokenizer",
"truncation_side": "right",
"truncation_strategy": "longest_first",
"unk_token": "<unk>"
}