|
{ |
|
"version": "1.0", |
|
"truncation": { |
|
"direction": "Right", |
|
"max_length": 512, |
|
"strategy": "LongestFirst", |
|
"stride": 0 |
|
}, |
|
"padding": { |
|
"strategy": "BatchLongest", |
|
"direction": "Right", |
|
"pad_to_multiple_of": null, |
|
"pad_id": 0, |
|
"pad_type_id": 0, |
|
"pad_token": "[PAD]" |
|
}, |
|
"added_tokens": [ |
|
{ |
|
"id": 0, |
|
"content": "[UNK]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 1, |
|
"content": "[PAD]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 2, |
|
"content": "[CLS]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 3, |
|
"content": "[SEP]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 4, |
|
"content": "[MASK]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
} |
|
], |
|
"normalizer": { |
|
"type": "BertNormalizer", |
|
"clean_text": true, |
|
"handle_chinese_chars": true, |
|
"strip_accents": null, |
|
"lowercase": false |
|
}, |
|
"pre_tokenizer": { |
|
"type": "WhitespaceSplit" |
|
}, |
|
"post_processor": { |
|
"type": "TemplateProcessing", |
|
"single": [ |
|
{ |
|
"SpecialToken": { |
|
"id": "[CLS]", |
|
"type_id": 0 |
|
} |
|
}, |
|
{ |
|
"Sequence": { |
|
"id": "A", |
|
"type_id": 0 |
|
} |
|
} |
|
], |
|
"pair": [ |
|
{ |
|
"Sequence": { |
|
"id": "A", |
|
"type_id": 0 |
|
} |
|
}, |
|
{ |
|
"Sequence": { |
|
"id": "B", |
|
"type_id": 1 |
|
} |
|
} |
|
], |
|
"special_tokens": { |
|
"[CLS]": { |
|
"id": "[CLS]", |
|
"ids": [ |
|
2 |
|
], |
|
"tokens": [ |
|
"[CLS]" |
|
] |
|
} |
|
} |
|
}, |
|
"decoder": { |
|
"type": "WordPiece", |
|
"prefix": "##", |
|
"cleanup": true |
|
}, |
|
"model": { |
|
"type": "WordPiece", |
|
"unk_token": "[UNK]", |
|
"continuing_subword_prefix": "##", |
|
"max_input_chars_per_word": 100, |
|
"vocab": { |
|
"[UNK]": 0, |
|
"[PAD]": 1, |
|
"[CLS]": 2, |
|
"[SEP]": 3, |
|
"[MASK]": 4, |
|
"\"": 5, |
|
"#": 6, |
|
"(": 7, |
|
")": 8, |
|
"+": 9, |
|
"-": 10, |
|
".": 11, |
|
"/": 12, |
|
"1": 13, |
|
"2": 14, |
|
"3": 15, |
|
"4": 16, |
|
"5": 17, |
|
"6": 18, |
|
"7": 19, |
|
"8": 20, |
|
"9": 21, |
|
"=": 22, |
|
"@": 23, |
|
"A": 24, |
|
"B": 25, |
|
"C": 26, |
|
"D": 27, |
|
"E": 28, |
|
"F": 29, |
|
"G": 30, |
|
"H": 31, |
|
"I": 32, |
|
"K": 33, |
|
"L": 34, |
|
"M": 35, |
|
"N": 36, |
|
"O": 37, |
|
"P": 38, |
|
"Q": 39, |
|
"R": 40, |
|
"S": 41, |
|
"T": 42, |
|
"V": 43, |
|
"W": 44, |
|
"X": 45, |
|
"Y": 46, |
|
"[": 47, |
|
"\\": 48, |
|
"]": 49, |
|
"a": 50, |
|
"c": 51, |
|
"i": 52, |
|
"l": 53, |
|
"n": 54, |
|
"o": 55, |
|
"r": 56, |
|
"s": 57, |
|
"##\"": 58, |
|
"##S": 59, |
|
"##E": 60, |
|
"##P": 61, |
|
"##]": 62, |
|
"##C": 63, |
|
"##O": 64, |
|
"##F": 65, |
|
"##[": 66, |
|
"##N": 67, |
|
"##B": 68, |
|
"##c": 69 |
|
} |
|
} |
|
} |