|
{ |
|
"version": "1.0", |
|
"truncation": null, |
|
"padding": null, |
|
"added_tokens": [ |
|
{ |
|
"id": 0, |
|
"content": "[PAD]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 1, |
|
"content": "[UNK]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 2, |
|
"content": "[CLS]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 3, |
|
"content": "[SEP]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 4, |
|
"content": "[MASK]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
} |
|
], |
|
"normalizer": { |
|
"type": "BertNormalizer", |
|
"clean_text": true, |
|
"handle_chinese_chars": true, |
|
"strip_accents": null, |
|
"lowercase": true |
|
}, |
|
"pre_tokenizer": { |
|
"type": "BertPreTokenizer" |
|
}, |
|
"post_processor": { |
|
"type": "TemplateProcessing", |
|
"single": [ |
|
{ |
|
"SpecialToken": { |
|
"id": "[CLS]", |
|
"type_id": 0 |
|
} |
|
}, |
|
{ |
|
"Sequence": { |
|
"id": "A", |
|
"type_id": 0 |
|
} |
|
}, |
|
{ |
|
"SpecialToken": { |
|
"id": "[SEP]", |
|
"type_id": 0 |
|
} |
|
} |
|
], |
|
"pair": [ |
|
{ |
|
"SpecialToken": { |
|
"id": "[CLS]", |
|
"type_id": 0 |
|
} |
|
}, |
|
{ |
|
"Sequence": { |
|
"id": "A", |
|
"type_id": 0 |
|
} |
|
}, |
|
{ |
|
"SpecialToken": { |
|
"id": "[SEP]", |
|
"type_id": 0 |
|
} |
|
}, |
|
{ |
|
"Sequence": { |
|
"id": "B", |
|
"type_id": 1 |
|
} |
|
}, |
|
{ |
|
"SpecialToken": { |
|
"id": "[SEP]", |
|
"type_id": 1 |
|
} |
|
} |
|
], |
|
"special_tokens": { |
|
"[CLS]": { |
|
"id": "[CLS]", |
|
"ids": [ |
|
2 |
|
], |
|
"tokens": [ |
|
"[CLS]" |
|
] |
|
}, |
|
"[SEP]": { |
|
"id": "[SEP]", |
|
"ids": [ |
|
3 |
|
], |
|
"tokens": [ |
|
"[SEP]" |
|
] |
|
} |
|
} |
|
}, |
|
"decoder": { |
|
"type": "WordPiece", |
|
"prefix": "##", |
|
"cleanup": true |
|
}, |
|
"model": { |
|
"type": "WordPiece", |
|
"unk_token": "[UNK]", |
|
"continuing_subword_prefix": "##", |
|
"max_input_chars_per_word": 100, |
|
"vocab": { |
|
"[PAD]": 0, |
|
"[UNK]": 1, |
|
"[CLS]": 2, |
|
"[SEP]": 3, |
|
"[MASK]": 4, |
|
"/": 5, |
|
"a": 6, |
|
"c": 7, |
|
"d": 8, |
|
"e": 9, |
|
"g": 10, |
|
"h": 11, |
|
"i": 12, |
|
"k": 13, |
|
"l": 14, |
|
"m": 15, |
|
"n": 16, |
|
"o": 17, |
|
"p": 18, |
|
"r": 19, |
|
"s": 20, |
|
"t": 21, |
|
"u": 22, |
|
"w": 23, |
|
"y": 24, |
|
"##a": 25, |
|
"##s": 26, |
|
"##t": 27, |
|
"##g": 28, |
|
"##e": 29, |
|
"##r": 30, |
|
"##h": 31, |
|
"##n": 32, |
|
"##i": 33, |
|
"##c": 34, |
|
"##y": 35, |
|
"##o": 36, |
|
"##l": 37, |
|
"##u": 38, |
|
"##m": 39, |
|
"##k": 40, |
|
"pl": 41, |
|
"##as": 42, |
|
"##ar": 43, |
|
"##re": 44, |
|
"##it": 45, |
|
"##ot": 46, |
|
"##le": 47, |
|
"plot": 48, |
|
"cas": 49, |
|
"di": 50, |
|
"et": 51, |
|
"ge": 52, |
|
"or": 53, |
|
"pa": 54, |
|
"re": 55, |
|
"tit": 56, |
|
"wi": 57, |
|
"ye": 58, |
|
"##su": 59, |
|
"##to": 60, |
|
"##ge": 61, |
|
"##gi": 62, |
|
"##hn": 63, |
|
"##nre": 64, |
|
"##ic": 65, |
|
"##igi": 66, |
|
"##cto": 67, |
|
"##mm": 68, |
|
"##ki": 69, |
|
"##ase": 70, |
|
"##ary": 71, |
|
"##recto": 72, |
|
"##ity": 73, |
|
"##lease": 74, |
|
"plotsu": 75, |
|
"cast": 76, |
|
"directo": 77, |
|
"ethn": 78, |
|
"genre": 79, |
|
"origi": 80, |
|
"page": 81, |
|
"release": 82, |
|
"title": 83, |
|
"wiki": 84, |
|
"year": 85, |
|
"##icity": 86, |
|
"##mmary": 87, |
|
"plotsummary": 88, |
|
"director": 89, |
|
"ethnicity": 90, |
|
"origin": 91 |
|
} |
|
} |
|
} |