{ | |
"added_tokens_decoder": { | |
"0": { | |
"content": "<pad>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"1": { | |
"content": "</s>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"2": { | |
"content": "<mask_1>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"3": { | |
"content": "<mask_2>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"4": { | |
"content": "<unk_2>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"5": { | |
"content": "<unk_3>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"6": { | |
"content": "<unk_4>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"7": { | |
"content": "<unk_5>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"8": { | |
"content": "<unk_6>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"9": { | |
"content": "<unk_7>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"10": { | |
"content": "<unk_8>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"11": { | |
"content": "<unk_9>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"12": { | |
"content": "<unk_10>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"13": { | |
"content": "<unk_11>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"14": { | |
"content": "<unk_12>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"15": { | |
"content": "<unk_13>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"16": { | |
"content": "<unk_14>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"17": { | |
"content": "<unk_15>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"18": { | |
"content": "<unk_16>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"19": { | |
"content": "<unk_17>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"20": { | |
"content": "<unk_18>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"21": { | |
"content": "<unk_19>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"22": { | |
"content": "<unk_20>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"23": { | |
"content": "<unk_21>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"24": { | |
"content": "<unk_22>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"25": { | |
"content": "<unk_23>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"26": { | |
"content": "<unk_24>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"27": { | |
"content": "<unk_25>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"28": { | |
"content": "<unk_26>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"29": { | |
"content": "<unk_27>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"30": { | |
"content": "<unk_28>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"31": { | |
"content": "<unk_29>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"32": { | |
"content": "<unk_30>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"33": { | |
"content": "<unk_31>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"34": { | |
"content": "<unk_32>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"35": { | |
"content": "<unk_33>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"36": { | |
"content": "<unk_34>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"37": { | |
"content": "<unk_35>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"38": { | |
"content": "<unk_36>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"39": { | |
"content": "<unk_37>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"40": { | |
"content": "<unk_38>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"41": { | |
"content": "<unk_39>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"42": { | |
"content": "<unk_40>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"43": { | |
"content": "<unk_41>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"44": { | |
"content": "<unk_42>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"45": { | |
"content": "<unk_43>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"46": { | |
"content": "<unk_44>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"47": { | |
"content": "<unk_45>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"48": { | |
"content": "<unk_46>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"49": { | |
"content": "<unk_47>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"50": { | |
"content": "<unk_48>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"51": { | |
"content": "<unk_49>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"52": { | |
"content": "<unk_50>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"53": { | |
"content": "<unk_51>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"54": { | |
"content": "<unk_52>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"55": { | |
"content": "<unk_53>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"56": { | |
"content": "<unk_54>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"57": { | |
"content": "<unk_55>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"58": { | |
"content": "<unk_56>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"59": { | |
"content": "<unk_57>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"60": { | |
"content": "<unk_58>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"61": { | |
"content": "<unk_59>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"62": { | |
"content": "<unk_60>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"63": { | |
"content": "<unk_61>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"64": { | |
"content": "<unk_62>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"65": { | |
"content": "<unk_63>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"66": { | |
"content": "<unk_64>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"67": { | |
"content": "<unk_65>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"68": { | |
"content": "<unk_66>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"69": { | |
"content": "<unk_67>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"70": { | |
"content": "<unk_68>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"71": { | |
"content": "<unk_69>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"72": { | |
"content": "<unk_70>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"73": { | |
"content": "<unk_71>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"74": { | |
"content": "<unk_72>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"75": { | |
"content": "<unk_73>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"76": { | |
"content": "<unk_74>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"77": { | |
"content": "<unk_75>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"78": { | |
"content": "<unk_76>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"79": { | |
"content": "<unk_77>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"80": { | |
"content": "<unk_78>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"81": { | |
"content": "<unk_79>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"82": { | |
"content": "<unk_80>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"83": { | |
"content": "<unk_81>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"84": { | |
"content": "<unk_82>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"85": { | |
"content": "<unk_83>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"86": { | |
"content": "<unk_84>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"87": { | |
"content": "<unk_85>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"88": { | |
"content": "<unk_86>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"89": { | |
"content": "<unk_87>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"90": { | |
"content": "<unk_88>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"91": { | |
"content": "<unk_89>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"92": { | |
"content": "<unk_90>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"93": { | |
"content": "<unk_91>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"94": { | |
"content": "<unk_92>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"95": { | |
"content": "<unk_93>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"96": { | |
"content": "<unk_94>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"97": { | |
"content": "<unk_95>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"98": { | |
"content": "<unk_96>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"99": { | |
"content": "<unk_97>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"100": { | |
"content": "<unk_98>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"101": { | |
"content": "<unk_99>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"102": { | |
"content": "<unk_100>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"103": { | |
"content": "<unk_101>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"104": { | |
"content": "<unk_102>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"105": { | |
"content": "<unk>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
} | |
}, | |
"additional_special_tokens": [ | |
"<mask_1>", | |
"<unk_2>", | |
"<unk_3>", | |
"<unk_4>", | |
"<unk_5>", | |
"<unk_6>", | |
"<unk_7>", | |
"<unk_8>", | |
"<unk_9>", | |
"<unk_10>", | |
"<unk_11>", | |
"<unk_12>", | |
"<unk_13>", | |
"<unk_14>", | |
"<unk_15>", | |
"<unk_16>", | |
"<unk_17>", | |
"<unk_18>", | |
"<unk_19>", | |
"<unk_20>", | |
"<unk_21>", | |
"<unk_22>", | |
"<unk_23>", | |
"<unk_24>", | |
"<unk_25>", | |
"<unk_26>", | |
"<unk_27>", | |
"<unk_28>", | |
"<unk_29>", | |
"<unk_30>", | |
"<unk_31>", | |
"<unk_32>", | |
"<unk_33>", | |
"<unk_34>", | |
"<unk_35>", | |
"<unk_36>", | |
"<unk_37>", | |
"<unk_38>", | |
"<unk_39>", | |
"<unk_40>", | |
"<unk_41>", | |
"<unk_42>", | |
"<unk_43>", | |
"<unk_44>", | |
"<unk_45>", | |
"<unk_46>", | |
"<unk_47>", | |
"<unk_48>", | |
"<unk_49>", | |
"<unk_50>", | |
"<unk_51>", | |
"<unk_52>", | |
"<unk_53>", | |
"<unk_54>", | |
"<unk_55>", | |
"<unk_56>", | |
"<unk_57>", | |
"<unk_58>", | |
"<unk_59>", | |
"<unk_60>", | |
"<unk_61>", | |
"<unk_62>", | |
"<unk_63>", | |
"<unk_64>", | |
"<unk_65>", | |
"<unk_66>", | |
"<unk_67>", | |
"<unk_68>", | |
"<unk_69>", | |
"<unk_70>", | |
"<unk_71>", | |
"<unk_72>", | |
"<unk_73>", | |
"<unk_74>", | |
"<unk_75>", | |
"<unk_76>", | |
"<unk_77>", | |
"<unk_78>", | |
"<unk_79>", | |
"<unk_80>", | |
"<unk_81>", | |
"<unk_82>", | |
"<unk_83>", | |
"<unk_84>", | |
"<unk_85>", | |
"<unk_86>", | |
"<unk_87>", | |
"<unk_88>", | |
"<unk_89>", | |
"<unk_90>", | |
"<unk_91>", | |
"<unk_92>", | |
"<unk_93>", | |
"<unk_94>", | |
"<unk_95>", | |
"<unk_96>", | |
"<unk_97>", | |
"<unk_98>", | |
"<unk_99>", | |
"<unk_100>", | |
"<unk_101>", | |
"<unk_102>" | |
], | |
"batch_size": null, | |
"batched": true, | |
"clean_up_tokenization_spaces": true, | |
"eos_token": "</s>", | |
"full_tokenizer_file": null, | |
"mask_token": "<mask_2>", | |
"mask_token_sent": "<mask_1>", | |
"model_max_length": 1024, | |
"offset": 103, | |
"pad_token": "<pad>", | |
"padding": true, | |
"sp_model_kwargs": {}, | |
"tokenizer_class": "PegasusTokenizer", | |
"truncation": true, | |
"unk_token": "<unk>" | |
} | |