|
{
|
|
"add_bos_token": true,
|
|
"add_eos_token": true,
|
|
"added_tokens_decoder": {
|
|
"0": {
|
|
"content": "</s>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"1": {
|
|
"content": "</s>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"2": {
|
|
"content": "<mask_1>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"3": {
|
|
"content": "<mask_2>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"4": {
|
|
"content": "<unk_2>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"5": {
|
|
"content": "<unk_3>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"6": {
|
|
"content": "<unk_4>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"7": {
|
|
"content": "<unk_5>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"8": {
|
|
"content": "<unk_6>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"9": {
|
|
"content": "<unk_7>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"10": {
|
|
"content": "<unk_8>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"11": {
|
|
"content": "<unk_9>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"12": {
|
|
"content": "<unk_10>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"13": {
|
|
"content": "<unk_11>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"14": {
|
|
"content": "<unk_12>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"15": {
|
|
"content": "<unk_13>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"16": {
|
|
"content": "<unk_14>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"17": {
|
|
"content": "<unk_15>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"18": {
|
|
"content": "<unk_16>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"19": {
|
|
"content": "<unk_17>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"20": {
|
|
"content": "<unk_18>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"21": {
|
|
"content": "<unk_19>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"22": {
|
|
"content": "<unk_20>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"23": {
|
|
"content": "<unk_21>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"24": {
|
|
"content": "<unk_22>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"25": {
|
|
"content": "<unk_23>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"26": {
|
|
"content": "<unk_24>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"27": {
|
|
"content": "<unk_25>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"28": {
|
|
"content": "<unk_26>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"29": {
|
|
"content": "<unk_27>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"30": {
|
|
"content": "<unk_28>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"31": {
|
|
"content": "<unk_29>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32": {
|
|
"content": "<unk_30>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"33": {
|
|
"content": "<unk_31>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"34": {
|
|
"content": "<unk_32>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"35": {
|
|
"content": "<unk_33>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"36": {
|
|
"content": "<unk_34>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"37": {
|
|
"content": "<unk_35>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"38": {
|
|
"content": "<unk_36>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"39": {
|
|
"content": "<unk_37>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"40": {
|
|
"content": "<unk_38>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"41": {
|
|
"content": "<unk_39>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"42": {
|
|
"content": "<unk_40>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"43": {
|
|
"content": "<unk_41>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"44": {
|
|
"content": "<unk_42>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"45": {
|
|
"content": "<unk_43>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"46": {
|
|
"content": "<unk_44>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"47": {
|
|
"content": "<unk_45>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"48": {
|
|
"content": "<unk_46>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"49": {
|
|
"content": "<unk_47>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"50": {
|
|
"content": "<unk_48>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"51": {
|
|
"content": "<unk_49>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"52": {
|
|
"content": "<unk_50>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"53": {
|
|
"content": "<unk_51>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"54": {
|
|
"content": "<unk_52>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"55": {
|
|
"content": "<unk_53>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"56": {
|
|
"content": "<unk_54>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"57": {
|
|
"content": "<unk_55>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"58": {
|
|
"content": "<unk_56>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"59": {
|
|
"content": "<unk_57>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"60": {
|
|
"content": "<unk_58>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"61": {
|
|
"content": "<unk_59>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"62": {
|
|
"content": "<unk_60>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"63": {
|
|
"content": "<unk_61>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"64": {
|
|
"content": "<unk_62>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"65": {
|
|
"content": "<unk_63>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"66": {
|
|
"content": "<unk_64>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"67": {
|
|
"content": "<unk_65>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"68": {
|
|
"content": "<unk_66>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"69": {
|
|
"content": "<unk_67>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"70": {
|
|
"content": "<unk_68>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"71": {
|
|
"content": "<unk_69>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"72": {
|
|
"content": "<unk_70>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"73": {
|
|
"content": "<unk_71>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"74": {
|
|
"content": "<unk_72>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"75": {
|
|
"content": "<unk_73>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"76": {
|
|
"content": "<unk_74>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"77": {
|
|
"content": "<unk_75>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"78": {
|
|
"content": "<unk_76>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"79": {
|
|
"content": "<unk_77>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"80": {
|
|
"content": "<unk_78>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"81": {
|
|
"content": "<unk_79>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"82": {
|
|
"content": "<unk_80>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"83": {
|
|
"content": "<unk_81>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"84": {
|
|
"content": "<unk_82>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"85": {
|
|
"content": "<unk_83>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"86": {
|
|
"content": "<unk_84>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"87": {
|
|
"content": "<unk_85>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"88": {
|
|
"content": "<unk_86>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"89": {
|
|
"content": "<unk_87>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"90": {
|
|
"content": "<unk_88>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"91": {
|
|
"content": "<unk_89>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"92": {
|
|
"content": "<unk_90>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"93": {
|
|
"content": "<unk_91>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"94": {
|
|
"content": "<unk_92>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"95": {
|
|
"content": "<unk_93>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"96": {
|
|
"content": "<unk_94>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"97": {
|
|
"content": "<unk_95>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"98": {
|
|
"content": "<unk_96>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"99": {
|
|
"content": "<unk_97>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"100": {
|
|
"content": "<unk_98>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"101": {
|
|
"content": "<unk_99>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"102": {
|
|
"content": "<unk_100>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"103": {
|
|
"content": "<unk_101>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"104": {
|
|
"content": "<unk_102>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"105": {
|
|
"content": "<unk>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
}
|
|
},
|
|
"additional_special_tokens": [
|
|
"<mask_1>",
|
|
"<unk_2>",
|
|
"<unk_3>",
|
|
"<unk_4>",
|
|
"<unk_5>",
|
|
"<unk_6>",
|
|
"<unk_7>",
|
|
"<unk_8>",
|
|
"<unk_9>",
|
|
"<unk_10>",
|
|
"<unk_11>",
|
|
"<unk_12>",
|
|
"<unk_13>",
|
|
"<unk_14>",
|
|
"<unk_15>",
|
|
"<unk_16>",
|
|
"<unk_17>",
|
|
"<unk_18>",
|
|
"<unk_19>",
|
|
"<unk_20>",
|
|
"<unk_21>",
|
|
"<unk_22>",
|
|
"<unk_23>",
|
|
"<unk_24>",
|
|
"<unk_25>",
|
|
"<unk_26>",
|
|
"<unk_27>",
|
|
"<unk_28>",
|
|
"<unk_29>",
|
|
"<unk_30>",
|
|
"<unk_31>",
|
|
"<unk_32>",
|
|
"<unk_33>",
|
|
"<unk_34>",
|
|
"<unk_35>",
|
|
"<unk_36>",
|
|
"<unk_37>",
|
|
"<unk_38>",
|
|
"<unk_39>",
|
|
"<unk_40>",
|
|
"<unk_41>",
|
|
"<unk_42>",
|
|
"<unk_43>",
|
|
"<unk_44>",
|
|
"<unk_45>",
|
|
"<unk_46>",
|
|
"<unk_47>",
|
|
"<unk_48>",
|
|
"<unk_49>",
|
|
"<unk_50>",
|
|
"<unk_51>",
|
|
"<unk_52>",
|
|
"<unk_53>",
|
|
"<unk_54>",
|
|
"<unk_55>",
|
|
"<unk_56>",
|
|
"<unk_57>",
|
|
"<unk_58>",
|
|
"<unk_59>",
|
|
"<unk_60>",
|
|
"<unk_61>",
|
|
"<unk_62>",
|
|
"<unk_63>",
|
|
"<unk_64>",
|
|
"<unk_65>",
|
|
"<unk_66>",
|
|
"<unk_67>",
|
|
"<unk_68>",
|
|
"<unk_69>",
|
|
"<unk_70>",
|
|
"<unk_71>",
|
|
"<unk_72>",
|
|
"<unk_73>",
|
|
"<unk_74>",
|
|
"<unk_75>",
|
|
"<unk_76>",
|
|
"<unk_77>",
|
|
"<unk_78>",
|
|
"<unk_79>",
|
|
"<unk_80>",
|
|
"<unk_81>",
|
|
"<unk_82>",
|
|
"<unk_83>",
|
|
"<unk_84>",
|
|
"<unk_85>",
|
|
"<unk_86>",
|
|
"<unk_87>",
|
|
"<unk_88>",
|
|
"<unk_89>",
|
|
"<unk_90>",
|
|
"<unk_91>",
|
|
"<unk_92>",
|
|
"<unk_93>",
|
|
"<unk_94>",
|
|
"<unk_95>",
|
|
"<unk_96>",
|
|
"<unk_97>",
|
|
"<unk_98>",
|
|
"<unk_99>",
|
|
"<unk_100>",
|
|
"<unk_101>",
|
|
"<unk_102>"
|
|
],
|
|
"clean_up_tokenization_spaces": true,
|
|
"eos_token": "</s>",
|
|
"full_tokenizer_file": null,
|
|
"mask_token": "<mask_2>",
|
|
"mask_token_sent": "<mask_1>",
|
|
"model_max_length": 16384,
|
|
"offset": 103,
|
|
"pad_token": "</s>",
|
|
"padding_side": "right",
|
|
"sp_model_kwargs": {},
|
|
"tokenizer_class": "PegasusTokenizer",
|
|
"unk_token": "<unk>"
|
|
}
|
|
|