|
{ |
|
"add_bos_token": false, |
|
"add_eos_token": true, |
|
"add_prefix_space": false, |
|
"added_tokens_decoder": { |
|
"50256": { |
|
"content": "<|endoftext|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"50257": { |
|
"content": "<|extratoken_1|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50258": { |
|
"content": "<|extratoken_2|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50259": { |
|
"content": "<|extratoken_3|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50260": { |
|
"content": "<|extratoken_4|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50261": { |
|
"content": "<|extratoken_5|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50262": { |
|
"content": "<|extratoken_6|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50263": { |
|
"content": "<|extratoken_7|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50264": { |
|
"content": "<|extratoken_8|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50265": { |
|
"content": "<|extratoken_9|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50266": { |
|
"content": "<|extratoken_10|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50267": { |
|
"content": "<|extratoken_11|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50268": { |
|
"content": "<|extratoken_12|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50269": { |
|
"content": "<|extratoken_13|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50270": { |
|
"content": "<|extratoken_14|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50271": { |
|
"content": "<|extratoken_15|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50272": { |
|
"content": "<|extratoken_16|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50273": { |
|
"content": "<|extratoken_17|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50274": { |
|
"content": "<|extratoken_18|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50275": { |
|
"content": "<|extratoken_19|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50276": { |
|
"content": "<|extratoken_20|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50277": { |
|
"content": "<|extratoken_21|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50278": { |
|
"content": "<|extratoken_22|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50279": { |
|
"content": "<|extratoken_23|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50280": { |
|
"content": "<|extratoken_24|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50281": { |
|
"content": "<|extratoken_25|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50282": { |
|
"content": "<|extratoken_26|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50283": { |
|
"content": "<|extratoken_27|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50284": { |
|
"content": "<|extratoken_28|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50285": { |
|
"content": "<|extratoken_29|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50286": { |
|
"content": "<|extratoken_30|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50287": { |
|
"content": "<|extratoken_31|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50288": { |
|
"content": "<|extratoken_32|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50289": { |
|
"content": "<|extratoken_33|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50290": { |
|
"content": "<|extratoken_34|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50291": { |
|
"content": "<|extratoken_35|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50292": { |
|
"content": "<|extratoken_36|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50293": { |
|
"content": "<|extratoken_37|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50294": { |
|
"content": "<|extratoken_38|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50295": { |
|
"content": "<|extratoken_39|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50296": { |
|
"content": "<|extratoken_40|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50297": { |
|
"content": "<|extratoken_41|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50298": { |
|
"content": "<|extratoken_42|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50299": { |
|
"content": "<|extratoken_43|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50300": { |
|
"content": "<|extratoken_44|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50301": { |
|
"content": "<|extratoken_45|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50302": { |
|
"content": "<|extratoken_46|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50303": { |
|
"content": "<|extratoken_47|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50304": { |
|
"content": "<|extratoken_48|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50305": { |
|
"content": "<|extratoken_49|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50306": { |
|
"content": "<|extratoken_50|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50307": { |
|
"content": "<|extratoken_51|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50308": { |
|
"content": "<|extratoken_52|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50309": { |
|
"content": "<|extratoken_53|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50310": { |
|
"content": "<|extratoken_54|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50311": { |
|
"content": "<|extratoken_55|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50312": { |
|
"content": "<|extratoken_56|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50313": { |
|
"content": "<|extratoken_57|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50314": { |
|
"content": "<|extratoken_58|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50315": { |
|
"content": "<|extratoken_59|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50316": { |
|
"content": "<|extratoken_60|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50317": { |
|
"content": "<|extratoken_61|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50318": { |
|
"content": "<|extratoken_62|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50319": { |
|
"content": "<|extratoken_63|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50320": { |
|
"content": "<|extratoken_64|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50321": { |
|
"content": "<|extratoken_65|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50322": { |
|
"content": "<|extratoken_66|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50323": { |
|
"content": "<|extratoken_67|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50324": { |
|
"content": "<|extratoken_68|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50325": { |
|
"content": "<|extratoken_69|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50326": { |
|
"content": "<|extratoken_70|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50327": { |
|
"content": "<|extratoken_71|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50328": { |
|
"content": "<|extratoken_72|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50329": { |
|
"content": "<|extratoken_73|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50330": { |
|
"content": "<|extratoken_74|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50331": { |
|
"content": "<|extratoken_75|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50332": { |
|
"content": "<|extratoken_76|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50333": { |
|
"content": "<|extratoken_77|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50334": { |
|
"content": "<|extratoken_78|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50335": { |
|
"content": "<|extratoken_79|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50336": { |
|
"content": "<|extratoken_80|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50337": { |
|
"content": "<|extratoken_81|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50338": { |
|
"content": "<|extratoken_82|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50339": { |
|
"content": "<|extratoken_83|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50340": { |
|
"content": "<|extratoken_84|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50341": { |
|
"content": "<|extratoken_85|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50342": { |
|
"content": "<|extratoken_86|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50343": { |
|
"content": "<|extratoken_87|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50344": { |
|
"content": "<|extratoken_88|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50345": { |
|
"content": "<|extratoken_89|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50346": { |
|
"content": "<|extratoken_90|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50347": { |
|
"content": "<|extratoken_91|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50348": { |
|
"content": "<|extratoken_92|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50349": { |
|
"content": "<|extratoken_93|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50350": { |
|
"content": "<|extratoken_94|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50351": { |
|
"content": "<|extratoken_95|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50352": { |
|
"content": "<|extratoken_96|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50353": { |
|
"content": "<|extratoken_97|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50354": { |
|
"content": "<|extratoken_98|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50355": { |
|
"content": "<|extratoken_99|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50356": { |
|
"content": "<|extratoken_100|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50357": { |
|
"content": "<|extratoken_101|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50358": { |
|
"content": "<|extratoken_102|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50359": { |
|
"content": "<|extratoken_103|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50360": { |
|
"content": "<|extratoken_104|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50361": { |
|
"content": "<|extratoken_105|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50362": { |
|
"content": "<|extratoken_106|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50363": { |
|
"content": "<|extratoken_107|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50364": { |
|
"content": "<|extratoken_108|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50365": { |
|
"content": "<|extratoken_109|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50366": { |
|
"content": "<|extratoken_110|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50367": { |
|
"content": "<|extratoken_111|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50368": { |
|
"content": "<|extratoken_112|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50369": { |
|
"content": "<|extratoken_113|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50370": { |
|
"content": "<|extratoken_114|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50371": { |
|
"content": "<|extratoken_115|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50372": { |
|
"content": "<|extratoken_116|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50373": { |
|
"content": "<|extratoken_117|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50374": { |
|
"content": "<|extratoken_118|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50375": { |
|
"content": "<|extratoken_119|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50376": { |
|
"content": "<|extratoken_120|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50377": { |
|
"content": "<|extratoken_121|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50378": { |
|
"content": "<|extratoken_122|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50379": { |
|
"content": "<|extratoken_123|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50380": { |
|
"content": "<|extratoken_124|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50381": { |
|
"content": "<|extratoken_125|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50382": { |
|
"content": "<|extratoken_126|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50383": { |
|
"content": "<|extratoken_127|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50384": { |
|
"content": "<|extratoken_128|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50385": { |
|
"content": "<|extratoken_129|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50386": { |
|
"content": "<|extratoken_130|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50387": { |
|
"content": "<|extratoken_131|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50388": { |
|
"content": "<|extratoken_132|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50389": { |
|
"content": "<|extratoken_133|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50390": { |
|
"content": "<|extratoken_134|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50391": { |
|
"content": "<|extratoken_135|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50392": { |
|
"content": "<|extratoken_136|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50393": { |
|
"content": "<|extratoken_137|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50394": { |
|
"content": "<|extratoken_138|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50395": { |
|
"content": "<|extratoken_139|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50396": { |
|
"content": "<|extratoken_140|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50397": { |
|
"content": "<|extratoken_141|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50398": { |
|
"content": "<|extratoken_142|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"50399": { |
|
"content": "<|extratoken_143|>", |
|
"lstrip": false, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
} |
|
}, |
|
"bos_token": "<|endoftext|>", |
|
"clean_up_tokenization_spaces": true, |
|
"eos_token": "<|endoftext|>", |
|
"errors": "replace", |
|
"model_max_length": 2048, |
|
"pad_token": "<|endoftext|>", |
|
"tokenizer_class": "GPT2Tokenizer", |
|
"unk_token": "<|endoftext|>" |
|
} |
|
|