claude-tokenizer-forT5 / added_tokens.json
pszemraj's picture
Upload tokenizer
9966ebc verified
{
"</s>": 65001,
"<extra_id_0>": 65003,
"<extra_id_10>": 65013,
"<extra_id_11>": 65014,
"<extra_id_12>": 65015,
"<extra_id_13>": 65016,
"<extra_id_14>": 65017,
"<extra_id_15>": 65018,
"<extra_id_16>": 65019,
"<extra_id_17>": 65020,
"<extra_id_18>": 65021,
"<extra_id_19>": 65022,
"<extra_id_1>": 65004,
"<extra_id_20>": 65023,
"<extra_id_21>": 65024,
"<extra_id_22>": 65025,
"<extra_id_23>": 65026,
"<extra_id_24>": 65027,
"<extra_id_25>": 65028,
"<extra_id_26>": 65029,
"<extra_id_27>": 65030,
"<extra_id_28>": 65031,
"<extra_id_29>": 65032,
"<extra_id_2>": 65005,
"<extra_id_30>": 65033,
"<extra_id_31>": 65034,
"<extra_id_32>": 65035,
"<extra_id_33>": 65036,
"<extra_id_34>": 65037,
"<extra_id_35>": 65038,
"<extra_id_36>": 65039,
"<extra_id_37>": 65040,
"<extra_id_38>": 65041,
"<extra_id_39>": 65042,
"<extra_id_3>": 65006,
"<extra_id_40>": 65043,
"<extra_id_41>": 65044,
"<extra_id_42>": 65045,
"<extra_id_43>": 65046,
"<extra_id_44>": 65047,
"<extra_id_45>": 65048,
"<extra_id_46>": 65049,
"<extra_id_47>": 65050,
"<extra_id_48>": 65051,
"<extra_id_49>": 65052,
"<extra_id_4>": 65007,
"<extra_id_50>": 65053,
"<extra_id_51>": 65054,
"<extra_id_52>": 65055,
"<extra_id_53>": 65056,
"<extra_id_54>": 65057,
"<extra_id_55>": 65058,
"<extra_id_56>": 65059,
"<extra_id_57>": 65060,
"<extra_id_58>": 65061,
"<extra_id_59>": 65062,
"<extra_id_5>": 65008,
"<extra_id_60>": 65063,
"<extra_id_61>": 65064,
"<extra_id_62>": 65065,
"<extra_id_63>": 65066,
"<extra_id_64>": 65067,
"<extra_id_65>": 65068,
"<extra_id_66>": 65069,
"<extra_id_67>": 65070,
"<extra_id_68>": 65071,
"<extra_id_69>": 65072,
"<extra_id_6>": 65009,
"<extra_id_70>": 65073,
"<extra_id_71>": 65074,
"<extra_id_72>": 65075,
"<extra_id_73>": 65076,
"<extra_id_74>": 65077,
"<extra_id_75>": 65078,
"<extra_id_76>": 65079,
"<extra_id_77>": 65080,
"<extra_id_78>": 65081,
"<extra_id_79>": 65082,
"<extra_id_7>": 65010,
"<extra_id_80>": 65083,
"<extra_id_81>": 65084,
"<extra_id_82>": 65085,
"<extra_id_83>": 65086,
"<extra_id_84>": 65087,
"<extra_id_85>": 65088,
"<extra_id_86>": 65089,
"<extra_id_87>": 65090,
"<extra_id_88>": 65091,
"<extra_id_89>": 65092,
"<extra_id_8>": 65011,
"<extra_id_90>": 65093,
"<extra_id_91>": 65094,
"<extra_id_92>": 65095,
"<extra_id_93>": 65096,
"<extra_id_94>": 65097,
"<extra_id_95>": 65098,
"<extra_id_96>": 65099,
"<extra_id_97>": 65100,
"<extra_id_98>": 65101,
"<extra_id_99>": 65102,
"<extra_id_9>": 65012,
"<pad>": 65000,
"<unk>": 65002
}