| import torch | |
| def keys_in_state_dict(ckpt, device='cpu'): | |
| if device=="cpu": | |
| a = torch.load(ckpt, map_location=torch.device('cpu'))["state_dict"] | |
| else: | |
| a = torch.load(ckpt)["state_dict"] | |
| print("keys_in_state_dict", a.keys()) | |
| def check_ckpt_diff(ckpt_a, ckpt_b, key_include=None, key_exclude=None, device='cpu', verbose=True): | |
| if device=="cpu": | |
| a = torch.load(ckpt_a, map_location=torch.device('cpu'))["state_dict"] | |
| b = torch.load(ckpt_b, map_location=torch.device('cpu'))["state_dict"] | |
| else: | |
| a = torch.load(ckpt_a)["state_dict"] | |
| b = torch.load(ckpt_b)["state_dict"] | |
| a_sum = 0 | |
| b_sum = 0 | |
| difference_count = 0 | |
| for k in a.keys(): | |
| if key_include is not None and key_include not in k: | |
| continue | |
| if key_exclude is not None and key_exclude in k: | |
| continue | |
| if k in b.keys(): | |
| a_sum += torch.sum(a[k]) | |
| b_sum += torch.sum(b[k]) | |
| if verbose: | |
| if torch.sum(a[k]) != torch.sum(b[k]): | |
| print(f"key {k} is different") | |
| difference_count += 1 | |
| print("a_sum: ", a_sum) | |
| print("b_sum: ", b_sum) | |
| print("diff: ", a_sum - b_sum) | |
| if verbose: | |
| print("difference_count: ", difference_count) | |
| return bool(a_sum - b_sum) | |
| # Transformer no freeze: | |
| # check_ckpt_diff("/fsx/clap_logs/2022_09_11-19_37_08-model_PANN-14-lr_0.001-b_160-j_4-p_fp32/checkpoints/epoch_10.pt", "/fsx/clap_logs/2022_09_11-19_37_08-model_PANN-14-lr_0.001-b_160-j_4-p_fp32/checkpoints/epoch_100.pt", "text_branch.resblocks") | |
| check_ckpt_diff("/fsx/clap_logs/2022_09_29-23_42_40-model_PANN-14-lr_0.001-b_160-j_4-p_fp32/checkpoints/epoch_1.pt", | |
| "/fsx/clap_logs/2022_09_29-23_42_40-model_PANN-14-lr_0.001-b_160-j_4-p_fp32/checkpoints/epoch_2.pt", | |
| "text_branch.resblocks") | |
| # key module.text_branch.resblocks.0.attn.in_proj_weight is different | |
| # key module.text_branch.resblocks.0.attn.in_proj_bias is different | |
| # key module.text_branch.resblocks.0.attn.out_proj.weight is different | |
| # key module.text_branch.resblocks.0.attn.out_proj.bias is different | |
| # key module.text_branch.resblocks.0.ln_1.weight is different | |
| # key module.text_branch.resblocks.0.ln_1.bias is different | |
| # key module.text_branch.resblocks.0.mlp.c_fc.weight is different | |
| # key module.text_branch.resblocks.0.mlp.c_fc.bias is different | |
| # key module.text_branch.resblocks.0.mlp.c_proj.weight is different | |
| # key module.text_branch.resblocks.0.mlp.c_proj.bias is different | |
| # key module.text_branch.resblocks.0.ln_2.weight is different | |
| # key module.text_branch.resblocks.0.ln_2.bias is different | |
| # key module.text_branch.resblocks.1.attn.in_proj_weight is different | |
| # key module.text_branch.resblocks.1.attn.in_proj_bias is different | |
| # key module.text_branch.resblocks.1.attn.out_proj.weight is different | |
| # key module.text_branch.resblocks.1.attn.out_proj.bias is different | |
| # key module.text_branch.resblocks.1.ln_1.weight is different | |
| # key module.text_branch.resblocks.1.ln_1.bias is different | |
| # key module.text_branch.resblocks.1.mlp.c_fc.weight is different | |
| # key module.text_branch.resblocks.1.mlp.c_fc.bias is different | |
| # key module.text_branch.resblocks.1.mlp.c_proj.weight is different | |
| # key module.text_branch.resblocks.1.mlp.c_proj.bias is different | |
| # key module.text_branch.resblocks.1.ln_2.weight is different | |
| # key module.text_branch.resblocks.1.ln_2.bias is different | |
| # key module.text_branch.resblocks.2.attn.in_proj_weight is different | |
| # key module.text_branch.resblocks.2.attn.in_proj_bias is different | |
| # key module.text_branch.resblocks.2.attn.out_proj.weight is different | |
| # key module.text_branch.resblocks.2.attn.out_proj.bias is different | |
| # key module.text_branch.resblocks.2.ln_1.weight is different | |
| # key module.text_branch.resblocks.2.ln_1.bias is different | |
| # key module.text_branch.resblocks.2.mlp.c_fc.weight is different | |
| # key module.text_branch.resblocks.2.mlp.c_fc.bias is different | |
| # key module.text_branch.resblocks.2.mlp.c_proj.weight is different | |
| # key module.text_branch.resblocks.2.mlp.c_proj.bias is different | |
| # key module.text_branch.resblocks.2.ln_2.weight is different | |
| # key module.text_branch.resblocks.2.ln_2.bias is different | |
| # key module.text_branch.resblocks.3.attn.in_proj_weight is different | |
| # key module.text_branch.resblocks.3.attn.in_proj_bias is different | |
| # key module.text_branch.resblocks.3.attn.out_proj.weight is different | |
| # key module.text_branch.resblocks.3.attn.out_proj.bias is different | |
| # key module.text_branch.resblocks.3.ln_1.weight is different | |
| # key module.text_branch.resblocks.3.ln_1.bias is different | |
| # key module.text_branch.resblocks.3.mlp.c_fc.weight is different | |
| # key module.text_branch.resblocks.3.mlp.c_fc.bias is different | |
| # key module.text_branch.resblocks.3.mlp.c_proj.weight is different | |
| # key module.text_branch.resblocks.3.mlp.c_proj.bias is different | |
| # key module.text_branch.resblocks.3.ln_2.weight is different | |
| # key module.text_branch.resblocks.3.ln_2.bias is different | |
| # key module.text_branch.resblocks.4.attn.in_proj_weight is different | |
| # key module.text_branch.resblocks.4.attn.in_proj_bias is different | |
| # key module.text_branch.resblocks.4.attn.out_proj.weight is different | |
| # key module.text_branch.resblocks.4.attn.out_proj.bias is different | |
| # key module.text_branch.resblocks.4.ln_1.weight is different | |
| # key module.text_branch.resblocks.4.ln_1.bias is different | |
| # key module.text_branch.resblocks.4.mlp.c_fc.weight is different | |
| # key module.text_branch.resblocks.4.mlp.c_fc.bias is different | |
| # key module.text_branch.resblocks.4.mlp.c_proj.weight is different | |
| # key module.text_branch.resblocks.4.mlp.c_proj.bias is different | |
| # key module.text_branch.resblocks.4.ln_2.weight is different | |
| # key module.text_branch.resblocks.4.ln_2.bias is different | |
| # key module.text_branch.resblocks.5.attn.in_proj_weight is different | |
| # key module.text_branch.resblocks.5.attn.in_proj_bias is different | |
| # key module.text_branch.resblocks.5.attn.out_proj.weight is different | |
| # key module.text_branch.resblocks.5.attn.out_proj.bias is different | |
| # key module.text_branch.resblocks.5.ln_1.weight is different | |
| # key module.text_branch.resblocks.5.ln_1.bias is different | |
| # key module.text_branch.resblocks.5.mlp.c_fc.weight is different | |
| # key module.text_branch.resblocks.5.mlp.c_fc.bias is different | |
| # key module.text_branch.resblocks.5.mlp.c_proj.weight is different | |
| # key module.text_branch.resblocks.5.mlp.c_proj.bias is different | |
| # key module.text_branch.resblocks.5.ln_2.weight is different | |
| # key module.text_branch.resblocks.5.ln_2.bias is different | |
| # key module.text_branch.resblocks.6.attn.in_proj_weight is different | |
| # key module.text_branch.resblocks.6.attn.in_proj_bias is different | |
| # key module.text_branch.resblocks.6.attn.out_proj.weight is different | |
| # key module.text_branch.resblocks.6.attn.out_proj.bias is different | |
| # key module.text_branch.resblocks.6.ln_1.weight is different | |
| # key module.text_branch.resblocks.6.ln_1.bias is different | |
| # key module.text_branch.resblocks.6.mlp.c_fc.weight is different | |
| # key module.text_branch.resblocks.6.mlp.c_fc.bias is different | |
| # key module.text_branch.resblocks.6.mlp.c_proj.weight is different | |
| # key module.text_branch.resblocks.6.mlp.c_proj.bias is different | |
| # key module.text_branch.resblocks.6.ln_2.weight is different | |
| # key module.text_branch.resblocks.6.ln_2.bias is different | |
| # key module.text_branch.resblocks.7.attn.in_proj_weight is different | |
| # key module.text_branch.resblocks.7.attn.in_proj_bias is different | |
| # key module.text_branch.resblocks.7.attn.out_proj.weight is different | |
| # key module.text_branch.resblocks.7.attn.out_proj.bias is different | |
| # key module.text_branch.resblocks.7.ln_1.weight is different | |
| # key module.text_branch.resblocks.7.ln_1.bias is different | |
| # key module.text_branch.resblocks.7.mlp.c_fc.weight is different | |
| # key module.text_branch.resblocks.7.mlp.c_fc.bias is different | |
| # key module.text_branch.resblocks.7.mlp.c_proj.weight is different | |
| # key module.text_branch.resblocks.7.mlp.c_proj.bias is different | |
| # key module.text_branch.resblocks.7.ln_2.weight is different | |
| # key module.text_branch.resblocks.7.ln_2.bias is different | |
| # key module.text_branch.resblocks.8.attn.in_proj_weight is different | |
| # key module.text_branch.resblocks.8.attn.in_proj_bias is different | |
| # key module.text_branch.resblocks.8.attn.out_proj.weight is different | |
| # key module.text_branch.resblocks.8.attn.out_proj.bias is different | |
| # key module.text_branch.resblocks.8.ln_1.weight is different | |
| # key module.text_branch.resblocks.8.ln_1.bias is different | |
| # key module.text_branch.resblocks.8.mlp.c_fc.weight is different | |
| # key module.text_branch.resblocks.8.mlp.c_fc.bias is different | |
| # key module.text_branch.resblocks.8.mlp.c_proj.weight is different | |
| # key module.text_branch.resblocks.8.mlp.c_proj.bias is different | |
| # key module.text_branch.resblocks.8.ln_2.weight is different | |
| # key module.text_branch.resblocks.8.ln_2.bias is different | |
| # key module.text_branch.resblocks.9.attn.in_proj_weight is different | |
| # key module.text_branch.resblocks.9.attn.in_proj_bias is different | |
| # key module.text_branch.resblocks.9.attn.out_proj.weight is different | |
| # key module.text_branch.resblocks.9.attn.out_proj.bias is different | |
| # key module.text_branch.resblocks.9.ln_1.weight is different | |
| # key module.text_branch.resblocks.9.ln_1.bias is different | |
| # key module.text_branch.resblocks.9.mlp.c_fc.weight is different | |
| # key module.text_branch.resblocks.9.mlp.c_fc.bias is different | |
| # key module.text_branch.resblocks.9.mlp.c_proj.weight is different | |
| # key module.text_branch.resblocks.9.mlp.c_proj.bias is different | |
| # key module.text_branch.resblocks.9.ln_2.weight is different | |
| # key module.text_branch.resblocks.9.ln_2.bias is different | |
| # key module.text_branch.resblocks.10.attn.in_proj_weight is different | |
| # key module.text_branch.resblocks.10.attn.in_proj_bias is different | |
| # key module.text_branch.resblocks.10.attn.out_proj.weight is different | |
| # key module.text_branch.resblocks.10.attn.out_proj.bias is different | |
| # key module.text_branch.resblocks.10.ln_1.weight is different | |
| # key module.text_branch.resblocks.10.ln_1.bias is different | |
| # key module.text_branch.resblocks.10.mlp.c_fc.weight is different | |
| # key module.text_branch.resblocks.10.mlp.c_fc.bias is different | |
| # key module.text_branch.resblocks.10.mlp.c_proj.weight is different | |
| # key module.text_branch.resblocks.10.mlp.c_proj.bias is different | |
| # key module.text_branch.resblocks.10.ln_2.weight is different | |
| # key module.text_branch.resblocks.10.ln_2.bias is different | |
| # key module.text_branch.resblocks.11.attn.in_proj_weight is different | |
| # key module.text_branch.resblocks.11.attn.in_proj_bias is different | |
| # key module.text_branch.resblocks.11.attn.out_proj.weight is different | |
| # key module.text_branch.resblocks.11.attn.out_proj.bias is different | |
| # key module.text_branch.resblocks.11.ln_1.weight is different | |
| # key module.text_branch.resblocks.11.ln_1.bias is different | |
| # key module.text_branch.resblocks.11.mlp.c_fc.weight is different | |
| # key module.text_branch.resblocks.11.mlp.c_fc.bias is different | |
| # key module.text_branch.resblocks.11.mlp.c_proj.weight is different | |
| # key module.text_branch.resblocks.11.mlp.c_proj.bias is different | |
| # key module.text_branch.resblocks.11.ln_2.weight is different | |
| # key module.text_branch.resblocks.11.ln_2.bias is different | |
| # a_sum: tensor(12113.6445) | |
| # b_sum: tensor(9883.4424) | |
| # diff: tensor(2230.2021) | |
| # True | |
| # Transformer freeze: | |
| # check_ckpt_diff("/fsx/clap_logs/2022_09_16-18_55_10-model_PANN-14-lr_0.001-b_160-j_4-p_fp32/checkpoints/epoch_10.pt", "/fsx/clap_logs/2022_09_16-18_55_10-model_PANN-14-lr_0.001-b_160-j_4-p_fp32/checkpoints/epoch_100.pt", "text_branch.resblocks") | |
| # key module.text_branch.resblocks.0.attn.in_proj_weight is different | |
| # key module.text_branch.resblocks.0.attn.in_proj_bias is different | |
| # key module.text_branch.resblocks.0.attn.out_proj.weight is different | |
| # key module.text_branch.resblocks.0.attn.out_proj.bias is different | |
| # key module.text_branch.resblocks.0.ln_1.weight is different | |
| # key module.text_branch.resblocks.0.ln_1.bias is different | |
| # key module.text_branch.resblocks.0.mlp.c_fc.weight is different | |
| # key module.text_branch.resblocks.0.mlp.c_fc.bias is different | |
| # key module.text_branch.resblocks.0.mlp.c_proj.weight is different | |
| # key module.text_branch.resblocks.0.mlp.c_proj.bias is different | |
| # key module.text_branch.resblocks.0.ln_2.weight is different | |
| # key module.text_branch.resblocks.0.ln_2.bias is different | |
| # key module.text_branch.resblocks.1.attn.in_proj_weight is different | |
| # key module.text_branch.resblocks.1.attn.in_proj_bias is different | |
| # key module.text_branch.resblocks.1.attn.out_proj.weight is different | |
| # key module.text_branch.resblocks.1.attn.out_proj.bias is different | |
| # key module.text_branch.resblocks.1.ln_1.weight is different | |
| # key module.text_branch.resblocks.1.ln_1.bias is different | |
| # key module.text_branch.resblocks.1.mlp.c_fc.weight is different | |
| # key module.text_branch.resblocks.1.mlp.c_fc.bias is different | |
| # key module.text_branch.resblocks.1.mlp.c_proj.weight is different | |
| # key module.text_branch.resblocks.1.mlp.c_proj.bias is different | |
| # key module.text_branch.resblocks.1.ln_2.weight is different | |
| # key module.text_branch.resblocks.1.ln_2.bias is different | |
| # key module.text_branch.resblocks.2.attn.in_proj_weight is different | |
| # key module.text_branch.resblocks.2.attn.in_proj_bias is different | |
| # key module.text_branch.resblocks.2.attn.out_proj.weight is different | |
| # key module.text_branch.resblocks.2.attn.out_proj.bias is different | |
| # key module.text_branch.resblocks.2.ln_1.weight is different | |
| # key module.text_branch.resblocks.2.ln_1.bias is different | |
| # key module.text_branch.resblocks.2.mlp.c_fc.weight is different | |
| # key module.text_branch.resblocks.2.mlp.c_fc.bias is different | |
| # key module.text_branch.resblocks.2.mlp.c_proj.weight is different | |
| # key module.text_branch.resblocks.2.mlp.c_proj.bias is different | |
| # key module.text_branch.resblocks.2.ln_2.weight is different | |
| # key module.text_branch.resblocks.2.ln_2.bias is different | |
| # key module.text_branch.resblocks.3.attn.in_proj_weight is different | |
| # key module.text_branch.resblocks.3.attn.in_proj_bias is different | |
| # key module.text_branch.resblocks.3.attn.out_proj.weight is different | |
| # key module.text_branch.resblocks.3.attn.out_proj.bias is different | |
| # key module.text_branch.resblocks.3.ln_1.weight is different | |
| # key module.text_branch.resblocks.3.ln_1.bias is different | |
| # key module.text_branch.resblocks.3.mlp.c_fc.weight is different | |
| # key module.text_branch.resblocks.3.mlp.c_fc.bias is different | |
| # key module.text_branch.resblocks.3.mlp.c_proj.weight is different | |
| # key module.text_branch.resblocks.3.mlp.c_proj.bias is different | |
| # key module.text_branch.resblocks.3.ln_2.weight is different | |
| # key module.text_branch.resblocks.3.ln_2.bias is different | |
| # key module.text_branch.resblocks.4.attn.in_proj_weight is different | |
| # key module.text_branch.resblocks.4.attn.in_proj_bias is different | |
| # key module.text_branch.resblocks.4.attn.out_proj.weight is different | |
| # key module.text_branch.resblocks.4.attn.out_proj.bias is different | |
| # key module.text_branch.resblocks.4.ln_1.weight is different | |
| # key module.text_branch.resblocks.4.ln_1.bias is different | |
| # key module.text_branch.resblocks.4.mlp.c_fc.weight is different | |
| # key module.text_branch.resblocks.4.mlp.c_fc.bias is different | |
| # key module.text_branch.resblocks.4.mlp.c_proj.weight is different | |
| # key module.text_branch.resblocks.4.mlp.c_proj.bias is different | |
| # key module.text_branch.resblocks.4.ln_2.weight is different | |
| # key module.text_branch.resblocks.4.ln_2.bias is different | |
| # key module.text_branch.resblocks.5.attn.in_proj_weight is different | |
| # key module.text_branch.resblocks.5.attn.in_proj_bias is different | |
| # key module.text_branch.resblocks.5.attn.out_proj.weight is different | |
| # key module.text_branch.resblocks.5.attn.out_proj.bias is different | |
| # key module.text_branch.resblocks.5.ln_1.weight is different | |
| # key module.text_branch.resblocks.5.ln_1.bias is different | |
| # key module.text_branch.resblocks.5.mlp.c_fc.weight is different | |
| # key module.text_branch.resblocks.5.mlp.c_fc.bias is different | |
| # key module.text_branch.resblocks.5.mlp.c_proj.weight is different | |
| # key module.text_branch.resblocks.5.mlp.c_proj.bias is different | |
| # key module.text_branch.resblocks.5.ln_2.weight is different | |
| # key module.text_branch.resblocks.5.ln_2.bias is different | |
| # key module.text_branch.resblocks.6.attn.in_proj_weight is different | |
| # key module.text_branch.resblocks.6.attn.in_proj_bias is different | |
| # key module.text_branch.resblocks.6.attn.out_proj.weight is different | |
| # key module.text_branch.resblocks.6.attn.out_proj.bias is different | |
| # key module.text_branch.resblocks.6.ln_1.weight is different | |
| # key module.text_branch.resblocks.6.ln_1.bias is different | |
| # key module.text_branch.resblocks.6.mlp.c_fc.weight is different | |
| # key module.text_branch.resblocks.6.mlp.c_fc.bias is different | |
| # key module.text_branch.resblocks.6.mlp.c_proj.weight is different | |
| # key module.text_branch.resblocks.6.mlp.c_proj.bias is different | |
| # key module.text_branch.resblocks.6.ln_2.weight is different | |
| # key module.text_branch.resblocks.6.ln_2.bias is different | |
| # key module.text_branch.resblocks.7.attn.in_proj_weight is different | |
| # key module.text_branch.resblocks.7.attn.in_proj_bias is different | |
| # key module.text_branch.resblocks.7.attn.out_proj.weight is different | |
| # key module.text_branch.resblocks.7.attn.out_proj.bias is different | |
| # key module.text_branch.resblocks.7.ln_1.weight is different | |
| # key module.text_branch.resblocks.7.ln_1.bias is different | |
| # key module.text_branch.resblocks.7.mlp.c_fc.weight is different | |
| # key module.text_branch.resblocks.7.mlp.c_fc.bias is different | |
| # key module.text_branch.resblocks.7.mlp.c_proj.weight is different | |
| # key module.text_branch.resblocks.7.mlp.c_proj.bias is different | |
| # key module.text_branch.resblocks.7.ln_2.weight is different | |
| # key module.text_branch.resblocks.7.ln_2.bias is different | |
| # key module.text_branch.resblocks.8.attn.in_proj_weight is different | |
| # key module.text_branch.resblocks.8.attn.in_proj_bias is different | |
| # key module.text_branch.resblocks.8.attn.out_proj.weight is different | |
| # key module.text_branch.resblocks.8.attn.out_proj.bias is different | |
| # key module.text_branch.resblocks.8.ln_1.weight is different | |
| # key module.text_branch.resblocks.8.ln_1.bias is different | |
| # key module.text_branch.resblocks.8.mlp.c_fc.weight is different | |
| # key module.text_branch.resblocks.8.mlp.c_fc.bias is different | |
| # key module.text_branch.resblocks.8.mlp.c_proj.weight is different | |
| # key module.text_branch.resblocks.8.mlp.c_proj.bias is different | |
| # key module.text_branch.resblocks.8.ln_2.weight is different | |
| # key module.text_branch.resblocks.8.ln_2.bias is different | |
| # key module.text_branch.resblocks.9.attn.in_proj_weight is different | |
| # key module.text_branch.resblocks.9.attn.in_proj_bias is different | |
| # key module.text_branch.resblocks.9.attn.out_proj.weight is different | |
| # key module.text_branch.resblocks.9.attn.out_proj.bias is different | |
| # key module.text_branch.resblocks.9.ln_1.weight is different | |
| # key module.text_branch.resblocks.9.ln_1.bias is different | |
| # key module.text_branch.resblocks.9.mlp.c_fc.weight is different | |
| # key module.text_branch.resblocks.9.mlp.c_fc.bias is different | |
| # key module.text_branch.resblocks.9.mlp.c_proj.weight is different | |
| # key module.text_branch.resblocks.9.mlp.c_proj.bias is different | |
| # key module.text_branch.resblocks.9.ln_2.weight is different | |
| # key module.text_branch.resblocks.9.ln_2.bias is different | |
| # key module.text_branch.resblocks.10.attn.in_proj_weight is different | |
| # key module.text_branch.resblocks.10.attn.in_proj_bias is different | |
| # key module.text_branch.resblocks.10.attn.out_proj.weight is different | |
| # key module.text_branch.resblocks.10.attn.out_proj.bias is different | |
| # key module.text_branch.resblocks.10.ln_1.weight is different | |
| # key module.text_branch.resblocks.10.ln_1.bias is different | |
| # key module.text_branch.resblocks.10.mlp.c_fc.weight is different | |
| # key module.text_branch.resblocks.10.mlp.c_fc.bias is different | |
| # key module.text_branch.resblocks.10.mlp.c_proj.weight is different | |
| # key module.text_branch.resblocks.10.mlp.c_proj.bias is different | |
| # key module.text_branch.resblocks.10.ln_2.weight is different | |
| # key module.text_branch.resblocks.10.ln_2.bias is different | |
| # key module.text_branch.resblocks.11.attn.in_proj_weight is different | |
| # key module.text_branch.resblocks.11.attn.in_proj_bias is different | |
| # key module.text_branch.resblocks.11.attn.out_proj.weight is different | |
| # key module.text_branch.resblocks.11.attn.out_proj.bias is different | |
| # key module.text_branch.resblocks.11.ln_1.weight is different | |
| # key module.text_branch.resblocks.11.ln_1.bias is different | |
| # key module.text_branch.resblocks.11.mlp.c_fc.weight is different | |
| # key module.text_branch.resblocks.11.mlp.c_fc.bias is different | |
| # key module.text_branch.resblocks.11.mlp.c_proj.weight is different | |
| # key module.text_branch.resblocks.11.mlp.c_proj.bias is different | |
| # key module.text_branch.resblocks.11.ln_2.weight is different | |
| # key module.text_branch.resblocks.11.ln_2.bias is different | |
| # a_sum: tensor(12133.6348) | |
| # b_sum: tensor(10423.9521) | |
| # diff: tensor(1709.6826) | |
| # True | |
| # bert no freeze: | |
| # check_ckpt_diff("/fsx/clap_logs/2022_09_14-02_33_11-model_PANN-14-lr_0.0001-b_160-j_4-p_fp32/checkpoints/epoch_10.pt", "/fsx/clap_logs/2022_09_14-02_33_11-model_PANN-14-lr_0.0001-b_160-j_4-p_fp32/checkpoints/epoch_100.pt", "text_branch.encoder") | |
| # key module.text_branch.encoder.layer.0.attention.self.query.weight is different | |
| # key module.text_branch.encoder.layer.0.attention.self.query.bias is different | |
| # key module.text_branch.encoder.layer.0.attention.self.key.weight is different | |
| # key module.text_branch.encoder.layer.0.attention.self.key.bias is different | |
| # key module.text_branch.encoder.layer.0.attention.self.value.weight is different | |
| # key module.text_branch.encoder.layer.0.attention.self.value.bias is different | |
| # key module.text_branch.encoder.layer.0.attention.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.0.attention.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.0.attention.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.0.attention.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.0.intermediate.dense.weight is different | |
| # key module.text_branch.encoder.layer.0.intermediate.dense.bias is different | |
| # key module.text_branch.encoder.layer.0.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.0.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.0.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.0.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.1.attention.self.query.weight is different | |
| # key module.text_branch.encoder.layer.1.attention.self.query.bias is different | |
| # key module.text_branch.encoder.layer.1.attention.self.key.weight is different | |
| # key module.text_branch.encoder.layer.1.attention.self.key.bias is different | |
| # key module.text_branch.encoder.layer.1.attention.self.value.weight is different | |
| # key module.text_branch.encoder.layer.1.attention.self.value.bias is different | |
| # key module.text_branch.encoder.layer.1.attention.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.1.attention.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.1.attention.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.1.attention.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.1.intermediate.dense.weight is different | |
| # key module.text_branch.encoder.layer.1.intermediate.dense.bias is different | |
| # key module.text_branch.encoder.layer.1.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.1.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.1.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.1.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.2.attention.self.query.weight is different | |
| # key module.text_branch.encoder.layer.2.attention.self.query.bias is different | |
| # key module.text_branch.encoder.layer.2.attention.self.key.weight is different | |
| # key module.text_branch.encoder.layer.2.attention.self.key.bias is different | |
| # key module.text_branch.encoder.layer.2.attention.self.value.weight is different | |
| # key module.text_branch.encoder.layer.2.attention.self.value.bias is different | |
| # key module.text_branch.encoder.layer.2.attention.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.2.attention.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.2.attention.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.2.attention.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.2.intermediate.dense.weight is different | |
| # key module.text_branch.encoder.layer.2.intermediate.dense.bias is different | |
| # key module.text_branch.encoder.layer.2.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.2.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.2.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.2.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.3.attention.self.query.weight is different | |
| # key module.text_branch.encoder.layer.3.attention.self.query.bias is different | |
| # key module.text_branch.encoder.layer.3.attention.self.key.weight is different | |
| # key module.text_branch.encoder.layer.3.attention.self.key.bias is different | |
| # key module.text_branch.encoder.layer.3.attention.self.value.weight is different | |
| # key module.text_branch.encoder.layer.3.attention.self.value.bias is different | |
| # key module.text_branch.encoder.layer.3.attention.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.3.attention.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.3.attention.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.3.attention.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.3.intermediate.dense.weight is different | |
| # key module.text_branch.encoder.layer.3.intermediate.dense.bias is different | |
| # key module.text_branch.encoder.layer.3.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.3.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.3.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.3.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.4.attention.self.query.weight is different | |
| # key module.text_branch.encoder.layer.4.attention.self.query.bias is different | |
| # key module.text_branch.encoder.layer.4.attention.self.key.weight is different | |
| # key module.text_branch.encoder.layer.4.attention.self.key.bias is different | |
| # key module.text_branch.encoder.layer.4.attention.self.value.weight is different | |
| # key module.text_branch.encoder.layer.4.attention.self.value.bias is different | |
| # key module.text_branch.encoder.layer.4.attention.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.4.attention.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.4.attention.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.4.attention.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.4.intermediate.dense.weight is different | |
| # key module.text_branch.encoder.layer.4.intermediate.dense.bias is different | |
| # key module.text_branch.encoder.layer.4.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.4.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.4.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.4.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.5.attention.self.query.weight is different | |
| # key module.text_branch.encoder.layer.5.attention.self.query.bias is different | |
| # key module.text_branch.encoder.layer.5.attention.self.key.weight is different | |
| # key module.text_branch.encoder.layer.5.attention.self.key.bias is different | |
| # key module.text_branch.encoder.layer.5.attention.self.value.weight is different | |
| # key module.text_branch.encoder.layer.5.attention.self.value.bias is different | |
| # key module.text_branch.encoder.layer.5.attention.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.5.attention.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.5.attention.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.5.attention.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.5.intermediate.dense.weight is different | |
| # key module.text_branch.encoder.layer.5.intermediate.dense.bias is different | |
| # key module.text_branch.encoder.layer.5.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.5.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.5.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.5.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.6.attention.self.query.weight is different | |
| # key module.text_branch.encoder.layer.6.attention.self.query.bias is different | |
| # key module.text_branch.encoder.layer.6.attention.self.key.weight is different | |
| # key module.text_branch.encoder.layer.6.attention.self.key.bias is different | |
| # key module.text_branch.encoder.layer.6.attention.self.value.weight is different | |
| # key module.text_branch.encoder.layer.6.attention.self.value.bias is different | |
| # key module.text_branch.encoder.layer.6.attention.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.6.attention.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.6.attention.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.6.attention.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.6.intermediate.dense.weight is different | |
| # key module.text_branch.encoder.layer.6.intermediate.dense.bias is different | |
| # key module.text_branch.encoder.layer.6.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.6.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.6.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.6.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.7.attention.self.query.weight is different | |
| # key module.text_branch.encoder.layer.7.attention.self.query.bias is different | |
| # key module.text_branch.encoder.layer.7.attention.self.key.weight is different | |
| # key module.text_branch.encoder.layer.7.attention.self.key.bias is different | |
| # key module.text_branch.encoder.layer.7.attention.self.value.weight is different | |
| # key module.text_branch.encoder.layer.7.attention.self.value.bias is different | |
| # key module.text_branch.encoder.layer.7.attention.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.7.attention.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.7.attention.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.7.attention.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.7.intermediate.dense.weight is different | |
| # key module.text_branch.encoder.layer.7.intermediate.dense.bias is different | |
| # key module.text_branch.encoder.layer.7.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.7.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.7.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.7.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.8.attention.self.query.weight is different | |
| # key module.text_branch.encoder.layer.8.attention.self.query.bias is different | |
| # key module.text_branch.encoder.layer.8.attention.self.key.weight is different | |
| # key module.text_branch.encoder.layer.8.attention.self.key.bias is different | |
| # key module.text_branch.encoder.layer.8.attention.self.value.weight is different | |
| # key module.text_branch.encoder.layer.8.attention.self.value.bias is different | |
| # key module.text_branch.encoder.layer.8.attention.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.8.attention.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.8.attention.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.8.attention.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.8.intermediate.dense.weight is different | |
| # key module.text_branch.encoder.layer.8.intermediate.dense.bias is different | |
| # key module.text_branch.encoder.layer.8.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.8.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.8.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.8.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.9.attention.self.query.weight is different | |
| # key module.text_branch.encoder.layer.9.attention.self.query.bias is different | |
| # key module.text_branch.encoder.layer.9.attention.self.key.weight is different | |
| # key module.text_branch.encoder.layer.9.attention.self.key.bias is different | |
| # key module.text_branch.encoder.layer.9.attention.self.value.weight is different | |
| # key module.text_branch.encoder.layer.9.attention.self.value.bias is different | |
| # key module.text_branch.encoder.layer.9.attention.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.9.attention.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.9.attention.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.9.attention.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.9.intermediate.dense.weight is different | |
| # key module.text_branch.encoder.layer.9.intermediate.dense.bias is different | |
| # key module.text_branch.encoder.layer.9.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.9.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.9.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.9.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.10.attention.self.query.weight is different | |
| # key module.text_branch.encoder.layer.10.attention.self.query.bias is different | |
| # key module.text_branch.encoder.layer.10.attention.self.key.weight is different | |
| # key module.text_branch.encoder.layer.10.attention.self.key.bias is different | |
| # key module.text_branch.encoder.layer.10.attention.self.value.weight is different | |
| # key module.text_branch.encoder.layer.10.attention.self.value.bias is different | |
| # key module.text_branch.encoder.layer.10.attention.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.10.attention.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.10.attention.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.10.attention.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.10.intermediate.dense.weight is different | |
| # key module.text_branch.encoder.layer.10.intermediate.dense.bias is different | |
| # key module.text_branch.encoder.layer.10.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.10.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.10.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.10.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.11.attention.self.query.weight is different | |
| # key module.text_branch.encoder.layer.11.attention.self.query.bias is different | |
| # key module.text_branch.encoder.layer.11.attention.self.key.weight is different | |
| # key module.text_branch.encoder.layer.11.attention.self.key.bias is different | |
| # key module.text_branch.encoder.layer.11.attention.self.value.weight is different | |
| # key module.text_branch.encoder.layer.11.attention.self.value.bias is different | |
| # key module.text_branch.encoder.layer.11.attention.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.11.attention.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.11.attention.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.11.attention.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.11.intermediate.dense.weight is different | |
| # key module.text_branch.encoder.layer.11.intermediate.dense.bias is different | |
| # key module.text_branch.encoder.layer.11.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.11.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.11.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.11.output.LayerNorm.bias is different | |
| # a_sum: tensor(15185.1230) | |
| # b_sum: tensor(15576.5596) | |
| # diff: tensor(-391.4365) | |
| # True | |
| # bert freeze: | |
| # check_ckpt_diff("/fsx/clap_logs/2022_09_13-01_25_15-model_PANN-14-lr_0.0001-b_160-j_4-p_fp32/checkpoints/epoch_10.pt", "/fsx/clap_logs/2022_09_13-01_25_15-model_PANN-14-lr_0.0001-b_160-j_4-p_fp32/checkpoints/epoch_100.pt", "text_branch.encoder") | |
| # key module.text_branch.encoder.layer.0.attention.self.query.weight is different | |
| # key module.text_branch.encoder.layer.0.attention.self.query.bias is different | |
| # key module.text_branch.encoder.layer.0.attention.self.key.weight is different | |
| # key module.text_branch.encoder.layer.0.attention.self.key.bias is different | |
| # key module.text_branch.encoder.layer.0.attention.self.value.weight is different | |
| # key module.text_branch.encoder.layer.0.attention.self.value.bias is different | |
| # key module.text_branch.encoder.layer.0.attention.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.0.attention.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.0.attention.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.0.attention.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.0.intermediate.dense.weight is different | |
| # key module.text_branch.encoder.layer.0.intermediate.dense.bias is different | |
| # key module.text_branch.encoder.layer.0.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.0.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.0.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.0.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.1.attention.self.query.weight is different | |
| # key module.text_branch.encoder.layer.1.attention.self.query.bias is different | |
| # key module.text_branch.encoder.layer.1.attention.self.key.weight is different | |
| # key module.text_branch.encoder.layer.1.attention.self.key.bias is different | |
| # key module.text_branch.encoder.layer.1.attention.self.value.weight is different | |
| # key module.text_branch.encoder.layer.1.attention.self.value.bias is different | |
| # key module.text_branch.encoder.layer.1.attention.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.1.attention.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.1.attention.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.1.attention.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.1.intermediate.dense.weight is different | |
| # key module.text_branch.encoder.layer.1.intermediate.dense.bias is different | |
| # key module.text_branch.encoder.layer.1.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.1.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.1.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.1.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.2.attention.self.query.weight is different | |
| # key module.text_branch.encoder.layer.2.attention.self.query.bias is different | |
| # key module.text_branch.encoder.layer.2.attention.self.key.weight is different | |
| # key module.text_branch.encoder.layer.2.attention.self.key.bias is different | |
| # key module.text_branch.encoder.layer.2.attention.self.value.weight is different | |
| # key module.text_branch.encoder.layer.2.attention.self.value.bias is different | |
| # key module.text_branch.encoder.layer.2.attention.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.2.attention.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.2.attention.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.2.attention.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.2.intermediate.dense.weight is different | |
| # key module.text_branch.encoder.layer.2.intermediate.dense.bias is different | |
| # key module.text_branch.encoder.layer.2.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.2.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.2.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.2.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.3.attention.self.query.weight is different | |
| # key module.text_branch.encoder.layer.3.attention.self.query.bias is different | |
| # key module.text_branch.encoder.layer.3.attention.self.key.weight is different | |
| # key module.text_branch.encoder.layer.3.attention.self.key.bias is different | |
| # key module.text_branch.encoder.layer.3.attention.self.value.weight is different | |
| # key module.text_branch.encoder.layer.3.attention.self.value.bias is different | |
| # key module.text_branch.encoder.layer.3.attention.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.3.attention.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.3.attention.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.3.attention.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.3.intermediate.dense.weight is different | |
| # key module.text_branch.encoder.layer.3.intermediate.dense.bias is different | |
| # key module.text_branch.encoder.layer.3.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.3.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.3.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.3.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.4.attention.self.query.weight is different | |
| # key module.text_branch.encoder.layer.4.attention.self.query.bias is different | |
| # key module.text_branch.encoder.layer.4.attention.self.key.weight is different | |
| # key module.text_branch.encoder.layer.4.attention.self.key.bias is different | |
| # key module.text_branch.encoder.layer.4.attention.self.value.weight is different | |
| # key module.text_branch.encoder.layer.4.attention.self.value.bias is different | |
| # key module.text_branch.encoder.layer.4.attention.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.4.attention.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.4.attention.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.4.attention.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.4.intermediate.dense.weight is different | |
| # key module.text_branch.encoder.layer.4.intermediate.dense.bias is different | |
| # key module.text_branch.encoder.layer.4.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.4.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.4.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.4.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.5.attention.self.query.weight is different | |
| # key module.text_branch.encoder.layer.5.attention.self.query.bias is different | |
| # key module.text_branch.encoder.layer.5.attention.self.key.weight is different | |
| # key module.text_branch.encoder.layer.5.attention.self.key.bias is different | |
| # key module.text_branch.encoder.layer.5.attention.self.value.weight is different | |
| # key module.text_branch.encoder.layer.5.attention.self.value.bias is different | |
| # key module.text_branch.encoder.layer.5.attention.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.5.attention.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.5.attention.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.5.attention.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.5.intermediate.dense.weight is different | |
| # key module.text_branch.encoder.layer.5.intermediate.dense.bias is different | |
| # key module.text_branch.encoder.layer.5.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.5.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.5.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.5.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.6.attention.self.query.weight is different | |
| # key module.text_branch.encoder.layer.6.attention.self.query.bias is different | |
| # key module.text_branch.encoder.layer.6.attention.self.key.weight is different | |
| # key module.text_branch.encoder.layer.6.attention.self.key.bias is different | |
| # key module.text_branch.encoder.layer.6.attention.self.value.weight is different | |
| # key module.text_branch.encoder.layer.6.attention.self.value.bias is different | |
| # key module.text_branch.encoder.layer.6.attention.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.6.attention.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.6.attention.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.6.attention.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.6.intermediate.dense.weight is different | |
| # key module.text_branch.encoder.layer.6.intermediate.dense.bias is different | |
| # key module.text_branch.encoder.layer.6.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.6.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.6.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.6.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.7.attention.self.query.weight is different | |
| # key module.text_branch.encoder.layer.7.attention.self.query.bias is different | |
| # key module.text_branch.encoder.layer.7.attention.self.key.weight is different | |
| # key module.text_branch.encoder.layer.7.attention.self.key.bias is different | |
| # key module.text_branch.encoder.layer.7.attention.self.value.weight is different | |
| # key module.text_branch.encoder.layer.7.attention.self.value.bias is different | |
| # key module.text_branch.encoder.layer.7.attention.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.7.attention.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.7.attention.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.7.attention.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.7.intermediate.dense.weight is different | |
| # key module.text_branch.encoder.layer.7.intermediate.dense.bias is different | |
| # key module.text_branch.encoder.layer.7.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.7.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.7.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.7.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.8.attention.self.query.weight is different | |
| # key module.text_branch.encoder.layer.8.attention.self.query.bias is different | |
| # key module.text_branch.encoder.layer.8.attention.self.key.weight is different | |
| # key module.text_branch.encoder.layer.8.attention.self.key.bias is different | |
| # key module.text_branch.encoder.layer.8.attention.self.value.weight is different | |
| # key module.text_branch.encoder.layer.8.attention.self.value.bias is different | |
| # key module.text_branch.encoder.layer.8.attention.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.8.attention.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.8.attention.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.8.attention.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.8.intermediate.dense.weight is different | |
| # key module.text_branch.encoder.layer.8.intermediate.dense.bias is different | |
| # key module.text_branch.encoder.layer.8.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.8.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.8.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.8.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.9.attention.self.query.weight is different | |
| # key module.text_branch.encoder.layer.9.attention.self.query.bias is different | |
| # key module.text_branch.encoder.layer.9.attention.self.key.weight is different | |
| # key module.text_branch.encoder.layer.9.attention.self.key.bias is different | |
| # key module.text_branch.encoder.layer.9.attention.self.value.weight is different | |
| # key module.text_branch.encoder.layer.9.attention.self.value.bias is different | |
| # key module.text_branch.encoder.layer.9.attention.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.9.attention.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.9.attention.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.9.attention.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.9.intermediate.dense.weight is different | |
| # key module.text_branch.encoder.layer.9.intermediate.dense.bias is different | |
| # key module.text_branch.encoder.layer.9.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.9.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.9.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.9.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.10.attention.self.query.weight is different | |
| # key module.text_branch.encoder.layer.10.attention.self.query.bias is different | |
| # key module.text_branch.encoder.layer.10.attention.self.key.weight is different | |
| # key module.text_branch.encoder.layer.10.attention.self.key.bias is different | |
| # key module.text_branch.encoder.layer.10.attention.self.value.weight is different | |
| # key module.text_branch.encoder.layer.10.attention.self.value.bias is different | |
| # key module.text_branch.encoder.layer.10.attention.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.10.attention.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.10.attention.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.10.attention.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.10.intermediate.dense.weight is different | |
| # key module.text_branch.encoder.layer.10.intermediate.dense.bias is different | |
| # key module.text_branch.encoder.layer.10.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.10.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.10.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.10.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.11.attention.self.query.weight is different | |
| # key module.text_branch.encoder.layer.11.attention.self.query.bias is different | |
| # key module.text_branch.encoder.layer.11.attention.self.key.weight is different | |
| # key module.text_branch.encoder.layer.11.attention.self.key.bias is different | |
| # key module.text_branch.encoder.layer.11.attention.self.value.weight is different | |
| # key module.text_branch.encoder.layer.11.attention.self.value.bias is different | |
| # key module.text_branch.encoder.layer.11.attention.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.11.attention.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.11.attention.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.11.attention.output.LayerNorm.bias is different | |
| # key module.text_branch.encoder.layer.11.intermediate.dense.weight is different | |
| # key module.text_branch.encoder.layer.11.intermediate.dense.bias is different | |
| # key module.text_branch.encoder.layer.11.output.dense.weight is different | |
| # key module.text_branch.encoder.layer.11.output.dense.bias is different | |
| # key module.text_branch.encoder.layer.11.output.LayerNorm.weight is different | |
| # key module.text_branch.encoder.layer.11.output.LayerNorm.bias is different | |
| # a_sum: tensor(15078.6641) | |
| # b_sum: tensor(15540.0723) | |
| # diff: tensor(-461.4082) | |
| # True | |
| # linear_prob_text | |
| # check_ckpt_diff("/fsx/clap_logs/2022_09_15-02_05_29-linear_probemodel_PANN-14-lr_0.0001-b_512-j_4-p_fp32/checkpoints/pretrain_epoch_10_lp_epoch_50.pt", "/fsx/clap_logs/2022_09_15-02_05_29-linear_probemodel_PANN-14-lr_0.0001-b_512-j_4-p_fp32/checkpoints/pretrain_epoch_10_lp_epoch_100.pt", "text_branch.resblocks") | |
| # a_sum: tensor(12111.0244) | |
| # b_sum: tensor(12111.0244) | |
| # diff: tensor(0.) | |
| # linear_prob_audio | |
| # check_ckpt_diff("/fsx/clap_logs/2022_09_15-02_05_29-linear_probemodel_PANN-14-lr_0.0001-b_512-j_4-p_fp32/checkpoints/pretrain_epoch_10_lp_epoch_50.pt", "/fsx/clap_logs/2022_09_15-02_05_29-linear_probemodel_PANN-14-lr_0.0001-b_512-j_4-p_fp32/checkpoints/pretrain_epoch_10_lp_epoch_100.pt", "clap_model") | |
| # key clap_model.audio_branch.bn0.num_batches_tracked is different | |
| # key clap_model.audio_branch.conv_block1.bn1.running_mean is different | |
| # key clap_model.audio_branch.conv_block1.bn1.running_var is different | |
| # key clap_model.audio_branch.conv_block1.bn1.num_batches_tracked is different | |
| # key clap_model.audio_branch.conv_block1.bn2.running_mean is different | |
| # key clap_model.audio_branch.conv_block1.bn2.running_var is different | |
| # key clap_model.audio_branch.conv_block1.bn2.num_batches_tracked is different | |
| # key clap_model.audio_branch.conv_block2.bn1.running_mean is different | |
| # key clap_model.audio_branch.conv_block2.bn1.running_var is different | |
| # key clap_model.audio_branch.conv_block2.bn1.num_batches_tracked is different | |
| # key clap_model.audio_branch.conv_block2.bn2.running_mean is different | |
| # key clap_model.audio_branch.conv_block2.bn2.running_var is different | |
| # key clap_model.audio_branch.conv_block2.bn2.num_batches_tracked is different | |
| # key clap_model.audio_branch.conv_block3.bn1.running_mean is different | |
| # key clap_model.audio_branch.conv_block3.bn1.running_var is different | |
| # key clap_model.audio_branch.conv_block3.bn1.num_batches_tracked is different | |
| # key clap_model.audio_branch.conv_block3.bn2.running_mean is different | |
| # key clap_model.audio_branch.conv_block3.bn2.running_var is different | |
| # key clap_model.audio_branch.conv_block3.bn2.num_batches_tracked is different | |
| # key clap_model.audio_branch.conv_block4.bn1.running_mean is different | |
| # key clap_model.audio_branch.conv_block4.bn1.running_var is different | |
| # key clap_model.audio_branch.conv_block4.bn1.num_batches_tracked is different | |
| # key clap_model.audio_branch.conv_block4.bn2.running_mean is different | |
| # key clap_model.audio_branch.conv_block4.bn2.running_var is different | |
| # key clap_model.audio_branch.conv_block4.bn2.num_batches_tracked is different | |
| # key clap_model.audio_branch.conv_block5.bn1.running_mean is different | |
| # key clap_model.audio_branch.conv_block5.bn1.running_var is different | |
| # key clap_model.audio_branch.conv_block5.bn1.num_batches_tracked is different | |
| # key clap_model.audio_branch.conv_block5.bn2.running_mean is different | |
| # key clap_model.audio_branch.conv_block5.bn2.running_var is different | |
| # key clap_model.audio_branch.conv_block5.bn2.num_batches_tracked is different | |
| # key clap_model.audio_branch.conv_block6.bn1.running_mean is different | |
| # key clap_model.audio_branch.conv_block6.bn1.running_var is different | |
| # key clap_model.audio_branch.conv_block6.bn1.num_batches_tracked is different | |
| # key clap_model.audio_branch.conv_block6.bn2.running_mean is different | |
| # key clap_model.audio_branch.conv_block6.bn2.running_var is different | |
| # key clap_model.audio_branch.conv_block6.bn2.num_batches_tracked is different | |
| # a_sum: tensor(120061.5078) | |
| # b_sum: tensor(122656.0469) | |
| # diff: tensor(-2594.5391) | |
| # True | |