import sys, os root = os.sep + os.sep.join(__file__.split(os.sep)[1:__file__.split(os.sep).index("Recurrent-Parameter-Generation")+1]) sys.path.append(root) os.chdir(root) import random import pandas as pd import numpy as np import torch import pickle import importlib item = importlib.import_module(f"dataset.{sys.argv[1]}.train") loader = item.test_loader model = item.model test = item.test # tag in this file is only the name of specific dirname instead of config["tag"] tag = os.path.basename(os.path.dirname(item.__file__)) config = { "checkpoint_path": f"./dataset/{tag}/checkpoint", "generated_path": f"./dataset/{tag}/generated", "cache_file": None, # None means default dataset/tag/cache.pt "resume": True, # if you updated the checkpoint and generated models, use "resume": False. "noise_intensity": [0.01, 0.02, 0.03, 0.04, 0.05], "total_noised_number": 25, } assert config["total_noised_number"] % len(config["noise_intensity"]) == 0, \ "total_noised_number must be a multiple of noise_intensity" globals().update(config) # load paths checkpoint_items = [os.path.join(checkpoint_path, i) for i in os.listdir(checkpoint_path)] generated_items = [os.path.join(generated_path, i) for i in os.listdir(generated_path)] generated_items.sort() total_items = list(checkpoint_items) + list(generated_items) num_checkpoint = len(checkpoint_items) num_generated = len(generated_items) # define compute IoU @torch.no_grad() def compute_wrong_indices(diction): model.load_state_dict(diction, strict=False) model.eval() _, acc, all_targets, all_predicts = test(model=model) not_agreement = torch.logical_not(torch.eq(torch.tensor(all_targets), torch.tensor(all_predicts))) return not_agreement, acc def compute_wrong_iou(a, b): inter = np.logical_and(a, b) union = np.logical_or(a, b) iou = np.sum(inter) / np.sum(union) return iou # prepare evaluate print("\n==> start evaluating..") total_result_list = [] total_acc_list = [] cache_file = os.path.join(os.path.dirname(checkpoint_path), "performance.cache") if cache_file is None else cache_file if resume is True and os.path.exists(cache_file): print(f"load cache from {cache_file}") with open(cache_file, "rb") as f: total_result_list, total_acc_list = pickle.load(f) else: # compute checkpoint and generated print(f"start inferencing on {tag}") for i, item in enumerate(total_items): print(f"start: {i+1}/{len(total_items)}") item = torch.load(item, map_location='cpu') result, acc = compute_wrong_indices(item) result = result.numpy() total_result_list.append(result) total_acc_list.append(acc) with open(cache_file, "wb") as f: pickle.dump([total_result_list, total_acc_list], f) # compute noised checkpoint_items_for_noise = checkpoint_items.copy() random.shuffle(checkpoint_items_for_noise) num_each_noised = total_noised_number // len(noise_intensity) num_noise_class = len(noise_intensity) bias = 0 for this_noise_intensity in noise_intensity: for i in range(num_each_noised): i = i + bias print(f"testing noised: {i+1}/{num_each_noised * num_noise_class}") item = checkpoint_items_for_noise[i % num_checkpoint] item = torch.load(item, map_location="cpu") new_diction = {} for key, value in item.items(): if ("num_batches_tracked" in key) or (value.numel() == 1) or not torch.is_floating_point(value): pass # not add noise to these elif "running_var" in key: pre_mean = value.mean() * 0.95 value = torch.log(value / pre_mean + 0.05) mean, std = value.mean(), value.std() value = (value - mean) / std value += torch.randn_like(value) * this_noise_intensity value = value * std + mean value = torch.clip(torch.exp(value) - 0.05, min=0.001) * pre_mean else: # conv & linear mean, std = value.mean(), value.std() value = (value - mean) / std value += torch.randn_like(value) * this_noise_intensity value = value * std + mean new_diction[key] = value result, acc = compute_wrong_indices(new_diction) result = result.numpy() total_result_list.append(result) total_acc_list.append(acc) bias += num_each_noised # compute iou_metrix print("start computing IoU...") total_num = num_checkpoint + num_generated + num_each_noised * num_noise_class assert total_num == len(total_result_list), \ f"total_num:{total_num}, len(total_result_list):{len(total_result_list)}" iou_matrix = np.zeros(shape=[total_num, total_num]) for i in range(total_num): for j in range(total_num): iou = compute_wrong_iou(total_result_list[i], total_result_list[j]) iou_matrix[i, j] = iou # save result df = pd.DataFrame(iou_matrix) df.to_excel(f"./iou_{tag}.xlsx", index=False) print(f"finished Saving ./iou_{tag}.xlsx!") # print summary print("\n\n===============================================") print(f"Summary: {tag}") print() print("num_checkpoint:", num_checkpoint) print("num_generated:", num_generated) print(f"num_noised: {num_each_noised}x{num_noise_class}") print(f"original_acc_mean:", np.array(total_acc_list[:num_checkpoint]).mean()) print(f"original_acc_max:", np.array(total_acc_list[:num_checkpoint]).max()) print(f"generated_acc_mean:", np.array(total_acc_list[num_checkpoint:num_checkpoint+num_generated]).mean()) print(f"generated_acc_max:", np.array(total_acc_list[num_checkpoint:num_checkpoint+num_generated]).max()) this_start = num_checkpoint + num_generated for this_noise_intensity in noise_intensity: print(f"noise={this_noise_intensity:.4f}_acc_mean:", np.array(total_acc_list[this_start:this_start+num_each_noised]).mean()) this_start += num_each_noised print() # empty line origin_origin = iou_matrix[:num_checkpoint, :num_checkpoint] origin_origin = (np.sum(origin_origin) - num_checkpoint) / (num_checkpoint * (num_checkpoint - 1)) print("origin-origin:", origin_origin) generated_generated = iou_matrix[num_checkpoint:num_checkpoint + num_generated, num_checkpoint:num_checkpoint + num_generated] generated_generated = (np.sum(generated_generated) - num_generated) / (num_generated * (num_generated - 1)) print("generated-generated:", generated_generated) origin_generated = iou_matrix[num_checkpoint:num_checkpoint + num_generated, :num_checkpoint] origin_generated = np.mean(origin_generated) print("origin-generated:", origin_generated) origin_generated_max = iou_matrix[num_checkpoint:num_checkpoint + num_generated, :num_checkpoint] origin_generated_max = np.amax(origin_generated_max, axis=-1) print("origin-generated(max):", origin_generated_max.mean()) # print noised noised_max_list = [] this_start = num_checkpoint + num_generated for this_noise_intensity in noise_intensity: print(f"\nnoise_intensity={this_noise_intensity}") noised_noised = iou_matrix[this_start:this_start + num_each_noised, this_start:this_start + num_each_noised] noised_noised = (np.sum(noised_noised) - num_each_noised) / (num_each_noised * (num_each_noised - 1)) print("noised-noised:", noised_noised) origin_noised = iou_matrix[this_start:this_start + num_each_noised, :num_checkpoint] origin_noised = np.mean(origin_noised) print("origin-noised:", origin_noised) origin_noised_max = iou_matrix[this_start:this_start + num_each_noised, :num_checkpoint] origin_noised_max = np.amax(origin_noised_max, axis=-1) noised_max_list.append(origin_noised_max) print("origin-noised(max):", origin_noised_max.mean()) this_start += num_each_noised # save summary summary = { "summary": tag, "num_checkpoint": num_checkpoint, "num_generated": num_generated, "num_each_noised": num_each_noised, "num_noise_class": num_noise_class, "noise_intensity": noise_intensity, "total_acc_list": total_acc_list, "iou_matrix": iou_matrix, } draw_cache = [summary,] # final draw print("\n==> start drawing..") import seaborn as sns import matplotlib.pyplot as plt # origin draw_origin_origin_max = np.amax(iou_matrix[:num_checkpoint, :num_checkpoint] - np.eye(num_checkpoint), axis=-1) draw_origin_origin_acc = np.array(total_acc_list[:num_checkpoint]) sns.scatterplot(x=draw_origin_origin_max, y=draw_origin_origin_acc, label="origin") draw_cache.append(dict(x=draw_origin_origin_max, y=draw_origin_origin_acc, label="origin")) # generated draw_origin_generated_max = origin_generated_max draw_origin_generated_acc = np.array(total_acc_list[num_checkpoint:num_checkpoint + num_generated]) sns.scatterplot(x=draw_origin_generated_max, y=draw_origin_generated_acc, label="generated") draw_cache.append(dict(x=draw_origin_generated_max, y=draw_origin_generated_acc, label="generated")) # noised this_start = num_checkpoint + num_generated for i, this_noise_intensity in enumerate(noise_intensity): draw_origin_noised_max = noised_max_list[i] draw_origin_noised_acc = total_acc_list[this_start: this_start+num_each_noised] sns.scatterplot(x=draw_origin_noised_max, y=draw_origin_noised_acc, label=f"noise={this_noise_intensity:.4f}") draw_cache.append(dict(x=draw_origin_noised_max, y=draw_origin_noised_acc, label=f"noise={this_noise_intensity:.4f}")) this_start += num_each_noised # draw plt.savefig(f'plot_{tag}.png') with open(f'plot_{tag}.cache', "wb") as f: pickle.dump(draw_cache, f) print(f"plot saved to plot_{tag}.png")