|
import os, sys |
|
|
|
|
|
import torch |
|
import numpy as np |
|
from umap.umap_ import find_ab_params |
|
import pickle |
|
import gc |
|
import base64 |
|
from .backend_adapter import TimeVisBackend, ActiveLearningTimeVisBackend, AnormalyTimeVisBackend |
|
|
|
timevis_path = "../../DLVisDebugger" |
|
sys.path.append(timevis_path) |
|
from singleVis.SingleVisualizationModel import VisModel |
|
from singleVis.losses import SingleVisLoss, UmapLoss, ReconstructionLoss, SmoothnessLoss, HybridLoss |
|
from singleVis.trainer import SingleVisTrainer, HybridVisTrainer |
|
from singleVis.data import NormalDataProvider, ActiveLearningDataProvider, DenseActiveLearningDataProvider |
|
from singleVis.eval.evaluator import Evaluator |
|
from singleVis.visualizer import visualizer, DenseALvisualizer |
|
from singleVis.projector import Projector, ALProjector, DenseALProjector |
|
from singleVis.segmenter import Segmenter |
|
|
|
|
|
|
|
def initialize_backend(CONTENT_PATH, dense_al=False): |
|
|
|
import config |
|
|
|
|
|
CLASSES = config["CLASSES"] |
|
DATASET = config["DATASET"] |
|
GPU_ID = config["GPU"] |
|
DEVICE = torch.device("cuda:{}".format(GPU_ID) if torch.cuda.is_available() else "cpu") |
|
|
|
PREPROCESS = config["VISUALIZATION"]["PREPROCESS"] |
|
B_N_EPOCHS = config["VISUALIZATION"]["BOUNDARY"]["B_N_EPOCHS"] |
|
L_BOUND = config["VISUALIZATION"]["BOUNDARY"]["L_BOUND"] |
|
LAMBDA = config["VISUALIZATION"]["LAMBDA"] |
|
|
|
ENCODER_DIMS = config["VISUALIZATION"]["ENCODER_DIMS"] |
|
DECODER_DIMS = config["VISUALIZATION"]["DECODER_DIMS"] |
|
N_NEIGHBORS = config["VISUALIZATION"]["N_NEIGHBORS"] |
|
MAX_EPOCH = config["VISUALIZATION"]["MAX_EPOCH"] |
|
S_N_EPOCHS = config["VISUALIZATION"]["S_N_EPOCHS"] |
|
PATIENT = config["VISUALIZATION"]["PATIENT"] |
|
VIS_MODEL_NAME = config["VISUALIZATION"]["VIS_MODEL_NAME"] |
|
RESOLUTION = config["VISUALIZATION"]["RESOLUTION"] |
|
EVALUATION_NAME = config["VISUALIZATION"]["EVALUATION_NAME"] |
|
NET = config["TRAINING"]["NET"] |
|
|
|
|
|
SETTING = config["SETTING"] |
|
if SETTING == "normal" or SETTING == "abnormal": |
|
EPOCH_START = config["EPOCH_START"] |
|
EPOCH_END = config["EPOCH_END"] |
|
EPOCH_PERIOD = config["EPOCH_PERIOD"] |
|
|
|
INIT_NUM = config["VISUALIZATION"]["INIT_NUM"] |
|
ALPHA = config["VISUALIZATION"]["ALPHA"] |
|
BETA = config["VISUALIZATION"]["BETA"] |
|
MAX_HAUSDORFF = config["VISUALIZATION"]["MAX_HAUSDORFF"] |
|
T_N_EPOCHS = config["VISUALIZATION"]["T_N_EPOCHS"] |
|
elif SETTING == "active learning": |
|
BASE_ITERATION = config["BASE_ITERATION"] |
|
TOTAL_EPOCH = config["TRAINING"]["total_epoch"] |
|
else: |
|
raise NotImplementedError |
|
|
|
import Model.model as subject_model |
|
net = eval("subject_model.{}()".format(NET)) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
model = VisModel(ENCODER_DIMS, DECODER_DIMS) |
|
|
|
if SETTING == "normal" or SETTING == "abnormal": |
|
data_provider = NormalDataProvider(CONTENT_PATH, net, EPOCH_START, EPOCH_END, EPOCH_PERIOD, device=DEVICE, classes=CLASSES,epoch_name="Epoch", verbose=1) |
|
segmenter = Segmenter(data_provider=data_provider, threshold=78.5, range_s=EPOCH_START, range_e=EPOCH_END, range_p=EPOCH_PERIOD) |
|
SEGMENTS = segmenter.segment() |
|
|
|
projector = Projector(vis_model=model, content_path=CONTENT_PATH, segments=SEGMENTS, device=DEVICE) |
|
elif SETTING == "active learning": |
|
DENSE_VIS_MODEL_NAME = config["VISUALIZATION"]["DENSE_VIS_MODEL_NAME"] |
|
if dense_al: |
|
data_provider = DenseActiveLearningDataProvider(CONTENT_PATH, net, BASE_ITERATION, epoch_num=TOTAL_EPOCH, split=-1, device=DEVICE, classes=CLASSES,verbose=1) |
|
projector = DenseALProjector(vis_model=model, content_path=CONTENT_PATH, vis_model_name=DENSE_VIS_MODEL_NAME, device=DEVICE) |
|
else: |
|
data_provider = ActiveLearningDataProvider(CONTENT_PATH, net, BASE_ITERATION, split=-1, device=DEVICE, classes=CLASSES, verbose=1) |
|
projector = ALProjector(vis_model=model, content_path=CONTENT_PATH, vis_model_name=VIS_MODEL_NAME, device=DEVICE) |
|
|
|
|
|
|
|
|
|
|
|
if SETTING == "active learning": |
|
negative_sample_rate = 5 |
|
min_dist = .1 |
|
_a, _b = find_ab_params(1.0, min_dist) |
|
umap_loss_fn = UmapLoss(negative_sample_rate, DEVICE, _a, _b, repulsion_strength=1.0) |
|
recon_loss_fn = ReconstructionLoss(beta=1.0) |
|
if dense_al: |
|
smooth_loss_fn = SmoothnessLoss(margin=1.) |
|
S_LAMBDA = config["VISUALIZATION"]["S_LAMBDA"] |
|
criterion = HybridLoss(umap_loss_fn, recon_loss_fn, smooth_loss_fn, lambd1=LAMBDA, lambd2=S_LAMBDA) |
|
optimizer = torch.optim.Adam(model.parameters(), lr=.01, weight_decay=1e-5) |
|
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=4, gamma=.1) |
|
trainer = HybridVisTrainer(model, criterion, optimizer, lr_scheduler,edge_loader=None, DEVICE=DEVICE) |
|
else: |
|
criterion = SingleVisLoss(umap_loss_fn, recon_loss_fn, lambd=LAMBDA) |
|
optimizer = torch.optim.Adam(model.parameters(), lr=.01, weight_decay=1e-5) |
|
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=4, gamma=.1) |
|
trainer = SingleVisTrainer(model, criterion, optimizer, lr_scheduler,edge_loader=None, DEVICE=DEVICE) |
|
|
|
|
|
|
|
|
|
|
|
if dense_al: |
|
vis = DenseALvisualizer(data_provider, projector, RESOLUTION) |
|
else: |
|
vis = visualizer(data_provider, projector, RESOLUTION) |
|
evaluator = Evaluator(data_provider, projector) |
|
|
|
if SETTING == "normal": |
|
timevis = TimeVisBackend(data_provider, projector, vis, evaluator, **config) |
|
elif SETTING == "abnormal": |
|
timevis = AnormalyTimeVisBackend(data_provider, projector, vis, evaluator, period=100, **config) |
|
elif SETTING == "active learning": |
|
timevis = ActiveLearningTimeVisBackend(data_provider, projector, trainer, vis, evaluator, dense_al, **config) |
|
|
|
del config |
|
gc.collect() |
|
return timevis |
|
|
|
|
|
def update_epoch_projection(timevis, EPOCH, predicates): |
|
train_data = timevis.data_provider.train_representation(EPOCH) |
|
test_data = timevis.data_provider.test_representation(EPOCH) |
|
all_data = np.concatenate((train_data, test_data), axis=0) |
|
|
|
fname = "Epoch" if timevis.data_provider.mode == "normal" or timevis.data_provider.mode == "abnormal" else "Iteration" |
|
embedding_path = os.path.join(timevis.data_provider.model_path, "{}_{}".format(fname, EPOCH), "embedding.npy") |
|
if os.path.exists(embedding_path): |
|
embedding_2d = np.load(embedding_path) |
|
else: |
|
embedding_2d = timevis.projector.batch_project(EPOCH, all_data) |
|
np.save(embedding_path, embedding_2d) |
|
|
|
train_labels = timevis.data_provider.train_labels(EPOCH) |
|
test_labels = timevis.data_provider.test_labels(EPOCH) |
|
labels = np.concatenate((train_labels, test_labels), axis=0).tolist() |
|
|
|
training_data_number = timevis.hyperparameters["TRAINING"]["train_num"] |
|
testing_data_number = timevis.hyperparameters["TRAINING"]["test_num"] |
|
training_data_index = list(range(training_data_number)) |
|
testing_data_index = list(range(training_data_number, training_data_number + testing_data_number)) |
|
|
|
|
|
|
|
fname = "Epoch" if timevis.data_provider.mode == "normal" or timevis.data_provider.mode == "abnormal" else "Iteration" |
|
bgimg_path = os.path.join(timevis.data_provider.model_path, "{}_{}".format(fname, EPOCH), "bgimg.png") |
|
grid_path = os.path.join(timevis.data_provider.model_path, "{}_{}".format(fname, EPOCH), "grid.pkl") |
|
if os.path.exists(bgimg_path) and os.path.exists(grid_path): |
|
with open(os.path.join(grid_path), "rb") as f: |
|
grid = pickle.load(f) |
|
with open(bgimg_path, 'rb') as img_f: |
|
img_stream = img_f.read() |
|
b_fig = base64.b64encode(img_stream).decode() |
|
else: |
|
x_min, y_min, x_max, y_max, b_fig = timevis.vis.get_background(EPOCH, timevis.hyperparameters["VISUALIZATION"]["RESOLUTION"]) |
|
grid = [x_min, y_min, x_max, y_max] |
|
|
|
grid = [float(i) for i in grid] |
|
b_fig = str(b_fig, encoding='utf-8') |
|
|
|
|
|
save_path = timevis.data_provider.model_path |
|
iteration_name = "Epoch" if timevis.data_provider.mode == "normal" or timevis.data_provider.mode == "abnormal" else "Iteration" |
|
save_path = os.path.join(save_path, "{}_{}".format(iteration_name, EPOCH)) |
|
with open(os.path.join(save_path, "grid.pkl"), "wb") as f: |
|
pickle.dump(grid, f) |
|
np.save(os.path.join(save_path, "embedding.npy"), embedding_2d) |
|
|
|
color = timevis.vis.get_standard_classes_color() * 255 |
|
color = color.astype(int).tolist() |
|
|
|
|
|
file_name = timevis.hyperparameters["VISUALIZATION"]["EVALUATION_NAME"] |
|
evaluation = timevis.evaluator.get_eval(file_name=file_name) |
|
eval_new = dict() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_new["train_acc"] = evaluation["train_acc"][str(EPOCH)] |
|
eval_new["test_acc"] = evaluation["test_acc"][str(EPOCH)] |
|
|
|
label_color_list = [] |
|
label_list = [] |
|
label_name_dict = dict() |
|
for i, label in enumerate(timevis.hyperparameters["CLASSES"]): |
|
label_name_dict[i] = label |
|
|
|
for label in labels: |
|
label_color_list.append(color[int(label)]) |
|
label_list.append(timevis.hyperparameters["CLASSES"][int(label)]) |
|
|
|
prediction_list = [] |
|
prediction = timevis.data_provider.get_pred(EPOCH, all_data).argmax(1) |
|
|
|
for i in range(len(prediction)): |
|
prediction_list.append(timevis.hyperparameters["CLASSES"][prediction[i]]) |
|
|
|
if timevis.hyperparameters["SETTING"] == "normal" or timevis.hyperparameters["SETTING"] == "abnormal": |
|
max_iter = (timevis.hyperparameters["EPOCH_END"] - timevis.hyperparameters["EPOCH_START"]) // timevis.hyperparameters["EPOCH_PERIOD"] + 1 |
|
elif timevis.hyperparameters["SETTING"] == "active learning": |
|
|
|
max_iter = timevis.get_max_iter() |
|
|
|
|
|
|
|
|
|
selected_points = np.arange(training_data_number + testing_data_number) |
|
for key in predicates.keys(): |
|
if key == "label": |
|
tmp = np.array(timevis.filter_label(predicates[key])) |
|
elif key == "type": |
|
tmp = np.array(timevis.filter_type(predicates[key], int(EPOCH))) |
|
else: |
|
tmp = np.arange(training_data_number + testing_data_number) |
|
selected_points = np.intersect1d(selected_points, tmp) |
|
|
|
properties = np.concatenate((np.zeros(training_data_number, dtype=np.int16), 2*np.ones(testing_data_number, dtype=np.int16)), axis=0) |
|
lb = timevis.get_epoch_index(EPOCH) |
|
ulb = np.setdiff1d(training_data_index, lb) |
|
properties[ulb] = 1 |
|
|
|
return embedding_2d.tolist(), grid, b_fig, label_name_dict, label_color_list, label_list, max_iter, training_data_index, testing_data_index, eval_new, prediction_list, selected_points, properties |
|
|