Kaggle competition: TGS Salt Identification Challenge
<SOS> metric: IntersectionOverUnionObjectSegmentation Kaggle data source: tgs-salt-identification-challenge<categorify>
plt.style.use('seaborn-white')
 sns.set_style("white")
 %matplotlib inline
class MoAStackingDataset(data.Dataset):
def __init__(self, feat: np.ndarray, label: np.ndarray = None):

 self.feat = feat
if label is None:
self.label = np.full(( len(feat), 1), -1)
 else:
self.label = label
self.model_order = None
def __len__(self):

 return len(self.feat)
 def __getitem__(self, index: int):

 return [
torch.from_numpy(self.feat[index] ).float() ,
torch.from_numpy(self.label[index] ).float()
]
def reset_model_order(self):

 pass
def shuffle_model_order(self, seed):

 pass
class MoAStackingDatasetForCNN(data.Dataset):
def __init__(self, feat: np.ndarray, label: np.ndarray = None):

 self.feat = feat
if label is None:
self.label = np.full(( len(feat), 1), -1)
 else:
self.label = label
self.reset_model_order()
def reset_model_order(self):
self.model_order = np.arange(self.feat.shape[-1])
 def shuffle_model_order(self, seed):
np.random.seed(seed)
 self.model_order = np.random.permutation(self.model_order)
 def __len__(self):

 return len(self.feat)
 def __getitem__(self, index: int):

 return [
torch.from_numpy(self.feat[index][..., self.model_order] ).float() ,
torch.from_numpy(self.label[index] ).float()
]
class MoAStackingDatasetForGCN(data.Dataset):
def __init__(self, feat: np.ndarray, label: np.ndarray = None):

 self.feat = feat
if label is None:
self.label = np.full(( len(feat), 1), -1)
 else:
self.label = label
self.model_order = None
def reset_model_order(self):

 pass
def shuffle_model_order(self, seed):

 pass
def __len__(self):

 return len(self.feat)
 def __getitem__(self, index: int):

 return [
torch.from_numpy(self.feat[index] ).float() ,
torch.from_numpy(self.label[index] ).float()
]<define_search_model>
im_width = 101
im_height = 101
im_chan = 1
basicpath = '.. /input/'
path_train = basicpath + 'train/'
path_test = basicpath + 'test/'
path_train_images = path_train + 'images/'
path_train_masks = path_train + 'masks/'
path_test_images = path_test + 'images/'
def get_activation(activ_name: str="relu"):

 act_dict = {
"relu": nn.ReLU() ,
"tanh": nn.Tanh() ,
"sigmoid": nn.Sigmoid() ,
"identity": nn.Identity() }
if activ_name in act_dict:
return act_dict[activ_name]
elif re.match(r"^htanh\_\d{4}$", activ_name):
bound = int(activ_name[-4:])/ 1000
return nn.Hardtanh(-bound, bound)
 else:
raise NotImplementedError
class LBAD(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(LBAD, self ).__init__()
layers = [nn.Linear(in_features, out_features)]
if use_wn:
layers[0] = nn.utils.weight_norm(layers[0])
 if use_bn:
layers.append(nn.BatchNorm1d(out_features))
layers.append(get_activation(activ))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class BDLA(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(BDLA, self ).__init__()
layers = []
if use_bn:
layers.append(nn.BatchNorm1d(in_features))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
layers.append(nn.Linear(in_features, out_features))
if use_wn:
layers[-1] = nn.utils.weight_norm(layers[-1])
 layers.append(get_activation(activ))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class LABD(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(LABD, self ).__init__()
layers = [nn.Linear(in_features, out_features), get_activation(activ)]
if use_wn:
layers[0] = nn.utils.weight_norm(layers[0])
 if use_bn:
layers.append(nn.BatchNorm1d(out_features))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x )<prepare_x_and_y>
img_size_ori = 101
img_size_target = 101
def upsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_target, img_size_target), mode='constant', preserve_range=True)
 def downsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_ori, img_size_ori), mode='constant', preserve_range=True)
def vector_wise_matmul(X: torch.Tensor, W: torch.Tensor)-> torch.Tensor:

 X = torch.transpose(X, 0, 1)
 W = torch.transpose(W, 1, 2)
 H = torch.matmul(X, W)
 H = torch.transpose(H, 0, 1)
 return H
def vector_wise_shared_matmul(X: torch.Tensor, W: torch.Tensor)-> torch.Tensor:

 H = nn.functional.linear(X, W)
 return H<normalization>
train_df = pd.read_csv(".. /input/train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(".. /input/depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]
len(train_df )
def _calculate_fan_in_and_fan_out_for_vwl(tensor)-> tp.Tuple[int]:

 dimensions = tensor.dim()
if dimensions < 2:
raise ValueError("Fan in and fan out can not be computed for tensor with fewer than 2 dimensions")
 fan_in = tensor.size(-1)
 fan_out = tensor.size(-2)
 return fan_in, fan_out
def _calculate_correct_fan_for_vwl(tensor, mode)-> int:

 mode = mode.lower()
valid_modes = ['fan_in', 'fan_out']
if mode not in valid_modes:
raise ValueError("Mode {} not supported, please use one of {}".format(mode, valid_modes))
fan_in, fan_out = _calculate_fan_in_and_fan_out_for_vwl(tensor)
 return fan_in if mode == 'fan_in' else fan_out
def kaiming_uniform_for_vwl(tensor, a=0, mode='fan_in', nonlinearity='leaky_relu'):

 fan = _calculate_correct_fan_for_vwl(tensor, mode)
 gain = nn.init.calculate_gain(nonlinearity, a)
 std = gain / np.sqrt(fan)
 bound = np.sqrt(3.0)* std
with torch.no_grad() :
return tensor.uniform_(-bound, bound )<prepare_x_and_y>
train_df["images"] = [np.array(load_img(".. /input/train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
class VectorWiseLinear(nn.Module):

 def __init__(
 self,
in_features: int, out_features: int, n_vectors: int,
bias: bool=True, weight_shared: bool=True 
)-> None:

 super(VectorWiseLinear, self ).__init__()
self.in_features = in_features
self.out_features = out_features
self.n_vectors = n_vectors
self.weight_shared = weight_shared
if self.weight_shared:
self.weight = nn.Parameter(
 torch.Tensor(self.out_features, self.in_features))
self.matmul_func = vector_wise_shared_matmul
else:
self.weight = nn.Parameter(
 torch.Tensor(self.n_vectors, self.out_features, self.in_features))
self.matmul_func = vector_wise_matmul
if bias:
self.bias = nn.Parameter(torch.Tensor(out_features))
else:
self.register_parameter('bias', None)
 self.reset_parameters()
def reset_parameters(self)-> None:

 kaiming_uniform_for_vwl(self.weight, a=np.sqrt(5))
if self.bias is not None:
fan_in, _ = _calculate_fan_in_and_fan_out_for_vwl(self.weight)
 bound = 1 / np.sqrt(fan_in)
 nn.init.uniform_(self.bias, -bound, bound)
 def forward(self, X: torch.Tensor)-> torch.Tensor:

 H = self.matmul_func(X, self.weight)
 if self.bias is not None:
H = H + self.bias
return H<import_modules>
train_df["masks"] = [np.array(load_img(".. /input/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
class GraphConv(nn.Module):

 def __init__(
 self,
in_channels: int, out_channels: int, n_nodes: int, shrare_msg: bool=True,
model_self: bool=True, share_model_self: bool=True,
bias: bool=True, share_bias: bool=True 
)-> None:

 self.in_channels = in_channels
self.out_channels = out_channels
self.n_nodes = n_nodes
self.model_self = model_self
super(GraphConv, self ).__init__()
self.msg = VectorWiseLinear(
 in_channels, out_channels, n_nodes, False, shrare_msg)
 if model_self:
self.model_self = VectorWiseLinear(
 in_channels, out_channels, n_nodes, False, share_model_self)
 if bias:
if share_bias:
self.bias = nn.Parameter(torch.Tensor(out_channels))
else:
self.bias = nn.Parameter(torch.Tensor(n_nodes, out_channels))
bound = 1 / np.sqrt(out_channels)
 nn.init.uniform_(self.bias, -bound, bound)
 else:
self.register_parameter('bias', None)
 def forward(self, X: torch.Tensor, A: torch.Tensor, W: torch.Tensor=None)-> torch.Tensor:

 if W is not None:
A = A * W
M = X
M = self.msg(M)
 M = torch.matmul(A, M)
 H = M
if self.model_self:
H = H + self.model_self(X)
 if self.bias is not None:
H = H + self.bias
return H<choose_model_class>
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size_ori, 2)
 def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class)
class MLP(nn.Module):

 def __init__(
 self, n_features_list: tp.List[int], use_tail_as_out: bool=False,
drop_rate: float=0.0, use_bn: bool=False, use_wn: bool=False,
activ:str="relu", block_name: str="LBAD" 
):

 super(MLP, self ).__init__()
n_layers = len(n_features_list)- 1
block_class = {
"LBAD": LBAD, "BDLA": BDLA, "LABD": LABD}[block_name]
layers = []
for i in range(n_layers):
in_feats, out_feats = n_features_list[i: i + 2]
if i == n_layers - 1 and use_tail_as_out:
if block_name in ["BDLA"]:
layer = block_class(in_feats, out_feats, drop_rate, use_bn, use_wn, "identity")
 else:
layer = nn.Linear(in_feats, out_feats)
 if use_wn:
layer = nn.utils.weight_norm(layer)
 else:
layer = block_class(in_feats, out_feats, drop_rate, use_bn, use_wn, activ)
 layers.append(layer)
 self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class CNNStacking1d(nn.Module):

 def __init__(
 self, n_models: int,
n_channels_list: tp.List[int], use_bias: bool=False,
kwargs_head: tp.Dict={}, 
):

 super(CNNStacking1d, self ).__init__()
self.n_conv_layers = len(n_channels_list)- 1
for i in range(self.n_conv_layers):
in_ch = n_channels_list[i]
out_ch = n_channels_list[i + 1]
layer = nn.Sequential(
 nn.Conv1d(
 in_ch, out_ch, kernel_size=3, stride=1, padding=0, bias=use_bias),
nn.ReLU(inplace=True))
setattr(self, "conv{}".format(i + 1), layer)
 kwargs_head["n_features_list"][0] =(n_models - 2 * self.n_conv_layers)* n_channels_list[-1]
self.head = MLP(**kwargs_head)
 def forward(self, x: torch.FloatTensor)-> torch.Tensor:

 bs = x.shape[0]
h = x
for i in range(self.n_conv_layers):
h = getattr(self, "conv{}".format(i + 1))(h)
 h = torch.reshape(h,(bs, -1))
h = self.head(h)
 return h
class CNNStacking2d(nn.Module):

 def __init__(
 self, n_models: int, n_classes: int,
n_channels_list: tp.List[int], use_bias: bool=False,
kwargs_head: tp.Dict={}, 
):

 super(CNNStacking2d, self ).__init__()
self.n_conv_layers = len(n_channels_list)- 1
for i in range(self.n_conv_layers):
in_ch = n_channels_list[i]
out_ch = n_channels_list[i + 1]
layer = nn.Sequential(
 nn.Conv2d(
 in_ch, out_ch, kernel_size=(1, 3), stride=1, padding=0, bias=use_bias),
nn.ReLU(inplace=True))
setattr(self, "conv{}".format(i + 1), layer)
 kwargs_head["n_features_list"][0] =(n_models - 2 * self.n_conv_layers)* n_classes * n_channels_list[-1]
self.head = MLP(**kwargs_head)
 def forward(self, x: torch.FloatTensor)-> torch.Tensor:

 bs = x.shape[0]
h = x
for i in range(self.n_conv_layers):
h = getattr(self, "conv{}".format(i + 1))(h)
 h = torch.reshape(h,(bs, -1))
h = self.head(h)
 return h
class GCNStacking(nn.Module):

 def __init__(
 self, n_classes: int,
n_channels_list: tp.List[int],
add_self_loop: bool=False,
kwargs_head: tp.Dict={}, 
):

 super(GCNStacking, self ).__init__()
self.n_conv_layers = len(n_channels_list)- 1
for i in range(self.n_conv_layers):
in_ch = n_channels_list[i]
out_ch = n_channels_list[i + 1]
layer = GraphConv(
 in_ch, out_ch, n_classes,
shrare_msg=False, share_model_self=False, share_bias=False)
 setattr(self, "conv{}".format(i + 1), layer)
 self.relu = nn.ReLU(inplace=True)
 if add_self_loop:
adj_mat = torch.ones(n_classes, n_classes)/ n_classes
else:
adj_mat =(1 - torch.eye(n_classes, n_classes)) /(n_classes - 1)
 self.register_buffer("A", adj_mat.float())
 kwargs_head["n_features_list"][0] = n_classes * n_channels_list[-1]
self.head = MLP(**kwargs_head)
 def forward(self, X: torch.FloatTensor)-> torch.Tensor:

 bs, n_classes = X.shape[:2]
H = X
for i in range(self.n_conv_layers):
H = getattr(self, "conv{}".format(i + 1))(H, self.A[None,...])
 H = self.relu(H)
 h = torch.reshape(H,(bs, -1))
h = self.head(h)
 return h<init_hyperparams>
ids_train, ids_valid, x_train, x_valid, y_train, y_valid, cov_train, cov_test, depth_train, depth_test = train_test_split(
 train_df.index.values,
np.array(train_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
np.array(train_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
train_df.coverage.values,
train_df.z.values,
test_size=0.2, stratify=train_df.coverage_class, random_state= 1234 )
class EvalFuncManager(nn.Module):

 def __init__(
 self, iters_per_epoch: int,
evalfunc_dict: tp.Dict[str, nn.Module], prefix: str = "val" 
)-> None:

 super(EvalFuncManager, self ).__init__()
self.tmp_iter = 0
self.iters_per_epoch = iters_per_epoch
self.prefix = prefix
self.metric_names = []
for k, v in evalfunc_dict.items() :
setattr(self, k, v)
 self.metric_names.append(k)
 self.reset()
def reset(self)-> None:

 self.tmp_iter = 0
for name in self.metric_names:
getattr(self, name ).reset()
def __call__(self, y: torch.Tensor, t: torch.Tensor)-> None:

 for name in self.metric_names:
getattr(self, name ).update(y, t)
 self.tmp_iter += 1
if self.tmp_iter == self.iters_per_epoch:
ppe.reporting.report({
"{}/{}".format(self.prefix, name): getattr(self, name ).compute()
for name in self.metric_names})
 self.reset()
class MeanLoss(nn.Module):
def __init__(self):
super(MeanLoss, self ).__init__()
self.loss_sum = 0
self.n_examples = 0
def forward(self, y: torch.Tensor, t: torch.Tensor):

 return self.loss_func(y, t)
 def reset(self):

 self.loss_sum = 0
self.n_examples = 0
def update(self, y: torch.Tensor, t: torch.Tensor):

 self.loss_sum += self(y, t ).item() * y.shape[0]
self.n_examples += y.shape[0]
def compute(self):

 return self.loss_sum / self.n_examples
class MyLogLoss(MeanLoss):
def __init__(self, **params):
super(MyLogLoss, self ).__init__()
self.loss_func = nn.BCEWithLogitsLoss(**params)
 class LSBCEWithLogitsLoss(nn.Module):

 def __init__(self, k: int, alpha: float=0.01):

 super(LSBCEWithLogitsLoss, self ).__init__()
self.k = k
self.alpha = alpha
self.loss_func = nn.BCEWithLogitsLoss()
def forward(self, y, t):

 t_s = t *(1 - self.alpha)+ self.alpha / self.k
loss = self.loss_func(y, t_s)
 return loss
class MyLSLogLoss(MeanLoss):
def __init__(self, **params):
super(MyLSLogLoss, self ).__init__()
self.loss_func = LSBCEWithLogitsLoss(**params )<train_model>
def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True):
x = Conv2D(filters, size, strides=strides, padding=padding )(x)
 x = BatchNormalization()(x)
 if activation == True:
x = Activation('relu' )(x)
 return x
def residual_block(blockInput, num_filters=16):
x = Activation('relu' )(blockInput)
 x = BatchNormalization()(x)
 x = convolution_block(x, num_filters,(3,3))
x = convolution_block(x, num_filters,(3,3), activation=False)
 x = Add()([x, blockInput])
 return x
def run_train_loop(
 manager, args, model, device,
train_loader, optimizer, scheduler, loss_func 
):

 while not manager.stop_trigger:
model.train()
for batch in train_loader:
x, t = batch
with manager.run_iteration() :
optimizer.zero_grad()
y = model(x.to(device))
loss = loss_func(y, t.to(device))
ppe.reporting.report({'train/loss': loss.item() })
 loss.backward()
optimizer.step()
if scheduler is not None:
scheduler.step()
def run_eval(args, model, device, batch, eval_manager):

 model.eval()
x, t = batch
y = model(x.to(device))
eval_manager(y, t.to(device))<choose_model_class>
def build_model(input_layer, start_neurons, DropoutRatio = 0.5):
conv1 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(input_layer)
 conv1 = residual_block(conv1,start_neurons * 1)
 conv1 = residual_block(conv1,start_neurons * 1)
 conv1 = Activation('relu' )(conv1)
 pool1 = MaxPooling2D(( 2, 2))(conv1)
 pool1 = Dropout(DropoutRatio/2 )(pool1)
 conv2 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(pool1)
 conv2 = residual_block(conv2,start_neurons * 2)
 conv2 = residual_block(conv2,start_neurons * 2)
 conv2 = Activation('relu' )(conv2)
 pool2 = MaxPooling2D(( 2, 2))(conv2)
 pool2 = Dropout(DropoutRatio )(pool2)
 conv3 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(pool2)
 conv3 = residual_block(conv3,start_neurons * 4)
 conv3 = residual_block(conv3,start_neurons * 4)
 conv3 = Activation('relu' )(conv3)
 pool3 = MaxPooling2D(( 2, 2))(conv3)
 pool3 = Dropout(DropoutRatio )(pool3)
 conv4 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(pool3)
 conv4 = residual_block(conv4,start_neurons * 8)
 conv4 = residual_block(conv4,start_neurons * 8)
 conv4 = Activation('relu' )(conv4)
 pool4 = MaxPooling2D(( 2, 2))(conv4)
 pool4 = Dropout(DropoutRatio )(pool4)
 convm = Conv2D(start_neurons * 16,(3, 3), activation=None, padding="same" )(pool4)
 convm = residual_block(convm,start_neurons * 16)
 convm = residual_block(convm,start_neurons * 16)
 convm = Activation('relu' )(convm)
 deconv4 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(convm)
 uconv4 = concatenate([deconv4, conv4])
 uconv4 = Dropout(DropoutRatio )(uconv4)
 uconv4 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(uconv4)
 uconv4 = residual_block(uconv4,start_neurons * 8)
 uconv4 = residual_block(uconv4,start_neurons * 8)
 uconv4 = Activation('relu' )(uconv4)
 deconv3 = Conv2DTranspose(start_neurons * 4,(3, 3), strides=(2, 2), padding="valid" )(uconv4)
 uconv3 = concatenate([deconv3, conv3])
 uconv3 = Dropout(DropoutRatio )(uconv3)
 uconv3 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(uconv3)
 uconv3 = residual_block(uconv3,start_neurons * 4)
 uconv3 = residual_block(uconv3,start_neurons * 4)
 uconv3 = Activation('relu' )(uconv3)
 deconv2 = Conv2DTranspose(start_neurons * 2,(3, 3), strides=(2, 2), padding="same" )(uconv3)
 uconv2 = concatenate([deconv2, conv2])
 uconv2 = Dropout(DropoutRatio )(uconv2)
 uconv2 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(uconv2)
 uconv2 = residual_block(uconv2,start_neurons * 2)
 uconv2 = residual_block(uconv2,start_neurons * 2)
 uconv2 = Activation('relu' )(uconv2)
 deconv1 = Conv2DTranspose(start_neurons * 1,(3, 3), strides=(2, 2), padding="valid" )(uconv2)
 uconv1 = concatenate([deconv1, conv1])
 uconv1 = Dropout(DropoutRatio )(uconv1)
 uconv1 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(uconv1)
 uconv1 = residual_block(uconv1,start_neurons * 1)
 uconv1 = residual_block(uconv1,start_neurons * 1)
 uconv1 = Activation('relu' )(uconv1)
 uconv1 = Dropout(DropoutRatio/2 )(uconv1)
 output_layer = Conv2D(1,(1,1), padding="same", activation="sigmoid" )(uconv1)
 return output_layer
def get_optimizer(settings, model):
optimizer = getattr(torch.optim, settings["optimizer"]["name"] )(
 model.parameters() , **settings["optimizer"]["params"])
 return optimizer
def get_scheduler(settings, train_loader, optimizer):
if settings["scheduler"]["name"] is None:
scheduler = None
else:
if settings["scheduler"]["name"] == "OneCycleLR":
settings["scheduler"]["params"]["epochs"] = settings["globals"]["max_epoch"]
settings["scheduler"]["params"]["steps_per_epoch"] = len(train_loader)
 scheduler = getattr(torch.optim.lr_scheduler, settings["scheduler"]["name"] )(
 optimizer, **settings["scheduler"]["params"])
 return scheduler
def get_loss_function(settings):
if hasattr(nn, settings["loss"]["name"]):
loss_func = getattr(nn, settings["loss"]["name"] )(**settings["loss"]["params"])
 else:
loss_func = eval(settings["loss"]["name"] )(**settings["loss"]["params"])
 return loss_func
def get_manager(
 settings, model, device, train_loader, val_loader, optimizer, eval_manager, output_path 
):
trigger = ppe.training.triggers.EarlyStoppingTrigger(
 check_trigger=(1, 'epoch'), monitor='val/metric', mode="min",
patience=settings["globals"]["patience"], verbose=True,
max_trigger=(settings["globals"]["max_epoch"], 'epoch'))
manager = ppe.training.ExtensionsManager(
 model, optimizer, settings["globals"]["max_epoch"],
iters_per_epoch=len(train_loader), stop_trigger=trigger, out_dir=output_path)
 log_extentions = [
ppe_extensions.observe_lr(optimizer=optimizer),
ppe_extensions.LogReport() ,
ppe_extensions.PlotReport(["train/loss", "val/loss"], 'epoch', filename='loss.png'),
ppe_extensions.PlotReport(["lr"], 'epoch', filename='lr.png'),
ppe_extensions.PrintReport([
"epoch", "iteration", "lr", "train/loss", "val/loss", "val/metric", "elapsed_time"])
 ]
for ext in log_extentions:
manager.extend(ext)
 manager.extend(
 ppe_extensions.Evaluator(
 val_loader, model,
eval_func=lambda *batch: run_eval(settings, model, device, batch, eval_manager)) ,
trigger=(1, "epoch"))
manager.extend(
 ppe_extensions.snapshot(target=model, filename="snapshot_epoch_{.epoch}.pth"),
trigger=ppe.training.triggers.MinValueTrigger(key="val/metric", trigger=(1, 'epoch')))
 return manager<set_options>
x_train2 = np.append(x_train, [np.fliplr(x)for x in x_train], axis=0)
 y_train2 = np.append(y_train, [np.fliplr(x)for x in y_train], axis=0)
 print(x_train2.shape)
 print(y_valid.shape )
def set_random_seed(seed: int = 42, deterministic: bool = False):

 random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 if deterministic:
torch.backends.cudnn.deterministic = True<train_model>
input_layer = Input(( img_size_target, img_size_target, 1))
output_layer = build_model(input_layer, 16,0.5)
 model = Model(input_layer, output_layer)
 model.compile(loss="binary_crossentropy", optimizer="adam", metrics=[my_iou_metric])
 model.summary()
def run_train_one_fold(
 settings, model, train_all_dataset, train_val_index, device, output_path):

 train_dataset = data.Subset(train_all_dataset, train_val_index[0])
 val_dataset = data.Subset(train_all_dataset, train_val_index[1])
 train_loader = data.DataLoader(train_dataset, **settings["loader"]["train"])
 val_loader = data.DataLoader(val_dataset, **settings["loader"]["val"])
 print("train: {}, val: {}".format(len(train_dataset), len(val_dataset)))
 model.to(device)
 optimizer = get_optimizer(settings, model)
 scheduler = get_scheduler(settings, train_loader, optimizer)
 loss_func = get_loss_function(settings)
 loss_func.to(device)
 eval_mgr = EvalFuncManager(len(val_loader), {
"loss": loss_func,
"metric": MyLogLoss() ,
})
 manager = get_manager(
 settings, model, device, train_loader, val_loader,
optimizer, eval_mgr, output_path)
 run_train_loop(
 manager, settings, model, device, train_loader, optimizer, scheduler, loss_func )<groupby>
early_stopping = EarlyStopping(monitor='val_my_iou_metric', mode = 'max',patience=20, verbose=1)
 model_checkpoint = ModelCheckpoint("./unet_best1.model",monitor='val_my_iou_metric',
mode = 'max', save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='val_my_iou_metric', mode = 'max',factor=0.2, patience=5, min_lr=0.00001, verbose=1)
 epochs = 200
batch_size = 32
history = model.fit(x_train2, y_train2,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[early_stopping, model_checkpoint, reduce_lr],
verbose=2 )
def get_drug_multilabel_index(target_in_drug_df, target_cols_list, vc_num, n_folds, seed):

 vc = target_in_drug_df.drug_id.value_counts()
vc1 = vc.loc[vc <= vc_num].index
vc2 = vc.loc[vc > vc_num].index
dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=n_folds, random_state=seed, shuffle=True)
 tmp = target_in_drug_df.groupby('drug_id')[target_cols_list].mean().loc[vc1]
if tmp.shape[0] != 0:
for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[target_cols_list])) :
dd = {k:fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=n_folds, random_state=seed, shuffle=True)
 tmp = target_in_drug_df.loc[target_in_drug_df.drug_id.isin(vc2)].reset_index(drop = True)
 if tmp.shape[0] != 0:
for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[target_cols_list])) :
dd = {k:fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 folds = []
target_in_drug_df['fold'] = target_in_drug_df.drug_id.map(dct1)
 target_in_drug_df.loc[target_in_drug_df.fold.isna() ,'fold'] = target_in_drug_df.loc[target_in_drug_df.fold.isna() ,'sig_id'].map(dct2)
 target_in_drug_df.fold = target_in_drug_df.fold.astype('int8')
 folds.append(target_in_drug_df.fold.values)
 return np.stack(folds)[0]
def drug_multilabel_stratified_kfold(k, target_in_drug_df, target_cols_list, n_folds, seed):

 folds_array = get_drug_multilabel_index(target_in_drug_df, target_cols_list, 18, n_folds, seed)
 train_idx = [i for i, x in enumerate(folds_array)if x != k]
validation_idx = [i for i, x in enumerate(folds_array)if x == k]
return train_idx, validation_idx<load_from_csv>
model = load_model("./unet_best1.model",custom_objects={'my_iou_metric': my_iou_metric})
target_in_drug = pd.merge(
 pd.read_csv(".. /input/lish-moa/train_targets_scored.csv"),
pd.read_csv(".. /input/lish-moa/train_drug.csv"),
how='left', on='sig_id')
 print(target_in_drug.shape)
 target_in_drug_trt = del_control(target_in_drug)
 print(target_in_drug_trt.shape )<concatenate>
def predict_result(model,x_test,img_size_target):
x_test_reflect = np.array([np.fliplr(x)for x in x_test])
 preds_test1 = model.predict(x_test ).reshape(-1, img_size_target, img_size_target)
 preds_test2_refect = model.predict(x_test_reflect ).reshape(-1, img_size_target, img_size_target)
 preds_test2 = np.array([ np.fliplr(x)for x in preds_test2_refect])
 preds_avg =(preds_test1 +preds_test2)/2
return preds_avg
X = np.concatenate([df.values for df in oof_list], axis=1)
 print(X.shape)
 y = y_true.values
print(y.shape)
 X_2d = np.stack([df.values for df in oof_list], axis=2)
 print(X_2d.shape)
 X_3d = X_2d[:, None,...]
print(X_3d.shape)
 X_node = np.stack([df.values for df in oof_list], axis=2 )<create_dataframe>
preds_valid = predict_result(model,x_valid,img_size_target)
 preds_valid2 = np.array([downsample(x)for x in preds_valid])
 y_valid2 = np.array([downsample(x)for x in y_valid] )
train_all_dataset = MoAStackingDatasetForCNN(X_2d, y)
<define_variables>
thresholds = np.linspace(0.3, 0.7, 31)
 ious = np.array([iou_metric_batch(y_valid2, np.int32(preds_valid2 > threshold)) for threshold in tqdm_notebook(thresholds)])
TRAIN_OUTPUT = Path.cwd().parent / "tmp"
TRAIN_OUTPUT.mkdir(exist_ok=True)
 WORK_DIR = Path.cwd()<init_hyperparams>

 def rle_encode(im):

 pixels = im.flatten(order = 'F')
 pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
settings_str =
settings = yaml.safe_load(settings_str )<set_options>
preds_test = predict_result(model,x_test,img_size_target )
gc.collect()<feature_engineering>
t1 = time.time()
pred_dict = {idx: rle_encode(np.round(downsample(preds_test[i])> threshold_best)) for i, idx in enumerate(tqdm_notebook(test_df.index.values)) }
t2 = time.time()
print(f"Usedtime = {t2-t1} s" )
device = torch.device(settings["globals"]["device"])
 cv_score_df_list = []
if settings["globals"]["fast_commit"]:
settings["globals"]["max_epoch"] = 1
stgs_list = []
for tmp_seed in settings["globals"]["seeds_for_avg"]:
stgs_list.append(deepcopy(settings))
stgs_list[-1]["globals"]["seed"] = tmp_seed<categorify>
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv('submission4.csv' )