Kaggle competition: TGS Salt Identification Challenge
<SOS> metric: IntersectionOverUnionObjectSegmentation Kaggle data source: tgs-salt-identification-challenge<install_modules>
plt.style.use('seaborn-white')
 sns.set_style("white")
 %matplotlib inline
t_start = time.time()
%%time
!pip install.. /input/iterative-stratification/iterative-stratification-master/<set_options>
version = 5
basic_name = f'Unet_resnet_v{version}'
save_model_name = basic_name + '.model'
submission_file = basic_name + '.csv'
print(save_model_name)
 print(submission_file )
warnings.resetwarnings()
warnings.simplefilter('ignore', FutureWarning)
 warnings.simplefilter('ignore', DeprecationWarning)
 os.environ["CUDA_VISIBLE_DEVICES"]="0"<define_variables>
img_size_ori = 101
img_size_target = 101
def upsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_target, img_size_target), mode='constant', preserve_range=True)
 def downsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_ori, img_size_ori), mode='constant', preserve_range=True )
def generate_virtual_private(test_feat, smpl_sub, rate_to_pub=4):
assert(test_feat.sig_id == smpl_sub.sig_id ).all()
test_feat_list = [test_feat]
smpl_sub_list = [smpl_sub]
for i in range(1, rate_to_pub):
tmp_test_feat = test_feat.copy()
tmp_smpl_sub = smpl_sub.copy()
tmp_test_feat.sig_id = tmp_test_feat.sig_id + f"_{i}"
tmp_smpl_sub.sig_id = tmp_smpl_sub.sig_id + f"_{i}"
assert(tmp_test_feat.sig_id == tmp_smpl_sub.sig_id ).all()
tmp_test_feat.iloc[:, 4:] += i * 10
assert(tmp_test_feat.iloc[:, 4:] != test_feat.iloc[:, 4:] ).all().all()
test_feat_list.append(tmp_test_feat)
 smpl_sub_list.append(tmp_smpl_sub)
 test_feat_concat = pd.concat(test_feat_list, axis=0, ignore_index=True)
 smpl_sub_concat = pd.concat(smpl_sub_list, axis=0, ignore_index=True)
 return test_feat_concat, smpl_sub_concat<define_variables>
train_df = pd.read_csv(".. /input/train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(".. /input/depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]
len(train_df )
TARGET_COL = ['5-alpha_reductase_inhibitor', '11-beta-hsd1_inhibitor', 'acat_inhibitor', 'acetylcholine_receptor_agonist', 'acetylcholine_receptor_antagonist', 'acetylcholinesterase_inhibitor', 'adenosine_receptor_agonist', 'adenosine_receptor_antagonist', 'adenylyl_cyclase_activator', 'adrenergic_receptor_agonist', 'adrenergic_receptor_antagonist', 'akt_inhibitor', 'aldehyde_dehydrogenase_inhibitor', 'alk_inhibitor', 'ampk_activator', 'analgesic', 'androgen_receptor_agonist', 'androgen_receptor_antagonist', 'anesthetic_-_local', 'angiogenesis_inhibitor', 'angiotensin_receptor_antagonist', 'anti-inflammatory', 'antiarrhythmic', 'antibiotic', 'anticonvulsant', 'antifungal', 'antihistamine', 'antimalarial', 'antioxidant', 'antiprotozoal', 'antiviral', 'apoptosis_stimulant', 'aromatase_inhibitor', 'atm_kinase_inhibitor', 'atp-sensitive_potassium_channel_antagonist', 'atp_synthase_inhibitor', 'atpase_inhibitor', 'atr_kinase_inhibitor', 'aurora_kinase_inhibitor', 'autotaxin_inhibitor', 'bacterial_30s_ribosomal_subunit_inhibitor', 'bacterial_50s_ribosomal_subunit_inhibitor', 'bacterial_antifolate', 'bacterial_cell_wall_synthesis_inhibitor', 'bacterial_dna_gyrase_inhibitor', 'bacterial_dna_inhibitor', 'bacterial_membrane_integrity_inhibitor', 'bcl_inhibitor', 'bcr-abl_inhibitor', 'benzodiazepine_receptor_agonist', 'beta_amyloid_inhibitor', 'bromodomain_inhibitor', 'btk_inhibitor', 'calcineurin_inhibitor', 'calcium_channel_blocker', 'cannabinoid_receptor_agonist', 'cannabinoid_receptor_antagonist', 'carbonic_anhydrase_inhibitor', 'casein_kinase_inhibitor', 'caspase_activator', 'catechol_o_methyltransferase_inhibitor', 'cc_chemokine_receptor_antagonist', 'cck_receptor_antagonist', 'cdk_inhibitor', 'chelating_agent', 'chk_inhibitor', 'chloride_channel_blocker', 'cholesterol_inhibitor', 'cholinergic_receptor_antagonist', 'coagulation_factor_inhibitor', 'corticosteroid_agonist', 'cyclooxygenase_inhibitor', 'cytochrome_p450_inhibitor', 'dihydrofolate_reductase_inhibitor', 'dipeptidyl_peptidase_inhibitor', 'diuretic', 'dna_alkylating_agent', 'dna_inhibitor', 'dopamine_receptor_agonist', 'dopamine_receptor_antagonist', 'egfr_inhibitor', 'elastase_inhibitor', 'erbb2_inhibitor', 'estrogen_receptor_agonist', 'estrogen_receptor_antagonist', 'faah_inhibitor', 'farnesyltransferase_inhibitor', 'fatty_acid_receptor_agonist', 'fgfr_inhibitor', 'flt3_inhibitor', 'focal_adhesion_kinase_inhibitor', 'free_radical_scavenger', 'fungal_squalene_epoxidase_inhibitor', 'gaba_receptor_agonist', 'gaba_receptor_antagonist', 'gamma_secretase_inhibitor', 'glucocorticoid_receptor_agonist', 'glutamate_inhibitor', 'glutamate_receptor_agonist', 'glutamate_receptor_antagonist', 'gonadotropin_receptor_agonist', 'gsk_inhibitor', 'hcv_inhibitor', 'hdac_inhibitor', 'histamine_receptor_agonist', 'histamine_receptor_antagonist', 'histone_lysine_demethylase_inhibitor', 'histone_lysine_methyltransferase_inhibitor', 'hiv_inhibitor', 'hmgcr_inhibitor', 'hsp_inhibitor', 'igf-1_inhibitor', 'ikk_inhibitor', 'imidazoline_receptor_agonist', 'immunosuppressant', 'insulin_secretagogue', 'insulin_sensitizer', 'integrin_inhibitor', 'jak_inhibitor', 'kit_inhibitor', 'laxative', 'leukotriene_inhibitor', 'leukotriene_receptor_antagonist', 'lipase_inhibitor', 'lipoxygenase_inhibitor', 'lxr_agonist', 'mdm_inhibitor', 'mek_inhibitor', 'membrane_integrity_inhibitor', 'mineralocorticoid_receptor_antagonist', 'monoacylglycerol_lipase_inhibitor', 'monoamine_oxidase_inhibitor', 'monopolar_spindle_1_kinase_inhibitor', 'mtor_inhibitor', 'mucolytic_agent', 'neuropeptide_receptor_antagonist', 'nfkb_inhibitor', 'nicotinic_receptor_agonist', 'nitric_oxide_donor', 'nitric_oxide_production_inhibitor', 'nitric_oxide_synthase_inhibitor', 'norepinephrine_reuptake_inhibitor', 'nrf2_activator', 'opioid_receptor_agonist', 'opioid_receptor_antagonist', 'orexin_receptor_antagonist', 'p38_mapk_inhibitor', 'p-glycoprotein_inhibitor', 'parp_inhibitor', 'pdgfr_inhibitor', 'pdk_inhibitor', 'phosphodiesterase_inhibitor', 'phospholipase_inhibitor', 'pi3k_inhibitor', 'pkc_inhibitor', 'potassium_channel_activator', 'potassium_channel_antagonist', 'ppar_receptor_agonist', 'ppar_receptor_antagonist', 'progesterone_receptor_agonist', 'progesterone_receptor_antagonist', 'prostaglandin_inhibitor', 'prostanoid_receptor_antagonist', 'proteasome_inhibitor', 'protein_kinase_inhibitor', 'protein_phosphatase_inhibitor', 'protein_synthesis_inhibitor', 'protein_tyrosine_kinase_inhibitor', 'radiopaque_medium', 'raf_inhibitor', 'ras_gtpase_inhibitor', 'retinoid_receptor_agonist', 'retinoid_receptor_antagonist', 'rho_associated_kinase_inhibitor', 'ribonucleoside_reductase_inhibitor', 'rna_polymerase_inhibitor', 'serotonin_receptor_agonist', 'serotonin_receptor_antagonist', 'serotonin_reuptake_inhibitor', 'sigma_receptor_agonist', 'sigma_receptor_antagonist', 'smoothened_receptor_antagonist', 'sodium_channel_inhibitor', 'sphingosine_receptor_agonist', 'src_inhibitor', 'steroid', 'syk_inhibitor', 'tachykinin_antagonist', 'tgf-beta_receptor_inhibitor', 'thrombin_inhibitor', 'thymidylate_synthase_inhibitor', 'tlr_agonist', 'tlr_antagonist', 'tnf_inhibitor', 'topoisomerase_inhibitor', 'transient_receptor_potential_channel_antagonist', 'tropomyosin_receptor_kinase_inhibitor', 'trpv_agonist', 'trpv_antagonist', 'tubulin_inhibitor', 'tyrosine_kinase_inhibitor', 'ubiquitin_specific_protease_inhibitor', 'vegfr_inhibitor', 'vitamin_b', 'vitamin_d_receptor_agonist', 'wnt_inhibitor']<define_variables>
train_df["images"] = [np.array(load_img(".. /input/train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
MODEL_NAMES = ["NN(drugCV)", "TabNet", "ResNet", "ThrNN", "ThrNN(drugCV)"]<load_pretrained>
train_df["masks"] = [np.array(load_img(".. /input/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
%%time
if DO_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/nn-inference-0.01833.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/virtual-nn-use-train-public-inference.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError<load_pretrained>
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size_ori, 2)
 def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class)
%%time
if DO_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/tabnet-inference-0.01840.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/virtual-tabnet-inference-add-param-n-shared.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError<install_modules>
ids_train, ids_valid, x_train, x_valid, y_train, y_valid, cov_train, cov_test, depth_train, depth_test = train_test_split(
 train_df.index.values,
np.array(train_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
np.array(train_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
train_df.coverage.values,
train_df.z.values,
test_size=0.2, stratify=train_df.coverage_class, random_state= 1234 )
%%time
if DO_SUBMISSION:
!python.. /input/moa-takapy-script/tf-rn-transfer-1layerother-selcol100.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-takapy-script/virtual-tf-rn-transfer-1layerother-selcol100.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError<install_modules>
def BatchActivate(x):
x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 return x
def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True):
x = Conv2D(filters, size, strides=strides, padding=padding )(x)
 if activation == True:
x = BatchActivate(x)
 return x
def residual_block(blockInput, num_filters=16, batch_activate = False):
x = BatchActivate(blockInput)
 x = convolution_block(x, num_filters,(3,3))
x = convolution_block(x, num_filters,(3,3), activation=False)
 x = Add()([x, blockInput])
 if batch_activate:
x = BatchActivate(x)
 return x
%%time
if DO_SUBMISSION:
!python.. /input/moa-tawara-scripts-for-final-submission/moa-for-final-thrnn-seed-cv-0.01836.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-tawara-scripts-for-final-submission/virtual-moa-for-final-thrnn-seed-cv-0.01836.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError<load_from_csv>
def build_model(input_layer, start_neurons, DropoutRatio = 0.4):
conv1 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(input_layer)
 conv1 = residual_block(conv1,start_neurons * 1)
 conv1 = residual_block(conv1,start_neurons * 1, True)
 pool1 = MaxPooling2D(( 2, 2))(conv1)
 pool1 = Dropout(DropoutRatio/2 )(pool1)
 conv2 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(pool1)
 conv2 = residual_block(conv2,start_neurons * 2)
 conv2 = residual_block(conv2,start_neurons * 2, True)
 pool2 = MaxPooling2D(( 2, 2))(conv2)
 pool2 = Dropout(DropoutRatio )(pool2)
 conv3 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(pool2)
 conv3 = residual_block(conv3,start_neurons * 4)
 conv3 = residual_block(conv3,start_neurons * 4, True)
 pool3 = MaxPooling2D(( 2, 2))(conv3)
 pool3 = Dropout(DropoutRatio )(pool3)
 conv4 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(pool3)
 conv4 = residual_block(conv4,start_neurons * 8)
 conv4 = residual_block(conv4,start_neurons * 8, True)
 pool4 = MaxPooling2D(( 2, 2))(conv4)
 pool4 = Dropout(DropoutRatio )(pool4)
 convm = Conv2D(start_neurons * 16,(3, 3), activation=None, padding="same" )(pool4)
 convm = residual_block(convm,start_neurons * 16)
 convm = residual_block(convm,start_neurons * 16, True)
 deconv4 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(convm)
 uconv4 = concatenate([deconv4, conv4])
 uconv4 = Dropout(DropoutRatio )(uconv4)
 uconv4 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(uconv4)
 uconv4 = residual_block(uconv4,start_neurons * 8)
 uconv4 = residual_block(uconv4,start_neurons * 8, True)
 deconv3 = Conv2DTranspose(start_neurons * 4,(3, 3), strides=(2, 2), padding="valid" )(uconv4)
 uconv3 = concatenate([deconv3, conv3])
 uconv3 = Dropout(DropoutRatio )(uconv3)
 uconv3 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(uconv3)
 uconv3 = residual_block(uconv3,start_neurons * 4)
 uconv3 = residual_block(uconv3,start_neurons * 4, True)
 deconv2 = Conv2DTranspose(start_neurons * 2,(3, 3), strides=(2, 2), padding="same" )(uconv3)
 uconv2 = concatenate([deconv2, conv2])
 uconv2 = Dropout(DropoutRatio )(uconv2)
 uconv2 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(uconv2)
 uconv2 = residual_block(uconv2,start_neurons * 2)
 uconv2 = residual_block(uconv2,start_neurons * 2, True)
 deconv1 = Conv2DTranspose(start_neurons * 1,(3, 3), strides=(2, 2), padding="valid" )(uconv2)
 uconv1 = concatenate([deconv1, conv1])
 uconv1 = Dropout(DropoutRatio )(uconv1)
 uconv1 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(uconv1)
 uconv1 = residual_block(uconv1,start_neurons * 1)
 uconv1 = residual_block(uconv1,start_neurons * 1, True)
 output_layer_noActi = Conv2D(1,(1,1), padding="same", activation=None )(uconv1)
 output_layer = Activation('sigmoid' )(output_layer_noActi)
 return output_layer
def order_sub(sub):
return sub.sort_values('sig_id' ).reset_index(drop=True)
 if DO_SUBMISSION or DO_VIRTUAL_SUBMISSION:
sub_list = [
pd.read_csv('./submission-sinchir0-nn.csv'),
pd.read_csv('./submission_sinchir0_tabnet.csv'),
pd.read_csv("./submission_takapy_tf-resnet.csv"),
pd.read_csv('./submission_tawara_thrnn_seed_cv.csv'),
pd.read_csv('./submission_tawara_thrnn_drug_seed_cv.csv'),
]
else:
sub_list = [
pd.read_csv('.. /input/nn-use-train-public/submission.csv'),
pd.read_csv('.. /input/tabnet-train-public-add-n-shared-1/submission.csv'),
pd.read_csv(".. /input/moa-takapy-tf-resnet-transfer/submission.csv"),
pd.read_csv('.. /input/moa-weight-thrnn-seed-cv/submission.csv'),
pd.read_csv('.. /input/moa-weight-thrnn-drug-seed-cv/submission.csv'),
]
for i, name in enumerate(["NN(drugCV)", "TabNet", "ResNet", "ThrNN", "ThrNN(drugCV)"]):
print(f"[{name}]:", sub_list[i].shape)
 sub_list = [order_sub(sub_df)for sub_df in sub_list]<categorify>
def get_iou_vector(A, B):
batch_size = A.shape[0]
metric = []
for batch in range(batch_size):
t, p = A[batch]>0, B[batch]>0
intersection = np.logical_and(t, p)
 union = np.logical_or(t, p)
 iou =(np.sum(intersection > 0)+ 1e-10)/(np.sum(union > 0)+ 1e-10)
 thresholds = np.arange(0.5, 1, 0.05)
 s = []
for thresh in thresholds:
s.append(iou > thresh)
 metric.append(np.mean(s))
return np.mean(metric)
 def my_iou_metric(label, pred):
return tf.py_func(get_iou_vector, [label, pred>0.5], tf.float64)
 def my_iou_metric_2(label, pred):
return tf.py_func(get_iou_vector, [label, pred >0], tf.float64 )
class MoAStackingDataset(data.Dataset):
def __init__(self, feat: np.ndarray, label: np.ndarray = None):

 self.feat = feat
if label is None:
self.label = np.full(( len(feat), 1), -1)
 else:
self.label = label
self.model_order = None
def __len__(self):

 return len(self.feat)
 def __getitem__(self, index: int):

 return [
torch.from_numpy(self.feat[index] ).float() ,
torch.from_numpy(self.label[index] ).float()
]
def reset_model_order(self):

 pass
def shuffle_model_order(self, seed):

 pass
class MoAStackingDatasetForCNN(data.Dataset):
def __init__(self, feat: np.ndarray, label: np.ndarray = None):

 self.feat = feat
if label is None:
self.label = np.full(( len(feat), 1), -1)
 else:
self.label = label
self.reset_model_order()
def reset_model_order(self):
self.model_order = np.arange(self.feat.shape[-1])
 def shuffle_model_order(self, seed):
np.random.seed(seed)
 self.model_order = np.random.permutation(self.model_order)
 def __len__(self):

 return len(self.feat)
 def __getitem__(self, index: int):

 return [
torch.from_numpy(self.feat[index][..., self.model_order] ).float() ,
torch.from_numpy(self.label[index] ).float()
]
class MoAStackingDatasetForGCN(data.Dataset):
def __init__(self, feat: np.ndarray, label: np.ndarray = None):

 self.feat = feat
if label is None:
self.label = np.full(( len(feat), 1), -1)
 else:
self.label = label
self.model_order = None
def reset_model_order(self):

 pass
def shuffle_model_order(self, seed):

 pass
def __len__(self):

 return len(self.feat)
 def __getitem__(self, index: int):

 return [
torch.from_numpy(self.feat[index] ).float() ,
torch.from_numpy(self.label[index] ).float()
]<define_search_model>
def lovasz_grad(gt_sorted):

 gts = tf.reduce_sum(gt_sorted)
 intersection = gts - tf.cumsum(gt_sorted)
 union = gts + tf.cumsum(1.- gt_sorted)
 jaccard = 1.- intersection / union
jaccard = tf.concat(( jaccard[0:1], jaccard[1:] - jaccard[:-1]), 0)
 return jaccard
def lovasz_hinge(logits, labels, per_image=True, ignore=None):

 if per_image:
def treat_image(log_lab):
log, lab = log_lab
log, lab = tf.expand_dims(log, 0), tf.expand_dims(lab, 0)
 log, lab = flatten_binary_scores(log, lab, ignore)
 return lovasz_hinge_flat(log, lab)
 losses = tf.map_fn(treat_image,(logits, labels), dtype=tf.float32)
 loss = tf.reduce_mean(losses)
 else:
loss = lovasz_hinge_flat(*flatten_binary_scores(logits, labels, ignore))
return loss
def lovasz_hinge_flat(logits, labels):

 def compute_loss() :
labelsf = tf.cast(labels, logits.dtype)
 signs = 2.* labelsf - 1.
 errors = 1.- logits * tf.stop_gradient(signs)
 errors_sorted, perm = tf.nn.top_k(errors, k=tf.shape(errors)[0], name="descending_sort")
 gt_sorted = tf.gather(labelsf, perm)
 grad = lovasz_grad(gt_sorted)
 loss = tf.tensordot(tf.nn.relu(errors_sorted), tf.stop_gradient(grad), 1, name="loss_non_void")
 return loss
loss = tf.cond(tf.equal(tf.shape(logits)[0], 0),
lambda: tf.reduce_sum(logits)* 0.,
compute_loss,
strict=True,
name="loss" 
)
 return loss
def flatten_binary_scores(scores, labels, ignore=None):

 scores = tf.reshape(scores,(-1,))
labels = tf.reshape(labels,(-1,))
if ignore is None:
return scores, labels
valid = tf.not_equal(labels, ignore)
 vscores = tf.boolean_mask(scores, valid, name='valid_scores')
 vlabels = tf.boolean_mask(labels, valid, name='valid_labels')
 return vscores, vlabels
def lovasz_loss(y_true, y_pred):
y_true, y_pred = K.cast(K.squeeze(y_true, -1), 'int32'), K.cast(K.squeeze(y_pred, -1), 'float32')
 logits = y_pred
loss = lovasz_hinge(logits, y_true, per_image = True, ignore = None)
 return loss
def get_activation(activ_name: str="relu"):

 act_dict = {
"relu": nn.ReLU() ,
"tanh": nn.Tanh() ,
"sigmoid": nn.Sigmoid() ,
"identity": nn.Identity() }
if activ_name in act_dict:
return act_dict[activ_name]
elif re.match(r"^htanh\_\d{4}$", activ_name):
bound = int(activ_name[-4:])/ 1000
return nn.Hardtanh(-bound, bound)
 else:
raise NotImplementedError
class LBAD(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(LBAD, self ).__init__()
layers = [nn.Linear(in_features, out_features)]
if use_wn:
layers[0] = nn.utils.weight_norm(layers[0])
 if use_bn:
layers.append(nn.BatchNorm1d(out_features))
layers.append(get_activation(activ))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class BDLA(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(BDLA, self ).__init__()
layers = []
if use_bn:
layers.append(nn.BatchNorm1d(in_features))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
layers.append(nn.Linear(in_features, out_features))
if use_wn:
layers[-1] = nn.utils.weight_norm(layers[-1])
 layers.append(get_activation(activ))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class LABD(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(LABD, self ).__init__()
layers = [nn.Linear(in_features, out_features), get_activation(activ)]
if use_wn:
layers[0] = nn.utils.weight_norm(layers[0])
 if use_bn:
layers.append(nn.BatchNorm1d(out_features))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x )<prepare_x_and_y>
x_train = np.append(x_train, [np.fliplr(x)for x in x_train], axis=0)
 y_train = np.append(y_train, [np.fliplr(x)for x in y_train], axis=0)
 print(x_train.shape)
 print(y_valid.shape )
def vector_wise_matmul(X: torch.Tensor, W: torch.Tensor)-> torch.Tensor:

 X = torch.transpose(X, 0, 1)
 W = torch.transpose(W, 1, 2)
 H = torch.matmul(X, W)
 H = torch.transpose(H, 0, 1)
 return H
def vector_wise_shared_matmul(X: torch.Tensor, W: torch.Tensor)-> torch.Tensor:

 H = nn.functional.linear(X, W)
 return H<normalization>
input_layer = Input(( img_size_target, img_size_target, 1))
output_layer = build_model(input_layer, 16,0.5)
 model1 = Model(input_layer, output_layer)
 c = optimizers.adam(lr = 0.01)
 model1.compile(loss="binary_crossentropy", optimizer=c, metrics=[my_iou_metric])
def _calculate_fan_in_and_fan_out_for_vwl(tensor)-> tp.Tuple[int]:

 dimensions = tensor.dim()
if dimensions < 2:
raise ValueError("Fan in and fan out can not be computed for tensor with fewer than 2 dimensions")
 fan_in = tensor.size(-1)
 fan_out = tensor.size(-2)
 return fan_in, fan_out
def _calculate_correct_fan_for_vwl(tensor, mode)-> int:

 mode = mode.lower()
valid_modes = ['fan_in', 'fan_out']
if mode not in valid_modes:
raise ValueError("Mode {} not supported, please use one of {}".format(mode, valid_modes))
fan_in, fan_out = _calculate_fan_in_and_fan_out_for_vwl(tensor)
 return fan_in if mode == 'fan_in' else fan_out
def kaiming_uniform_for_vwl(tensor, a=0, mode='fan_in', nonlinearity='leaky_relu'):

 fan = _calculate_correct_fan_for_vwl(tensor, mode)
 gain = nn.init.calculate_gain(nonlinearity, a)
 std = gain / np.sqrt(fan)
 bound = np.sqrt(3.0)* std
with torch.no_grad() :
return tensor.uniform_(-bound, bound )<prepare_x_and_y>
model_checkpoint = ModelCheckpoint(save_model_name,monitor='my_iou_metric',
mode = 'max', save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='my_iou_metric', mode = 'max',factor=0.6, patience=5, min_lr=0.0001, verbose=1)
 epochs = 50
batch_size = 32
history = model1.fit(x_train, y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[ model_checkpoint,reduce_lr],
verbose=2 )
class VectorWiseLinear(nn.Module):

 def __init__(
 self,
in_features: int, out_features: int, n_vectors: int,
bias: bool=True, weight_shared: bool=True 
)-> None:

 super(VectorWiseLinear, self ).__init__()
self.in_features = in_features
self.out_features = out_features
self.n_vectors = n_vectors
self.weight_shared = weight_shared
if self.weight_shared:
self.weight = nn.Parameter(
 torch.Tensor(self.out_features, self.in_features))
self.matmul_func = vector_wise_shared_matmul
else:
self.weight = nn.Parameter(
 torch.Tensor(self.n_vectors, self.out_features, self.in_features))
self.matmul_func = vector_wise_matmul
if bias:
self.bias = nn.Parameter(torch.Tensor(out_features))
else:
self.register_parameter('bias', None)
 self.reset_parameters()
def reset_parameters(self)-> None:

 kaiming_uniform_for_vwl(self.weight, a=np.sqrt(5))
if self.bias is not None:
fan_in, _ = _calculate_fan_in_and_fan_out_for_vwl(self.weight)
 bound = 1 / np.sqrt(fan_in)
 nn.init.uniform_(self.bias, -bound, bound)
 def forward(self, X: torch.Tensor)-> torch.Tensor:

 H = self.matmul_func(X, self.weight)
 if self.bias is not None:
H = H + self.bias
return H<import_modules>
model1 = load_model(save_model_name,custom_objects={'my_iou_metric': my_iou_metric})
 input_x = model1.layers[0].input
output_layer = model1.layers[-1].input
model = Model(input_x, output_layer)
 c = optimizers.adam(lr = 0.01)
 model.compile(loss=lovasz_loss, optimizer=c, metrics=[my_iou_metric_2])
class GraphConv(nn.Module):

 def __init__(
 self,
in_channels: int, out_channels: int, n_nodes: int, shrare_msg: bool=True,
model_self: bool=True, share_model_self: bool=True,
bias: bool=True, share_bias: bool=True 
)-> None:

 self.in_channels = in_channels
self.out_channels = out_channels
self.n_nodes = n_nodes
self.model_self = model_self
super(GraphConv, self ).__init__()
self.msg = VectorWiseLinear(
 in_channels, out_channels, n_nodes, False, shrare_msg)
 if model_self:
self.model_self = VectorWiseLinear(
 in_channels, out_channels, n_nodes, False, share_model_self)
 if bias:
if share_bias:
self.bias = nn.Parameter(torch.Tensor(out_channels))
else:
self.bias = nn.Parameter(torch.Tensor(n_nodes, out_channels))
bound = 1 / np.sqrt(out_channels)
 nn.init.uniform_(self.bias, -bound, bound)
 else:
self.register_parameter('bias', None)
 def forward(self, X: torch.Tensor, A: torch.Tensor, W: torch.Tensor=None)-> torch.Tensor:

 if W is not None:
A = A * W
M = X
M = self.msg(M)
 M = torch.matmul(A, M)
 H = M
if self.model_self:
H = H + self.model_self(X)
 if self.bias is not None:
H = H + self.bias
return H<choose_model_class>
early_stopping = EarlyStopping(monitor='val_my_iou_metric_2', mode = 'max',patience=20, verbose=1)
 model_checkpoint = ModelCheckpoint(save_model_name,monitor='val_my_iou_metric_2',
mode = 'max', save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='val_my_iou_metric_2', mode = 'max',factor=0.6, patience=5, min_lr=0.0001, verbose=1)
 epochs = 50
batch_size = 32
history = model.fit(x_train, y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[ model_checkpoint,reduce_lr,early_stopping],
verbose=2 )
class MLP(nn.Module):

 def __init__(
 self, n_features_list: tp.List[int], use_tail_as_out: bool=False,
drop_rate: float=0.0, use_bn: bool=False, use_wn: bool=False,
activ:str="relu", block_name: str="LBAD" 
):

 super(MLP, self ).__init__()
n_layers = len(n_features_list)- 1
block_class = {
"LBAD": LBAD, "BDLA": BDLA, "LABD": LABD}[block_name]
layers = []
for i in range(n_layers):
in_feats, out_feats = n_features_list[i: i + 2]
if i == n_layers - 1 and use_tail_as_out:
if block_name in ["BDLA"]:
layer = block_class(in_feats, out_feats, drop_rate, use_bn, use_wn, "identity")
 else:
layer = nn.Linear(in_feats, out_feats)
 if use_wn:
layer = nn.utils.weight_norm(layer)
 else:
layer = block_class(in_feats, out_feats, drop_rate, use_bn, use_wn, activ)
 layers.append(layer)
 self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class CNNStacking1d(nn.Module):

 def __init__(
 self, n_models: int,
n_channels_list: tp.List[int], use_bias: bool=False,
kwargs_head: tp.Dict={}, 
):

 super(CNNStacking1d, self ).__init__()
self.n_conv_layers = len(n_channels_list)- 1
for i in range(self.n_conv_layers):
in_ch = n_channels_list[i]
out_ch = n_channels_list[i + 1]
layer = nn.Sequential(
 nn.Conv1d(
 in_ch, out_ch, kernel_size=3, stride=1, padding=0, bias=use_bias),
nn.ReLU(inplace=True))
setattr(self, "conv{}".format(i + 1), layer)
 kwargs_head["n_features_list"][0] =(n_models - 2 * self.n_conv_layers)* n_channels_list[-1]
self.head = MLP(**kwargs_head)
 def forward(self, x: torch.FloatTensor)-> torch.Tensor:

 bs = x.shape[0]
h = x
for i in range(self.n_conv_layers):
h = getattr(self, "conv{}".format(i + 1))(h)
 h = torch.reshape(h,(bs, -1))
h = self.head(h)
 return h
class CNNStacking2d(nn.Module):

 def __init__(
 self, n_models: int, n_classes: int,
n_channels_list: tp.List[int], use_bias: bool=False,
kwargs_head: tp.Dict={}, 
):

 super(CNNStacking2d, self ).__init__()
self.n_conv_layers = len(n_channels_list)- 1
for i in range(self.n_conv_layers):
in_ch = n_channels_list[i]
out_ch = n_channels_list[i + 1]
layer = nn.Sequential(
 nn.Conv2d(
 in_ch, out_ch, kernel_size=(1, 3), stride=1, padding=0, bias=use_bias),
nn.ReLU(inplace=True))
setattr(self, "conv{}".format(i + 1), layer)
 kwargs_head["n_features_list"][0] =(n_models - 2 * self.n_conv_layers)* n_classes * n_channels_list[-1]
self.head = MLP(**kwargs_head)
 def forward(self, x: torch.FloatTensor)-> torch.Tensor:

 bs = x.shape[0]
h = x
for i in range(self.n_conv_layers):
h = getattr(self, "conv{}".format(i + 1))(h)
 h = torch.reshape(h,(bs, -1))
h = self.head(h)
 return h
class GCNStacking(nn.Module):

 def __init__(
 self, n_classes: int,
n_channels_list: tp.List[int],
add_self_loop: bool=False,
kwargs_head: tp.Dict={}, 
):

 super(GCNStacking, self ).__init__()
self.n_conv_layers = len(n_channels_list)- 1
for i in range(self.n_conv_layers):
in_ch = n_channels_list[i]
out_ch = n_channels_list[i + 1]
layer = GraphConv(
 in_ch, out_ch, n_classes,
shrare_msg=False, share_model_self=False, share_bias=False)
 setattr(self, "conv{}".format(i + 1), layer)
 self.relu = nn.ReLU(inplace=True)
 if add_self_loop:
adj_mat = torch.ones(n_classes, n_classes)/ n_classes
else:
adj_mat =(1 - torch.eye(n_classes, n_classes)) /(n_classes - 1)
 self.register_buffer("A", adj_mat.float())
 kwargs_head["n_features_list"][0] = n_classes * n_channels_list[-1]
self.head = MLP(**kwargs_head)
 def forward(self, X: torch.FloatTensor)-> torch.Tensor:

 bs, n_classes = X.shape[:2]
H = X
for i in range(self.n_conv_layers):
H = getattr(self, "conv{}".format(i + 1))(H, self.A[None,...])
 H = self.relu(H)
 h = torch.reshape(H,(bs, -1))
h = self.head(h)
 return h<set_options>
model = load_model(save_model_name,custom_objects={'my_iou_metric_2': my_iou_metric_2,
'lovasz_loss': lovasz_loss})
def set_random_seed(seed: int = 42, deterministic: bool = False):

 random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 if deterministic:
torch.backends.cudnn.deterministic = True<predict_on_test>
def predict_result(model,x_test,img_size_target):
x_test_reflect = np.array([np.fliplr(x)for x in x_test])
 preds_test = model.predict(x_test ).reshape(-1, img_size_target, img_size_target)
 preds_test2_refect = model.predict(x_test_reflect ).reshape(-1, img_size_target, img_size_target)
 preds_test += np.array([ np.fliplr(x)for x in preds_test2_refect])
 return preds_test/2
def inference_function(settings, model, loader, device):
model.to(device)
 model.eval()
pred_list = []
with torch.no_grad() :
for x, t in loader:
y = model(x.to(device))
pred_list.append(y.sigmoid().detach().cpu().numpy())
 pred_arr = np.concatenate(pred_list)
 del pred_list
return pred_arr<feature_engineering>
preds_valid = predict_result(model,x_valid,img_size_target )
def inference_test(stgs, model_dir, test_dataset, target_col_list):
device = torch.device(stgs["globals"]["device"])
 stgs_list = []
for tmp_seed in stgs["globals"]["seeds_for_avg"]:
stgs_list.append(deepcopy(stgs))
stgs_list[-1]["globals"]["seed"] = tmp_seed
test_pred_arr_avg = np.zeros(( len(test_dataset), len(target_col_list)))
 for m_id, tmp_stgs in enumerate(stgs_list):
n_folds = tmp_stgs["globals"]["n_folds_split"]
tmp_seed = tmp_stgs["globals"]["seed"]
test_preds_arr = np.zeros(( n_folds, len(test_dataset), len(target_col_list)))
 test_dataset.reset_model_order()
test_dataset.shuffle_model_order(tmp_seed)
 print("[model {}'s order]: {}".format(m_id, test_dataset.model_order))
for fold_id in range(n_folds):
print("[fold: {} - model: {}]".format(fold_id, m_id))
model_path = model_dir / "best_model_fold{}_model{}.pth".format(fold_id, m_id)
 model = eval(tmp_stgs["model"]["name"] )(**tmp_stgs["model"]["params"])
 model.load_state_dict(torch.load(model_path, map_location=torch.device(device)))
 test_loader = data.DataLoader(test_dataset, **tmp_stgs["loader"]["val"])
 test_pred = inference_function(tmp_stgs, model, test_loader, device)
 test_preds_arr[fold_id] = test_pred
del test_loader; del test_pred;
gc.collect()
test_pred_arr_avg += test_preds_arr.mean(axis=0)
 test_pred_arr_avg /= len(stgs_list)
 return test_pred_arr_avg<concatenate>
thresholds_ori = np.linspace(0.3, 0.7, 31)
 thresholds = np.log(thresholds_ori/(1-thresholds_ori))
ious = np.array([iou_metric_batch(y_valid, preds_valid > threshold)for threshold in tqdm_notebook(thresholds)])
 print(ious )
X_test = np.concatenate([
sub_df.iloc[:, 1:].values for sub_df in sub_list], axis=1)
 print(X_test.shape)
 X_test_2d = np.stack([
sub_df.iloc[:, 1:].values for sub_df in sub_list], axis=2)
 print(X_test_2d.shape)
 X_test_3d = X_test_2d[:, None,...]
print(X_test_3d.shape)
 X_test_node = np.stack([
sub_df.iloc[:, 1:].values for sub_df in sub_list], axis=2)
 test_dataset_mlp = MoAStackingDataset(X_test, None)
 test_dataset_1dcnn = MoAStackingDatasetForCNN(X_test_2d, None)
 test_dataset_2dcnn = MoAStackingDatasetForCNN(X_test_3d, None)
 test_dataset_gcn = MoAStackingDatasetForGCN(X_test_node, None )<init_hyperparams>

 def rle_encode(im):

 pixels = im.flatten(order = 'F')
 pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
settings_str_mlp =
settings_mlp = yaml.safe_load(settings_str_mlp)
 model_dir_mlp = Path(".. /input/moa-mlp-stacking-drug-cv" )<init_hyperparams>
preds_test = predict_result(model,x_test,img_size_target )
settings_str_1dcnn =
settings_1dcnn = yaml.safe_load(settings_str_1dcnn)
 model_dir_1dcnn = Path(".. /input/moa-1dcnn-stacking-drug-cv" )<init_hyperparams>
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv(submission_file )
<init_hyperparams><EOS> 
t_finish = time.time()
print(f"Kernel run time = {(t_finish-t_start)/3600} hours" )