Kaggle competition: TGS Salt Identification Challenge
<SOS> metric: IntersectionOverUnionObjectSegmentation Kaggle data source: tgs-salt-identification-challenge<categorify>
plt.style.use('seaborn-white')
 sns.set_style('white')
 plt.rcParams['figure.figsize'] = [16, 10]
plt.rcParams['font.size'] = 14
%matplotlib inline
warnings.filterwarnings("ignore")
 BatchNormalization, Activation, GlobalAveragePooling2D,
MaxPooling2D, concatenate, Reshape, Add, multiply)
 t_start = time.time()
use_test_for_preprocessing = False
for col in(GENES + CELLS):
if IS_TRAIN:
transformer = QuantileTransformer(n_quantiles=100, random_state=0, output_distribution="normal")
 if use_test_for_preprocessing:
raw_vec = pd.concat([train_features, test_features])[col].values.reshape(vec_len+vec_len_test, 1)
 transformer.fit(raw_vec)
 else:
raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 pd.to_pickle(transformer, f'{MODEL_DIR}/{NB}_{col}_quantile_transformer.pkl')
 else:
transformer = pd.read_pickle(f'{MODEL_DIR}/{NB}_{col}_quantile_transformer.pkl')
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 train_features[col] = transformer.transform(train_features[col].values.reshape(vec_len, 1)).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<categorify>
basic_name = f'Unet_resnet'
save_model_name = basic_name + '.model'
submission_file = basic_name + '.csv'
TRAIN_IMAGE_DIR = '.. /input/train/images/'
TRAIN_MASK_DIR = '.. /input/train/masks/'
TEST_IMAGE_DIR = '.. /input/test/images/'
img_size = 101
seed=1994
batch_size = 128
epochs = 120
n_comp = 90
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 if IS_TRAIN:
fa = FactorAnalysis(n_components=n_comp, random_state=42 ).fit(data[GENES])
 pd.to_pickle(fa, f'{MODEL_DIR}/{NB}_factor_analysis_g.pkl')
 else:
fa = pd.read_pickle(f'{MODEL_DIR}/{NB}_factor_analysis_g.pkl')
 data2 =(fa.transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 if IS_TRAIN:
fa = FactorAnalysis(n_components=n_comp, random_state=42 ).fit(data[CELLS])
 pd.to_pickle(fa, f'{MODEL_DIR}/{NB}_factor_analysis_c.pkl')
 else:
fa = pd.read_pickle(f'{MODEL_DIR}/{NB}_factor_analysis_c.pkl')
 data2 =(fa.transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<train_on_grid>
def get_iou_vector(A, B):
batch_size = A.shape[0]
metric = []
for batch in range(batch_size):
t, p = A[batch]>0, B[batch]>0
intersection = np.logical_and(t, p)
 union = np.logical_or(t, p)
 iou =(np.sum(intersection > 0)+ 1e-10)/(np.sum(union > 0)+ 1e-10)
 thresholds = np.arange(0.5, 1, 0.05)
 s = []
for thresh in thresholds:
s.append(iou > thresh)
 metric.append(np.mean(s))
return np.mean(metric)
 def my_iou_metric(label, pred):
return tf.py_func(get_iou_vector, [label, pred>0.5], tf.float64)
 def my_iou_metric_2(label, pred):
return tf.py_func(get_iou_vector, [label, pred >0], tf.float64)
 def iou_metric(y_true_in, y_pred_in, print_table=False):
labels = y_true_in
y_pred = y_pred_in
true_objects = 2
pred_objects = 2
temp1 = np.histogram2d(labels.flatten() , y_pred.flatten() , bins=([0,0.5,1], [0,0.5, 1]))
intersection = temp1[0]
area_true = np.histogram(labels,bins=[0,0.5,1])[0]
area_pred = np.histogram(y_pred, bins=[0,0.5,1])[0]
area_true = np.expand_dims(area_true, -1)
 area_pred = np.expand_dims(area_pred, 0)
 union = area_true + area_pred - intersection
intersection = intersection[1:,1:]
intersection[intersection == 0] = 1e-9
union = union[1:,1:]
union[union == 0] = 1e-9
iou = intersection / union
def precision_at(threshold, iou):
matches = iou > threshold
true_positives = np.sum(matches, axis=1)== 1
false_positives = np.sum(matches, axis=0)== 0
false_negatives = np.sum(matches, axis=1)== 0
tp, fp, fn = np.sum(true_positives), np.sum(false_positives), np.sum(false_negatives)
 return tp, fp, fn
prec = []
if print_table:
print("Thresh\tTP\tFP\tFN\tPrec.")
 for t in np.arange(0.5, 1.0, 0.05):
tp, fp, fn = precision_at(t, iou)
 if(tp + fp + fn)> 0:
p = tp /(tp + fp + fn)
 else:
p = 0
if print_table:
print("{:1.3f}\t{}\t{}\t{}\t{:1.3f}".format(t, tp, fp, fn, p))
prec.append(p)
 if print_table:
print("AP\t-\t-\t-\t{:1.3f}".format(np.mean(prec)))
 return np.mean(prec)
 def iou_metric_batch(y_true_in, y_pred_in):
batch_size = y_true_in.shape[0]
metric = []
for batch in range(batch_size):
value = iou_metric(y_true_in[batch], y_pred_in[batch])
 metric.append(value)
 return np.mean(metric)
 def lovasz_grad(gt_sorted):

 gts = tf.reduce_sum(gt_sorted)
 intersection = gts - tf.cumsum(gt_sorted)
 union = gts + tf.cumsum(1.- gt_sorted)
 jaccard = 1.- intersection / union
jaccard = tf.concat(( jaccard[0:1], jaccard[1:] - jaccard[:-1]), 0)
 return jaccard
def lovasz_hinge(logits, labels, per_image=True, ignore=None):

 if per_image:
def treat_image(log_lab):
log, lab = log_lab
log, lab = tf.expand_dims(log, 0), tf.expand_dims(lab, 0)
 log, lab = flatten_binary_scores(log, lab, ignore)
 return lovasz_hinge_flat(log, lab)
 losses = tf.map_fn(treat_image,(logits, labels), dtype=tf.float32)
 loss = tf.reduce_mean(losses)
 else:
loss = lovasz_hinge_flat(*flatten_binary_scores(logits, labels, ignore))
return loss
def lovasz_hinge_flat(logits, labels):

 def compute_loss() :
labelsf = tf.cast(labels, logits.dtype)
 signs = 2.* labelsf - 1.
 errors = 1.- logits * tf.stop_gradient(signs)
 errors_sorted, perm = tf.nn.top_k(errors, k=tf.shape(errors)[0], name="descending_sort")
 gt_sorted = tf.gather(labelsf, perm)
 grad = lovasz_grad(gt_sorted)
 loss = tf.tensordot(tf.nn.elu(errors_sorted), tf.stop_gradient(grad), 1, name="loss_non_void")
 return loss
loss = tf.cond(tf.equal(tf.shape(logits)[0], 0),
lambda: tf.reduce_sum(logits)* 0.,
compute_loss, strict=True, name="loss")
 return loss
def flatten_binary_scores(scores, labels, ignore=None):

 scores = tf.reshape(scores,(-1,))
labels = tf.reshape(labels,(-1,))
if ignore is None:
return scores, labels
valid = tf.not_equal(labels, ignore)
 vscores = tf.boolean_mask(scores, valid, name='valid_scores')
 vlabels = tf.boolean_mask(labels, valid, name='valid_labels')
 return vscores, vlabels
def lovasz_loss(y_true, y_pred):
y_true, y_pred = K.cast(K.squeeze(y_true, -1), 'int32'), K.cast(K.squeeze(y_pred, -1), 'float32')
 logits = y_pred
loss = lovasz_hinge(logits, y_true, per_image = True, ignore = None)
 return loss
def symmetric_lovasz(y_true, y_pred):
y_true, y_pred = K.cast(K.squeeze(y_true, -1), 'int32'), K.cast(K.squeeze(y_pred, -1), 'float32')
 loss =(( lovasz_hinge(y_pred, y_true, per_image = True, ignore = None)) + \ 
(lovasz_hinge(-y_pred, 1 - y_true, per_image = True, ignore = None)))/2
return loss
def fe_cluster(train, test, n_clusters_g = 35, n_clusters_c = 5, SEED = 123):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 if IS_TRAIN:
kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 pd.to_pickle(kmeans, f"{MODEL_DIR}/{NB}_kmeans_{kind}.pkl")
 else:
kmeans = pd.read_pickle(f"{MODEL_DIR}/{NB}_kmeans_{kind}.pkl")
 train[f'clusters_{kind}'] = kmeans.predict(train_)
 test[f'clusters_{kind}'] = kmeans.predict(test_)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features ,test_features=fe_cluster(train_features,test_features )<compute_test_metric>
train_df = pd.read_csv(".. /input/train.csv", index_col="id", usecols=[0])
 train_df["images"] = [np.array(load_img(TRAIN_IMAGE_DIR + "{}.png".format(idx), color_mode = "grayscale")) / 255
for idx in tqdm_notebook(train_df.index)]
train_df["masks"] = [np.array(load_img(TRAIN_MASK_DIR + "{}.png".format(idx), color_mode = "grayscale")) / 255
for idx in tqdm_notebook(train_df.index)]
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size, 2)
 def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i : return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class)
 train_df.head()
def fe_stats(train, test):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 for df in train, test:
df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 return train, test
train_features,test_features=fe_stats(train_features,test_features )<filter>
X = np.array(train_df.images.tolist() ).reshape(-1, img_size, img_size, 1)
 y = np.array(train_df.masks.tolist() ).reshape(-1, img_size, img_size, 1)
 x_train, x_valid, y_train, y_valid = \
train_test_split(
 X, y,
test_size=0.2,
stratify=train_df.coverage_class,
random_state=seed
)
remove_vehicle = True
if remove_vehicle:
trt_idx = train_features['cp_type']=='trt_cp'
train_features = train_features.loc[trt_idx].reset_index(drop=True)
 train_targets_scored = train_targets_scored.loc[trt_idx].reset_index(drop=True)
 train_targets_nonscored = train_targets_nonscored.loc[trt_idx].reset_index(drop=True)
 else:
pass<drop_column>
def BatchActivate(x):
x = BatchNormalization()(x)
 x = Activation('elu' )(x)
 return x
def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True):
x = Conv2D(filters, size, strides=strides, padding=padding )(x)
 if activation==True: x = BatchActivate(x)
 return x
def residual_block(blockInput, num_filters=16, batch_activate=False):
x = BatchActivate(blockInput)
 x = convolution_block(x, num_filters,(3,3))
x = convolution_block(x, num_filters,(3,3), activation=False)
 x = Add()([x, blockInput])
 if batch_activate: x = BatchActivate(x)
 return x
def squeeze_excite_block_cSE(input_, ratio=2):
init = input_
filters = K.int_shape(init)[-1]
se_shape =(1, 1, filters)
 se = GlobalAveragePooling2D()(init)
 se = Reshape(se_shape )(se)
 se = Dense(filters // ratio, activation='relu', kernel_initializer='he_normal', use_bias=True )(se)
 se = Dense(filters, activation='sigmoid', kernel_initializer='he_normal', use_bias=True )(se)
 x = multiply([init, se])
 return x
def squeeze_excite_block_sSE(input_):
sSE_scale = Conv2D(1,(1, 1), activation='sigmoid', padding="same", use_bias = True )(input_)
 return multiply([input_, sSE_scale])
 def unet_layer(blockInput, num_filters, use_csSE_ratio = 2):
x = Conv2D(num_filters,(3, 3), activation=None, padding="same" )(blockInput)
 x = residual_block(x, num_filters)
 x = residual_block(x, num_filters , batch_activate = True)
 if use_csSE_ratio > 0:
sSEx = squeeze_excite_block_sSE(x)
 cSEx = squeeze_excite_block_cSE(x,ratio = use_csSE_ratio)
 x = Add()([sSEx, cSEx])
 return x
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )<load_pretrained>
def build_model(input_layer, start_neurons, DropoutRatio=0.5):
conv1 = unet_layer(input_layer,start_neurons * 1,use_csSE_ratio=2)
 pool1 = MaxPooling2D(( 2,2))(conv1)
 pool1 = Dropout(DropoutRatio/3 )(pool1)
 conv2 = unet_layer(pool1, start_neurons * 2,use_csSE_ratio=2)
 pool2 = MaxPooling2D(( 2,2))(conv2)
 pool2 = Dropout(DropoutRatio/2 )(pool2)
 conv3 = unet_layer(pool2, start_neurons * 4,use_csSE_ratio=2)
 pool3 = MaxPooling2D(( 2,2))(conv3)
 pool3 = Dropout(DropoutRatio )(pool3)
 conv4 = unet_layer(pool3, start_neurons * 8,use_csSE_ratio=2)
 pool4 = MaxPooling2D(( 2,2))(conv4)
 pool4 = Dropout(DropoutRatio )(pool4)
 convm = Conv2D(start_neurons*16,(3,3), activation=None, padding='same' )(pool4)
 convm = residual_block(convm, start_neurons*16)
 convm = residual_block(convm, start_neurons*16, True)
 deconv4 = Conv2DTranspose(start_neurons*8,(3,3), strides=(2,2), padding='same' )(convm)
 uconv4 = concatenate([deconv4, conv4])
 uconv4 = Dropout(DropoutRatio )(uconv4)
 uconv4 = Conv2D(start_neurons*8,(3,3), activation=None, padding='same' )(uconv4)
 uconv4 = residual_block(uconv4, start_neurons*8)
 uconv4 = residual_block(uconv4, start_neurons*8, True)
 deconv3 = Conv2DTranspose(start_neurons*4,(3,3), strides=(2,2), padding='valid' )(uconv4)
 uconv3 = concatenate([deconv3, conv3])
 uconv3 = Dropout(DropoutRatio )(uconv3)
 uconv3 = Conv2D(start_neurons*4,(3,3), activation=None, padding='same' )(uconv3)
 uconv3 = residual_block(uconv3, start_neurons*4)
 uconv3 = residual_block(uconv3, start_neurons*4, True)
 deconv2 = Conv2DTranspose(start_neurons*2,(3,3), strides=(2,2), padding='same' )(uconv3)
 uconv2 = concatenate([deconv2, conv2])
 uconv2 = Dropout(DropoutRatio )(uconv2)
 uconv2 = Conv2D(start_neurons*2,(3,3), activation=None, padding='same' )(uconv2)
 uconv2 = residual_block(uconv2, start_neurons*2)
 uconv2 = residual_block(uconv2, start_neurons*2, True)
 deconv1 = Conv2DTranspose(start_neurons*1,(3,3), strides=(2,2), padding='valid' )(uconv2)
 uconv1 = concatenate([deconv1, conv1])
 uconv1 = Dropout(DropoutRatio )(uconv1)
 uconv1 = Conv2D(start_neurons*1,(3,3), activation=None, padding='same' )(uconv1)
 uconv1 = residual_block(uconv1, start_neurons*1)
 uconv1 = residual_block(uconv1, start_neurons*1, True)
 output_layer_noActi = Conv2D(1,(1,1), padding='same', activation=None )(uconv1)
 output_layer = Activation('sigmoid' )(output_layer_noActi)
 return output_layer
train_nonscored_pred = pd.read_pickle(f'{INT_DIR}/{NB_PREV}_train_nonscored_pred.pkl')
 test_nonscored_pred = pd.read_pickle(f'{INT_DIR}/{NB_PREV}_test_nonscored_pred.pkl' )<filter>
def do_augmentation(seqs, seq2_train, X_train, y_train):
seq_det = seqs.to_deterministic()
X_train_aug = seq_det.augment_image(X_train)
 X_train_aug = seq2_train.augment_image(X_train_aug)
 y_train_aug = seq_det.augment_image(y_train)
 if y_train_aug.shape !=(101, 101):
X_train_aug = ia.imresize_single_image(X_train_aug,(101, 101), interpolation="linear")
 y_train_aug = ia.imresize_single_image(y_train_aug,(101, 101), interpolation="nearest")
 return np.array(X_train_aug), np.array(y_train_aug)
 sometimes = lambda aug: iaa.Sometimes(0.5, aug)
 seq = iaa.Sequential([
iaa.Fliplr(0.5),
iaa.OneOf([
iaa.Noop() ,
iaa.Affine(rotate=(-10, 10), translate_percent={"x":(-0.25, 0.25)}, mode='symmetric', cval=(0), backend="cv2"),
iaa.Noop() ,
iaa.CropAndPad(
 percent=(-0.2, 0.2),
pad_mode="reflect",
pad_cval=0,
keep_size=False 
),
])
 ])
 seq_train = iaa.Sequential(
 sometimes(iaa.Multiply(( 0.8, 1.2))),
sometimes(iaa.Add(( -0.2, 0.2))),
sometimes(iaa.OneOf([
iaa.AdditiveGaussianNoise(scale=(0, 0.05)) ,
iaa.GaussianBlur(sigma=(0.0, 1.0)) ,
])) 
)
 def make_image_gen(features, labels, batch_size=32):
all_batches_index = np.arange(0, features.shape[0])
 out_images = []
out_masks = []
while True:
np.random.shuffle(all_batches_index)
 for index in all_batches_index:
c_img, c_mask = do_augmentation(seq, seq_train, features[index], labels[index])
 out_images += [c_img]
out_masks += [c_mask]
if len(out_images)>= batch_size:
yield np.stack(out_images, 0), np.stack(out_masks, 0)
 out_images, out_masks = [], []
train_targets_nonscored = train_targets_nonscored.loc[:, train_targets_nonscored.sum() != 0] 
<merge>
input_layer = Input(( img_size, img_size, 1))
output_layer = build_model(input_layer, 16,0.5)
 model1 = Model(input_layer, output_layer)
 model1.compile(loss="binary_crossentropy", optimizer=Adam(lr = 0.005), metrics=[my_iou_metric])
train = train.merge(train_nonscored_pred[train_targets_nonscored.columns], on='sig_id')
 test = test.merge(test_nonscored_pred[train_targets_nonscored.columns], on='sig_id' )<categorify>
early_stopping = EarlyStopping(monitor='val_my_iou_metric', mode = 'max',patience=20, verbose=1)
 model_checkpoint = ModelCheckpoint(save_model_name, monitor='val_my_iou_metric', mode='max', save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='val_my_iou_metric', mode='max', factor=0.5, patience=10, min_lr=0.0001, verbose=1)
 train_generator = make_image_gen(x_train, y_train,batch_size)
 t_model1_start = time.time()
history = model1.fit_generator(
 train_generator,
steps_per_epoch = x_train.shape[0]*2//batch_size,
validation_data=[x_valid, y_valid],
epochs=epochs,
callbacks = [early_stopping, model_checkpoint, reduce_lr],
verbose = 1 
)
 t_model1_end = time.time()
print(f"Run time = {(t_model1_end-t_model1_start)/3600} hours" )
nonscored_target = [c for c in train_targets_nonscored.columns if c != "sig_id"]
for col in(nonscored_target):
vec_len = len(train[col].values)
 vec_len_test = len(test[col].values)
 raw_vec = train[col].values.reshape(vec_len, 1)
 if IS_TRAIN:
transformer = QuantileTransformer(n_quantiles=100, random_state=0, output_distribution="normal")
 transformer.fit(raw_vec)
 pd.to_pickle(transformer, f'{MODEL_DIR}/{NB}_{col}_quantile_transformer.pkl')
 else:
transformer = pd.read_pickle(f'{MODEL_DIR}/{NB}_{col}_quantile_transformer.pkl')
 train[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test[col] = transformer.transform(test[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<define_variables>
model1 = load_model(save_model_name, custom_objects={'my_iou_metric':my_iou_metric})
 input_x = model1.layers[0].input
output_layer = model1.layers[-1].input
model2 = Model(input_x, output_layer)
 model2.compile(loss=symmetric_lovasz, optimizer=Adam(lr=0.01), metrics=[my_iou_metric_2] )
feature_cols = [c for c in train.columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['sig_id']]
len(feature_cols )<define_variables>
early_stopping = EarlyStopping(monitor='val_my_iou_metric_2', mode = 'max',patience=30, verbose=1)
 model_checkpoint = ModelCheckpoint(save_model_name,monitor='val_my_iou_metric_2', mode = 'max', save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='val_my_iou_metric_2', mode = 'max',factor=0.5, patience=10, min_lr=0.00005, verbose=1)
 train_generator = make_image_gen(x_train, y_train,batch_size//2)
 t_model2_start = time.time()
history = model2.fit_generator(
 train_generator,
steps_per_epoch = x_train.shape[0]*2//batch_size,
validation_data=[x_valid, y_valid],
epochs=epochs + int(epochs*(1/5)) ,
callbacks = [early_stopping, model_checkpoint, reduce_lr],
verbose = 1 
)
 t_model2_end = time.time()
print(f"Run time = {(t_model2_end - t_model2_start)/3600} hours" )
num_features=len(feature_cols)
 num_targets=len(target_cols )<normalization>
model = load_model(save_model_name,custom_objects={'my_iou_metric_2': my_iou_metric_2, 'symmetric_lovasz': symmetric_lovasz} )
class LabelSmoothing(nn.Module):

 def __init__(self, smoothing=0.0, n_cls=2):

 super(LabelSmoothing, self ).__init__()
self.confidence = 1.0 - smoothing + smoothing / n_cls
self.smoothing = smoothing / n_cls
def forward(self, x, target):
probs = torch.nn.functional.sigmoid(x,)
 target1 = self.confidence * target +(1-target)* self.smoothing
loss = -(torch.log(probs+1e-15)* target1 +(1-target1)* torch.log(1-probs+1e-15))
return loss.mean()
class SmoothedLogLossMetric(Metric):

 def __init__(self, smoothing=0.001):
self._name = f"{smoothing:.3f}"
self._maximize = False
self._lossfn = LabelSmoothing(smoothing)
 def __call__(self, y_true, y_score):

 y_true = torch.from_numpy(y_true.astype(np.float32)).clone()
y_score = torch.from_numpy(y_score.astype(np.float32)).clone()
return self._lossfn(y_score, y_true ).to('cpu' ).detach().numpy().copy().take(0)
 class LogLossMetric(Metric):

 def __init__(self, smoothing=0.0):
self._name = f"{smoothing:.3f}"
self._maximize = False
self._lossfn = LabelSmoothing(smoothing)
 def __call__(self, y_true, y_score):

 y_true = torch.from_numpy(y_true.astype(np.float32)).clone()
y_score = torch.from_numpy(y_score.astype(np.float32)).clone()
return self._lossfn(y_score, y_true ).to('cpu' ).detach().numpy().copy().take(0 )<categorify>
def predict_result(model,x_test,img_size):
x_test_reflect = np.array([np.fliplr(x)for x in x_test])
 preds_test = model.predict(x_test ).reshape(-1, img_size, img_size)
 preds_test2_refect = model.predict(x_test_reflect ).reshape(-1, img_size, img_size)
 preds_test += np.array([ np.fliplr(x)for x in preds_test2_refect])
 return preds_test/2
preds_valid = predict_result(model,x_valid,img_size )
def process_data(data):
data.loc[:, 'cp_time'] = data.loc[:, 'cp_time'].map({24: 0, 48: 1, 72: 2, 0: 0, 1: 1, 2: 2})
 data.loc[:, 'cp_dose'] = data.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1, 0: 0, 1: 1})
 return data
def run_training_tabnet(train, test, trn_idx, val_idx, feature_cols, target_cols, fold, seed, filename="tabnet"):
seed_everything(seed)
 train_ = process_data(train)
 test_ = process_data(test)
 train_df = train_.loc[trn_idx,:].reset_index(drop=True)
 valid_df = train_.loc[val_idx,:].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
model = TabNetRegressor(n_d=32, n_a=32, n_steps=1, lambda_sparse=0,
cat_dims=[3, 2], cat_emb_dim=[1, 1], cat_idxs=[0, 1],
optimizer_fn=torch.optim.Adam, optimizer_params=dict(lr=2e-2, weight_decay=1e-5),
mask_type='entmax',
scheduler_params=dict(milestones=[100, 150], gamma=0.9),
scheduler_fn=torch.optim.lr_scheduler.MultiStepLR,
verbose=10,
seed = seed)
 loss_fn = LabelSmoothing(0.001)
 oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
if IS_TRAIN:
model.fit(X_train=x_train, y_train=y_train,
eval_set=[(x_valid, y_valid)], eval_metric=[LogLossMetric, SmoothedLogLossMetric],
max_epochs=200, patience=50, batch_size=1024, virtual_batch_size=128,
num_workers=0, drop_last=False, loss_fn=loss_fn 
)
 model.save_model(f"{MODEL_DIR}/{NB}_{filename}_SEED{seed}_FOLD{fold}")
 x_test = test_[feature_cols].values
model = TabNetRegressor(n_d=32, n_a=32, n_steps=1, lambda_sparse=0,
cat_dims=[3, 2], cat_emb_dim=[1, 1], cat_idxs=[0, 1],
optimizer_fn=torch.optim.Adam, optimizer_params=dict(lr=2e-2, weight_decay=1e-5),
mask_type='entmax',
scheduler_params=dict(milestones=[100, 150], gamma=0.9),
scheduler_fn=torch.optim.lr_scheduler.MultiStepLR,
verbose=10,
seed = seed)
 model.load_model(f"{MODEL_DIR}/{NB}_{filename}_SEED{seed}_FOLD{fold}.model")
 valid_preds = model.predict(x_valid)
 valid_preds = torch.sigmoid(torch.as_tensor(valid_preds)).detach().cpu().numpy()
oof[val_idx] = valid_preds
predictions = model.predict(x_test)
 predictions = torch.sigmoid(torch.as_tensor(predictions)).detach().cpu().numpy()
return oof, predictions<split>
thresholds_ori = np.linspace(0.3, 0.7, 31)
 thresholds = np.log(thresholds_ori/(1-thresholds_ori))
ious = np.array([iou_metric_batch(y_valid, preds_valid > threshold)for threshold in tqdm_notebook(thresholds)])
 print(ious);
def run_k_fold(train, test, feature_cols, target_cols, NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 mskf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state = seed)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
oof_, pred_ = run_training_tabnet(train, test, t_idx, v_idx, feature_cols, target_cols, f, seed)
 predictions += pred_ / NFOLDS / NREPEATS
oof += oof_ / NREPEATS
return oof, predictions
def run_seeds(train, test, feature_cols, target_cols, nfolds=NFOLDS, nseed=NSEEDS):
seed_list = range(nseed)
 oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 time_start = time.time()
for seed in seed_list:
oof_, predictions_ = run_k_fold(train, test, feature_cols, target_cols, nfolds, seed)
 oof += oof_ / nseed
predictions += predictions_ / nseed
print(f"seed {seed}, elapsed time: {time.time() - time_start}")
 train[target_cols] = oof
test[target_cols] = predictions<load_pretrained>
def rle_encode(im):

 pixels = im.flatten(order='F')
 pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
train.to_pickle(f"{INT_DIR}/{NB}_pre_train.pkl")
 test.to_pickle(f"{INT_DIR}/{NB}_pre_test.pkl" )<compute_test_metric>
test_images = os.listdir(TEST_IMAGE_DIR)
 x_test = np.array([(np.array(load_img(TEST_IMAGE_DIR + "{}".format(idx), color_mode = "grayscale")))/ 255
for idx in tqdm_notebook(test_images)] ).reshape(-1, img_size, img_size, 1)
 preds_test = predict_result(model,x_test,img_size)
 pred_dict = {idx[:10]: rle_encode(np.round(preds_test[i] > threshold_best))
for i, idx in enumerate(tqdm_notebook(test_images)) };
run_seeds(train, test, feature_cols, target_cols, NFOLDS, NSEEDS )<load_pretrained>
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv(submission_file )
<compute_test_metric><EOS> 
t_finish = time.time()
print(f"Kernel run time = {(t_finish-t_start)/3600} hours" )