Kaggle competition: TGS Salt Identification Challenge
<SOS> metric: IntersectionOverUnionObjectSegmentation Kaggle data source: tgs-salt-identification-challenge<drop_column>
plt.style.use('seaborn-white')
 sns.set_style("white")
train = train_features[train_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )<load_pretrained>
img_size_ori = 101
img_size_target = 128
def upsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_target, img_size_target), mode='constant', preserve_range=True)
 def downsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_ori, img_size_ori), mode='constant', preserve_range=True)
train.to_pickle(f"{INT_DIR}/{NB}_train_preprocessed.pkl")
 test.to_pickle(f"{INT_DIR}/{NB}_test_preprocessed.pkl" )<init_hyperparams>
def dice_coef(y_true, y_pred):
y_true_f = K.flatten(y_true)
 y_pred = K.cast(y_pred, 'float32')
 y_pred_f = K.cast(K.greater(K.flatten(y_pred), 0.5), 'float32')
 intersection = y_true_f * y_pred_f
score = 2.* K.sum(intersection)/(K.sum(y_true_f)+ K.sum(y_pred_f))
return score
def dice_loss(y_true, y_pred):
smooth = 1.
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = y_true_f * y_pred_f
score =(2.* K.sum(intersection)+ smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 return 1.- score
def bce_dice_loss(y_true, y_pred):
return binary_crossentropy(y_true, K.sigmoid(y_pred)) + dice_loss(y_true, K.sigmoid(y_pred))
def bce_logdice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)- K.log(1.- dice_loss(y_true, y_pred))
def weighted_bce_loss(y_true, y_pred, weight):
epsilon = 1e-7
y_pred = K.clip(y_pred, epsilon, 1.- epsilon)
 logit_y_pred = K.log(y_pred /(1.- y_pred))
loss = weight *(logit_y_pred *(1.- y_true)+
K.log(1.+ K.exp(-K.abs(logit_y_pred)))+ K.maximum(-logit_y_pred, 0.))
 return K.sum(loss)/ K.sum(weight)
 def weighted_dice_loss(y_true, y_pred, weight):
smooth = 1.
 w, m1, m2 = weight, y_true, y_pred
intersection =(m1 * m2)
 score =(2.* K.sum(w * intersection)+ smooth)/(K.sum(w * m1)+ K.sum(w * m2)+ smooth)
 loss = 1.- K.sum(score)
 return loss
def weighted_bce_dice_loss(y_true, y_pred):
y_true = K.cast(y_true, 'float32')
 y_pred = K.cast(y_pred, 'float32')
 averaged_mask = K.pool2d(
 y_true, pool_size=(50, 50), strides=(1, 1), padding='same', pool_mode='avg')
 weight = K.ones_like(averaged_mask)
 w0 = K.sum(weight)
 weight = 5.* K.exp(-5.* K.abs(averaged_mask - 0.5))
w1 = K.sum(weight)
 weight *=(w0 / w1)
 loss = weighted_bce_loss(y_true, y_pred, weight)+ dice_loss(y_true, y_pred)
 return loss

 def lovasz_grad(gt_sorted):

 gts = tf.reduce_sum(gt_sorted)
 intersection = gts - tf.cumsum(gt_sorted)
 union = gts + tf.cumsum(1.- gt_sorted)
 jaccard = 1.- intersection / union
jaccard = tf.concat(( jaccard[0:1], jaccard[1:] - jaccard[:-1]), 0)
 return jaccard
def lovasz_hinge(logits, labels, per_image=True, ignore=None):

 if per_image:
def treat_image(log_lab):
log, lab = log_lab
log, lab = tf.expand_dims(log, 0), tf.expand_dims(lab, 0)
 log, lab = flatten_binary_scores(log, lab, ignore)
 return lovasz_hinge_flat(log, lab)
 losses = tf.map_fn(treat_image,(logits, labels), dtype=tf.float32)
 loss = tf.reduce_mean(losses)
 else:
loss = lovasz_hinge_flat(*flatten_binary_scores(logits, labels, ignore))
return loss
def lovasz_hinge_flat(logits, labels):

 def compute_loss() :
labelsf = tf.cast(labels, logits.dtype)
 signs = 2.* labelsf - 1.
 errors = 1.- logits * tf.stop_gradient(signs)
 errors_sorted, perm = tf.nn.top_k(errors, k=tf.shape(errors)[0], name="descending_sort")
 gt_sorted = tf.gather(labelsf, perm)
 grad = lovasz_grad(gt_sorted)
 loss = tf.tensordot(tf.nn.relu(errors_sorted), tf.stop_gradient(grad), 1, name="loss_non_void")
 return loss
loss = tf.cond(tf.equal(tf.shape(logits)[0], 0),
lambda: tf.reduce_sum(logits)* 0.,
compute_loss,
strict=True,
name="loss" 
)
 return loss
def flatten_binary_scores(scores, labels, ignore=None):

 scores = tf.reshape(scores,(-1,))
labels = tf.reshape(labels,(-1,))
if ignore is None:
return scores, labels
valid = tf.not_equal(labels, ignore)
 vscores = tf.boolean_mask(scores, valid, name='valid_scores')
 vlabels = tf.boolean_mask(labels, valid, name='valid_labels')
 return vscores, vlabels
def lovasz_softmax(probas, labels, classes='all', per_image=False, ignore=None, order='BHWC'):

 if per_image:
def treat_image(prob_lab):
prob, lab = prob_lab
prob, lab = tf.expand_dims(prob, 0), tf.expand_dims(lab, 0)
 prob, lab = flatten_probas(prob, lab, ignore, order)
 return lovasz_softmax_flat(prob, lab, classes=classes)
 losses = tf.map_fn(treat_image,(probas, labels), dtype=tf.float32)
 loss = tf.reduce_mean(losses)
 else:
loss = lovasz_softmax_flat(*flatten_probas(probas, labels, ignore, order), classes=classes)
 return loss
def lovasz_softmax_flat(probas, labels, classes='all'):

 C = 1
losses = []
present = []
class_to_sum = list(range(C)) if classes in ['all', 'present'] else classes
for c in class_to_sum:
fg = tf.cast(tf.equal(labels, c), probas.dtype)
 if classes == 'present':
present.append(tf.reduce_sum(fg)> 0)
 errors = tf.abs(fg - probas[:, c])
 errors_sorted, perm = tf.nn.top_k(errors, k=tf.shape(errors)[0], name="descending_sort_{}".format(c))
fg_sorted = tf.gather(fg, perm)
 grad = lovasz_grad(fg_sorted)
 losses.append(
 tf.tensordot(errors_sorted, tf.stop_gradient(grad), 1, name="loss_class_{}".format(c)) 
)
 if len(class_to_sum)== 1:
return losses[0]
losses_tensor = tf.stack(losses)
 if classes == 'present':
present = tf.stack(present)
 losses_tensor = tf.boolean_mask(losses_tensor, present)
 loss = tf.reduce_mean(losses_tensor)
 return loss
def flatten_probas(probas, labels, ignore=None, order='BHWC'):

 if order == 'BCHW':
probas = tf.transpose(probas,(0, 2, 3, 1), name="BCHW_to_BHWC")
 order = 'BHWC'
if order != 'BHWC':
raise NotImplementedError('Order {} unknown'.format(order))
C = 1
probas = tf.reshape(probas,(-1, C))
labels = tf.reshape(labels,(-1,))
if ignore is None:
return probas, labels
valid = tf.not_equal(labels, ignore)
 vprobas = tf.boolean_mask(probas, valid, name='valid_probas')
 vlabels = tf.boolean_mask(labels, valid, name='valid_labels')
 return vprobas, vlabels
def keras_lovasz_softmax(labels,probas):
return lovasz_softmax(probas, labels)
 def keras_lovasz_hinge(labels,logits):
return lovasz_hinge(logits, labels, per_image=True, ignore=None)
 def mean_iou(y_true, y_pred):
prec = []
for t in np.arange(0.5, 1.0, 0.05):
y_pred_ = tf.to_int32(y_pred > t)
 score, up_opt = tf.metrics.mean_iou(y_true, y_pred_, 2)
 K.get_session().run(tf.local_variables_initializer())
 with tf.control_dependencies([up_opt]):
score = tf.identity(score)
 prec.append(score)
 return K.mean(K.stack(prec), axis=0)
 def focal_loss(y_true, y_pred):
gamma=0.75
alpha=0.25
pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
pt_1 = K.clip(pt_1, 1e-3,.999)
 pt_0 = K.clip(pt_0, 1e-3,.999)
 return -K.sum(alpha * K.pow(1.- pt_1, gamma)* K.log(pt_1)) -K.sum(( 1-alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0))
NB = '203'
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 PMIN = 0.0
PMAX = 1.0
SMIN = 0.0
SMAX = 1.0
HIDDEN_SIZE = 2048
EPOCHS = 15
BATCH_SIZE = 2048
NFOLDS = 10
NREPEATS = 1
NSEEDS = 5
LEARNING_RATE = 5e-4
WEIGHT_DECAY = 1e-5
PCT_START = 0.2
DIV_FACS = 1e3
MAX_LR = 1e-2<categorify>
train_df = pd.read_csv(".. /input/train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(".. /input/depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
def calc_valid_log_loss(train, target, target_cols):
y_pred = train[target_cols].values
y_true = target[target_cols].values
y_true_t = torch.from_numpy(y_true.astype(np.float64)).clone()
y_pred_t = torch.from_numpy(y_pred.astype(np.float64)).clone()
return torch.nn.BCELoss()(y_pred_t, y_true_t ).to('cpu' ).detach().numpy().copy()<normalization>
train_df["images"] = [np.array(load_img(".. /input/train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size=HIDDEN_SIZE):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.25)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x<prepare_x_and_y>
train_df["masks"] = [np.array(load_img(".. /input/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
def run_training(train, test, trn_idx, val_idx, feature_cols, target_cols, fold, seed):
seed_everything(seed)
 train_ = process_data(train)
 test_ = process_data(test)
 train_df = train_.loc[trn_idx,:].reset_index(drop=True)
 valid_df = train_.loc[val_idx,:].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=len(feature_cols),
num_targets=len(target_cols), 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=PCT_START, div_factor=DIV_FACS,
max_lr=MAX_LR, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
best_loss_epoch = -1
if IS_TRAIN:
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer, scheduler, loss_fn, trainloader, DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 if valid_loss < best_loss:
best_loss = valid_loss
best_loss_epoch = epoch
oof[val_idx] = valid_preds
model.to('cpu')
 torch.save(model.state_dict() , f"{MODEL_DIR}/{NB}_nonscored_SEED{seed}_FOLD{fold}_.pth")
 model.to(DEVICE)
 if epoch % 10 == 0 or epoch == EPOCHS-1:
print(f"seed: {seed}, FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss:.6f}, valid_loss: {valid_loss:.6f}, best_loss: {best_loss:.6f}, best_loss_epoch: {best_loss_epoch}")
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=len(feature_cols),
num_targets=len(target_cols), 
)
 model.load_state_dict(torch.load(f"{MODEL_DIR}/{NB}_nonscored_SEED{seed}_FOLD{fold}_.pth"))
model.to(DEVICE)
 if not IS_TRAIN:
valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 oof[val_idx] = valid_preds
predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions<split>
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size_ori, 2 )
def run_k_fold(train, test, feature_cols, target_cols, NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 mskf = RepeatedMultilabelStratifiedKFold(n_splits=NFOLDS, n_repeats=NREPEATS, random_state=None)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
oof_, pred_ = run_training(train, test, t_idx, v_idx, feature_cols, target_cols, f, seed)
 predictions += pred_ / NFOLDS / NREPEATS
oof += oof_ / NREPEATS
return oof, predictions<feature_engineering>
def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class )
def run_seeds(train, test, feature_cols, target_cols, nfolds=NFOLDS, nseed=NSEEDS):
seed_list = range(nseed)
 oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 time_start = time.time()
for seed in seed_list:
oof_, predictions_ = run_k_fold(train, test, feature_cols, target_cols, nfolds, seed)
 oof += oof_ / nseed
predictions += predictions_ / nseed
print(f"seed {seed}, elapsed time: {time.time() - time_start}")
 train[target_cols] = oof
test[target_cols] = predictions<load_from_csv>
ids_train, ids_valid, x_train, x_valid, y_train, y_valid, cov_train, cov_test, depth_train, depth_test = train_test_split(
 train_df.index.values,
np.array(train_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
np.array(train_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
train_df.coverage.values,
train_df.z.values,
test_size=0.2, stratify=train_df.coverage_class, random_state=1337 )
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<load_pretrained>
def conv_block(m, dim, acti, bn, res, do=0):
	n = Conv2D(dim, 3, activation=acti, padding='same' )(m)
 	n = BatchNormalization()(n)if bn else n
	n = Dropout(do )(n)if do else n
	n = Conv2D(dim, 3, activation=acti, padding='same' )(n)
 	n = BatchNormalization()(n)if bn else n
	return Concatenate()([m, n])if res else n
def level_block(m, dim, depth, inc, acti, do, bn, mp, up, res):
	if depth > 0:
		n = conv_block(m, dim, acti, bn, res)
 		m = MaxPooling2D()(n)if mp else Conv2D(dim, 3, strides=2, padding='same' )(n)
 		m = level_block(m, int(inc*dim), depth-1, inc, acti, do, bn, mp, up, res)
 		if up:
			m = UpSampling2D()(m)
 			m = Conv2D(dim, 2, activation=acti, padding='same' )(m)
 		else:
			m = Conv2DTranspose(dim, 3, strides=2, activation=acti, padding='same' )(m)
 		n = Concatenate()([n, m])
 		m = conv_block(n, dim, acti, bn, res)
 	else:
		m = conv_block(m, dim, acti, bn, res, do)
 	return m
def UNet(img_shape, out_ch=1, start_ch=64, depth=4, inc_rate=2., activation='relu',
		 dropout=0.5, batchnorm=False, maxpool=True, upconv=True, residual=False):
	i = Input(shape=img_shape)
 	o = level_block(i, start_ch, depth, inc_rate, activation, dropout, batchnorm, maxpool, upconv, residual)
 	o = Conv2D(out_ch, 1 )(o)
 	o = Activation("sigmoid" )(o)
 	return Model(inputs=i, outputs=o)
 def Logit_UNet(img_shape, out_ch=1, start_ch=64, depth=4, inc_rate=2., activation='relu',
		 dropout=0.5, batchnorm=False, maxpool=True, upconv=True, residual=False):
	i = Input(shape=img_shape)
 	o = level_block(i, start_ch, depth, inc_rate, activation, dropout, batchnorm, maxpool, upconv, residual)
 	o = Conv2D(out_ch, 1 )(o)
 	return Model(inputs=i, outputs=o )
train = pd.read_pickle(f"{INT_DIR}/101_train_preprocessed.pkl")
 test = pd.read_pickle(f"{INT_DIR}/101_test_preprocessed.pkl" )<load_pretrained>
model = Logit_UNet(( img_size_target,img_size_target,1),start_ch=16,depth=5,batchnorm=True)
train_trainbook = pd.read_pickle(".. /input/503-203-tabnet-with-nonscored-features-train/interim/101_train_preprocessed.pkl")
 test_trainbook = pd.read_pickle(".. /input/503-203-tabnet-with-nonscored-features-train/interim/101_test_preprocessed.pkl" )<merge>
model.compile(loss=bce_dice_loss, optimizer="adam", metrics=["accuracy",mean_iou])
train_targets_nonscored = train_targets_nonscored.loc[:, train_targets_nonscored.sum() != 0]
print(train_targets_nonscored.shape)
 train = train.merge(train_targets_nonscored, on='sig_id' )<compute_test_metric>
train_gen = ImageDataGenerator(
 horizontal_flip=True,
shear_range=0)
 val_gen = ImageDataGenerator(
 horizontal_flip=True,
shear_range=0 )
run_seeds(train, test, feature_cols, target_cols )<compute_test_metric>
early_stopping = EarlyStopping(patience=10, verbose=1)
 model_checkpoint = ModelCheckpoint("./keras.model", save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(factor=0.5, patience=5, min_lr=0.00001, verbose=1)
 epochs = 20
batch_size = 128
history1 = model.fit_generator(train_gen.flow(x_train, y_train, batch_size=batch_size),
steps_per_epoch=len(x_train)/ 32,
epochs=epochs,
validation_data=val_gen.flow(x_valid, y_valid, batch_size=batch_size),
callbacks=[early_stopping, model_checkpoint, reduce_lr],shuffle=True)
 model.compile(loss=focal_loss, optimizer="sgd", metrics=["accuracy",mean_iou])
 epochs = 80
batch_size = 128
history2 = model.fit_generator(train_gen.flow(x_train, y_train, batch_size=batch_size),
steps_per_epoch=len(x_train)/ 32,
epochs=epochs,
validation_data=val_gen.flow(x_valid, y_valid, batch_size=batch_size),
callbacks=[early_stopping, model_checkpoint, reduce_lr],shuffle=True )
valid_loss_total = calc_valid_log_loss(train, target, target_cols)
 print(f"CV loss: {valid_loss_total}" )<load_pretrained>
model.load_weights("./keras.model" )
train.to_pickle(f"{INT_DIR}/{NB}_train_nonscored_pred.pkl")
 test.to_pickle(f"{INT_DIR}/{NB}_test_nonscored_pred.pkl" )<compute_test_metric>
preds_valid = model.predict(x_valid ).reshape(-1, img_size_target, img_size_target)
 preds_valid = np.array([downsample(x)for x in preds_valid])
 y_valid_ori = np.array([train_df.loc[idx].masks for idx in ids_valid] )
valid_results = train_targets_nonscored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_nonscored[target_cols].values
y_true = y_true > 0.5
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )<install_modules>
thresholds = np.linspace(0, 1, 50)
 ious = np.array([iou_metric_batch(y_valid_ori, np.int32(preds_valid > threshold)) for threshold in tqdm_notebook(thresholds)] )
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet<import_modules>
threshold_best_index = np.argmax(ious[9:-10])+ 9
iou_best = ious[threshold_best_index]
threshold_best = thresholds[threshold_best_index]
from pytorch_tabnet.tab_model import TabNetRegressor<set_options>
def RLenc(img, order='F', format=True):

 bytes = img.reshape(img.shape[0] * img.shape[1], order=order)
 runs = []
r = 0
pos = 1
for c in bytes:
if(c == 0):
if r != 0:
runs.append(( pos, r))
pos += r
r = 0
pos += 1
else:
r += 1
if r != 0:
runs.append(( pos, r))
pos += r
r = 0
if format:
z = ''
for rr in runs:
z += '{} {} '.format(rr[0], rr[1])
 return z[:-1]
else:
return runs
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42 )<define_variables>
preds_test = model.predict(x_test )
NB = '503'
NB_PREV = '203'
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 PMIN = 0.0
PMAX = 1.0
SMIN = 0.0
SMAX = 1.0
NFOLDS = 10
NREPEATS = 1
NSEEDS = 3
LEARNING_RATE = 5e-4
WEIGHT_DECAY = 1e-5
PCT_START = 0.2
DIV_FACS = 1e3
MAX_LR = 1e-2<load_from_csv>
pred_dict = {idx: RLenc(np.round(downsample(preds_test[i])> threshold_best)) for i, idx in enumerate(tqdm_notebook(test_df.index.values)) }
<define_variables><EOS> 
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv('submission.csv' )