Kaggle competition: Severstal: Steel Defect Detection
<SOS> metric: Dice Kaggle data source: severstal:-steel-defect-detection<prepare_x_and_y>
import os
import json
import cv2
import keras
from keras import backend as K
from keras.models import Model
from keras.layers import Input
from keras.layers.convolutional import Conv2D, Conv2DTranspose
from keras.layers.pooling import MaxPooling2D
from keras.layers.merge import concatenate
from keras.losses import binary_crossentropy
from keras.callbacks import Callback, ModelCheckpoint
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from tqdm import tqdm
from sklearn.model_selection import train_test_split
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img, augment=True, dim=256):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img,DIM=dim)
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 img = tf.reshape(img, [dim,dim, 3])
 return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )<create_dataframe>
train_df_origin = pd.read_csv('.. /input/severstal-steel-defect-detection/train.csv')
 train_df_origin.shape
def get_dataset(files, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True, batch_size=16, dim=256):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, imgname_or_label:(prepare_image(img, augment=augment, dim=dim),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(batch_size * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds<choose_model_class>
train_df = pd.DataFrame(columns=['ImageId_ClassId','EncodedPixels'])
 for i in tqdm(range(len(train_fns_split))):
for j in range(4):
tmp_se = pd.Series([train_fns_split[4][i]+'_{}'.format(j+1),None], index=train_df.columns)
 train_df = train_df.append(tmp_se, ignore_index=True)
 train_df.head()
EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3,
efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6]
def build_model(dim = 128, ef = 0):
inp = Input(shape =(dim, dim, 3))
base = EFNS[ef](
 input_shape =(dim, dim, 3),
weights = "imagenet",
include_top = False,
drop_connect_rate = 0.3 
)
 pt_depth = base.get_output_shape_at(0)[-1]
pt_features = base(inp)
 bn_features = BatchNormalization()(pt_features)
 attn_layer = Conv2D(64, kernel_size =(1, 1), padding = "same", activation = "relu" )(Dropout(0.5 )(bn_features))
attn_layer = Conv2D(16, kernel_size =(1, 1), padding = "same", activation = "relu" )(attn_layer)
 attn_layer = Conv2D(8, kernel_size =(1, 1), padding = "same", activation = "relu" )(attn_layer)
 attn_layer = Conv2D(1, kernel_size =(1, 1), padding = "valid", activation = "sigmoid" )(attn_layer)
 up_c2_w = np.ones(( 1, 1, 1, pt_depth))
up_c2 = Conv2D(
 pt_depth, kernel_size =(1, 1),
padding = "same",
activation = "linear",
use_bias = False,
weights = [up_c2_w] 
)
 up_c2.trainable = False
attn_layer = up_c2(attn_layer)
 mask_features = multiply([attn_layer, bn_features])
 gap_features = GlobalAveragePooling2D()(mask_features)
 gap_mask = GlobalAveragePooling2D()(attn_layer)
 gap = Lambda(lambda x: x[0] / x[1], name = "RescaleGAP" )([gap_features, gap_mask])
 gap_dr = Dropout(0.25 )(gap)
 dr_steps = Dropout(0.25 )(Dense(128, activation = "relu" )(gap_dr))
x = Dense(1, activation = "sigmoid" )(dr_steps)
 model = Model(inputs = inp, outputs = x)
 opt = tf.keras.optimizers.Adam(learning_rate = 0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.05)
 model.compile(optimizer = opt, loss = loss, metrics = ["AUC"])
 return model<init_hyperparams>
for i in tqdm(range(len(train_df_origin))):
imageid_classid = train_df_origin['ImageId'][i]+'_{}'.format(train_df_origin['ClassId'][i])
 idx=train_df.query('ImageId_ClassId == "{}"'.format(imageid_classid)).index[0]
train_df['EncodedPixels'][idx] = train_df_origin['EncodedPixels'][i]
train_df.head()
def get_lr_callback(batch_size=8):
lr_start = 0.000005
lr_max = 0.00000125 * REPLICAS * batch_size
lr_min = 0.000001
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.8
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback<define_variables>
train_df['ImageId'] = train_df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 train_df['ClassId'] = train_df['ImageId_ClassId'].apply(lambda x: x.split('_')[1])
 train_df['hasMask'] = ~ train_df['EncodedPixels'].isna()
print(train_df.shape)
 train_df.head()
VERBOSE = 0
DISPLAY_PLOT = True
skf = KFold(n_splits = FOLDS,shuffle = True,random_state = SEED)
 oof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = []
preds = np.zeros(( count_data_items(files_test),1))
for fold,(idxT,idxV)in enumerate(skf.split(np.arange(15))):
if DEVICE=='TPU':
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 print('
print(' 
(IMG_SIZES[fold],EFF_NETS[fold],BATCH_SIZES[fold]*REPLICAS))
files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxT])
 if INC2019[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1])
 print('
if INC2018[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2])
 print('
np.random.shuffle(files_train); print('
files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxV])
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/test*.tfrec')))
 K.clear_session()
with strategy.scope() :
model = build_model(dim=IMG_SIZES[fold],ef=EFF_NETS[fold])
 sv = tf.keras.callbacks.ModelCheckpoint(
 'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min', save_freq='epoch')
 print('Training...')
 history = model.fit(
 get_dataset(
 files_train,
augment = True,
shuffle = True,
repeat = True,
dim = IMG_SIZES[fold],
batch_size = BATCH_SIZES[fold] 
),
epochs = EPOCHS[fold],
callbacks = [sv,get_lr_callback(BATCH_SIZES[fold])],
steps_per_epoch = count_data_items(files_train)/ BATCH_SIZES[fold]//REPLICAS,
validation_data = get_dataset(
 files_valid,augment = False,
shuffle = False,
repeat = False,
dim = IMG_SIZES[fold] 
),
verbose = VERBOSE 
)
 print('Loading best model...')
 model.load_weights('fold-%i.h5'%fold)
 print('Predicting OOF with TTA...')
 ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_valid = count_data_items(files_valid); STEPS = TTA * ct_valid/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,]
oof_pred.append(np.mean(pred.reshape(( ct_valid,TTA),order='F'),axis=1))
ds_valid = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=True, return_image_names=True)
 oof_tar.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]))
oof_folds.append(np.ones_like(oof_tar[-1],dtype='int8')*fold)
 ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 oof_names.append(np.array([img_name.numpy().decode("utf-8")for img, img_name in iter(ds.unbatch())]))
print('Predicting Test with TTA...')
 ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,]
preds[:,0] += np.mean(pred.reshape(( ct_test,TTA),order='F'),axis=1)* WGTS[fold]
auc = roc_auc_score(oof_tar[-1],oof_pred[-1])
 oof_val.append(np.max(history.history['val_auc']))
print('
if DISPLAY_PLOT:
plt.figure(figsize=(15,5))
plt.plot(np.arange(EPOCHS[fold]),history.history['auc'],'-o',label='Train AUC',color='
plt.plot(np.arange(EPOCHS[fold]),history.history['val_auc'],'-o',label='Val AUC',color='
x = np.argmax(history.history['val_auc']); y = np.max(history.history['val_auc'])
 xdist = plt.xlim() [1] - plt.xlim() [0]; ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
%.2f'%y,size=14)
 plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)
 plt.legend(loc=2)
 plt2 = plt.gca().twinx()
plt2.plot(np.arange(EPOCHS[fold]),history.history['loss'],'-o',label='Train Loss',color='
plt2.plot(np.arange(EPOCHS[fold]),history.history['val_loss'],'-o',label='Val Loss',color='
x = np.argmin(history.history['val_loss']); y = np.min(history.history['val_loss'])
 ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
plt.ylabel('Loss',size=14)
 plt.title('FOLD %i - Image Size %i, EfficientNet B%i, inc2019=%i, inc2018=%i'% 
(fold+1,IMG_SIZES[fold],EFF_NETS[fold],INC2019[fold],INC2018[fold]),size=18)
 plt.legend(loc=3)
 plt.show()<save_to_csv>
mask_count_df = train_df.groupby('ImageId' ).agg(np.sum ).reset_index()
mask_count_df.sort_values('hasMask', ascending=False, inplace=True)
 print(mask_count_df.shape)
 mask_count_df.head()
oof = np.concatenate(oof_pred); true = np.concatenate(oof_tar);
names = np.concatenate(oof_names); folds = np.concatenate(oof_folds)
 auc = roc_auc_score(true,oof)
 print('Overall OOF AUC with TTA = %.3f'%auc)
 df_oof = pd.DataFrame(dict(
 image_name = names, target=true, pred = oof, fold=folds))
df_oof.to_csv('oof.csv',index=False)
 df_oof.head()<create_dataframe>
sub_df_origin = pd.read_csv('.. /input/severstal-steel-defect-detection/sample_submission.csv')
 sub_df_origin.head()
ds = get_dataset(files_test, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )<save_to_csv>
sub_df = pd.DataFrame(columns=['ImageId_ClassId','EncodedPixels'])
 for i in tqdm(range(len(sub_df_origin))):
for j in range(4):
tmp_se = pd.Series([sub_df_origin['ImageId'][i]+'_{}'.format(j+1),None], index=sub_df.columns)
 sub_df = sub_df.append(tmp_se, ignore_index=True)
 sub_df.head(9 )
submission = pd.DataFrame(dict(image_name=image_names, target=preds[:,0]))
submission = submission.sort_values('image_name')
 submission.to_csv('submission.csv', index=False)
 submission.head()<install_modules>
sub = pd.read_csv('.. /input/submain2/sub_main2.csv')
 sub_df['ImageId'] = sub_df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 test_imgs = pd.DataFrame(sub_df['ImageId'].unique() , columns=['ImageId'] )
!pip install wtfml==0.0.2
!pip install pretrainedmodels<install_modules>
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def rle2mask(mask_rle, shape=(256,1600)) :

 s = mask_rle.split()
starts, lengths = [np.asarray(x, dtype=int)for x in(s[0:][::2], s[1:][::2])]
starts -= 1
ends = starts + lengths
img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 for lo, hi in zip(starts, ends):
img[lo:hi] = 1
return img.reshape(shape ).T
!pip install efficientnet_pytorch<import_modules>
def build_masks(rles, input_shape):
depth = len(rles)
 height, width = input_shape
masks = np.zeros(( height, width, depth))
for i, rle in enumerate(rles):
if type(rle)is str:
masks[:, :, i] = rle2mask(rle,(width, height))
return masks
def build_rles(masks):
width, height, depth = masks.shape
rles = [mask2rle(masks[:, :, i])
 for i in range(depth)]
return rles
import os
import torch
import albumentations
import numpy as np
import pandas as pd
import torch.nn as nn
from sklearn import metrics
from sklearn import model_selection
from torch.nn import functional as F
from wtfml.utils import EarlyStopping
from wtfml.engine import Engine
from wtfml.data_loaders.image import ClassificationLoader
import efficientnet_pytorch
import pretrainedmodels<load_pretrained>
def dice_coef(y_true, y_pred, smooth=1):
y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = K.sum(y_true_f * y_pred_f)
 return(2.* intersection + smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 def dice_loss(y_true, y_pred):
smooth = 1.
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = y_true_f * y_pred_f
score =(2.* K.sum(intersection)+ smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 return 1.- score
def bce_dice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)+ dice_loss(y_true, y_pred )
class SEResnext50_32x4d(nn.Module):
def __init__(self, pretrained='imagenet'):
super(SEResnext50_32x4d, self ).__init__()
self.base_model = pretrainedmodels.__dict__[
"se_resnext50_32x4d"
](pretrained=None)
 if pretrained is not None:
self.base_model.load_state_dict(
 torch.load(
 ".. /input/pretrained-model-weights-pytorch/se_resnext50_32x4d-a260b3a4.pth" 
)
)
 self.l0 = nn.Linear(2048, 1)
 def forward(self, image, targets):
batch_size, _, _, _ = image.shape
x = self.base_model.features(image)
 x = F.adaptive_avg_pool2d(x, 1 ).reshape(batch_size, -1)
 out = self.l0(x)
 loss = nn.BCEWithLogitsLoss()(out, targets.view(-1, 1 ).type_as(x))
return out, loss<choose_model_class>
class DataGenerator(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, df, target_df=None, mode='fit',
base_path='.. /input/severstal-steel-defect-detection/train_images',
batch_size=32, dim=(256, 1600), n_channels=1,
n_classes=4, random_state=2019, shuffle=True):
self.dim = dim
self.batch_size = batch_size
self.df = df
self.mode = mode
self.base_path = base_path
self.target_df = target_df
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.random_state = random_state
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
list_IDs_batch = [self.list_IDs[k] for k in indexes]
X = self.__generate_X(list_IDs_batch)
 if self.mode == 'fit':
y = self.__generate_y(list_IDs_batch)
 return X, y
elif self.mode == 'predict':
return X
else:
raise AttributeError('The mode parameter should be set to "fit" or "predict".')
 def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.seed(self.random_state)
 np.random.shuffle(self.indexes)
 def __generate_X(self, list_IDs_batch):
'Generates data containing batch_size samples'
X = np.empty(( self.batch_size, *self.dim, self.n_channels))
for i, ID in enumerate(list_IDs_batch):
im_name = self.df['ImageId'].iloc[ID]
img_path = f"{self.base_path}/{im_name}"
img = self.__load_grayscale(img_path)
 X[i,] = img
return X
def __generate_y(self, list_IDs_batch):
y = np.empty(( self.batch_size, *self.dim, self.n_classes), dtype=int)
 for i, ID in enumerate(list_IDs_batch):
im_name = self.df['ImageId'].iloc[ID]
image_df = self.target_df[self.target_df['ImageId'] == im_name]
rles = image_df['EncodedPixels'].values
masks = build_masks(rles, input_shape=self.dim)
 y[i, ] = masks
return y
def __load_grayscale(self, img_path):
img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)
 img = img.astype(np.float32)/ 255.
 img = np.expand_dims(img, axis=-1)
 return img
def __load_rgb(self, img_path):
img = cv2.imread(img_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img = img.astype(np.float32)/ 255.
 return img
class EfficientNet(nn.Module):
def __init__(self):
super(EfficientNet, self ).__init__()
self.base_model = efficientnet_pytorch.EfficientNet.from_pretrained(
 'efficientnet-b4' 
)
 self.base_model._fc = nn.Linear(
 in_features=1792,
out_features=1,
bias=True 
)
 def forward(self, image, targets):
out = self.base_model(image)
 loss = nn.BCEWithLogitsLoss()(out, targets.view(-1, 1 ).type_as(out))
return out, loss<save_to_csv>
BATCH_SIZE = 16
train_idx, val_idx = train_test_split(
 mask_count_df.index, random_state=2019, test_size=0.15 
)
 train_generator = DataGenerator(
 train_idx,
df=mask_count_df,
target_df=train_df,
batch_size=BATCH_SIZE,
n_classes=4 
)
 val_generator = DataGenerator(
 val_idx,
df=mask_count_df,
target_df=train_df,
batch_size=BATCH_SIZE,
n_classes=4
)
df = pd.read_csv(".. /input/siim-isic-melanoma-classification/train.csv")
 df["kfold"] = -1
df = df.sample(frac=1 ).reset_index(drop=True)
 y = df.target.values
kf = model_selection.StratifiedKFold(n_splits=5)
 for f,(t_, v_)in enumerate(kf.split(X=df, y=y)) :
df.loc[v_, 'kfold'] = f
df.to_csv("train_folds.csv", index=False )<load_from_csv>
def build_model(input_shape):
inputs = Input(input_shape)
 c1 = Conv2D(8,(3, 3), activation='elu', padding='same' )(inputs)
 c1 = Conv2D(8,(3, 3), activation='elu', padding='same' )(c1)
 p1 = MaxPooling2D(( 2, 2))(c1)
 c2 = Conv2D(16,(3, 3), activation='elu', padding='same' )(p1)
 c2 = Conv2D(16,(3, 3), activation='elu', padding='same' )(c2)
 p2 = MaxPooling2D(( 2, 2))(c2)
 c3 = Conv2D(32,(3, 3), activation='elu', padding='same' )(p2)
 c3 = Conv2D(32,(3, 3), activation='elu', padding='same' )(c3)
 p3 = MaxPooling2D(( 2, 2))(c3)
 c4 = Conv2D(64,(3, 3), activation='elu', padding='same' )(p3)
 c4 = Conv2D(64,(3, 3), activation='elu', padding='same' )(c4)
 p4 = MaxPooling2D(pool_size=(2, 2))(c4)
 c5 = Conv2D(64,(3, 3), activation='elu', padding='same' )(p4)
 c5 = Conv2D(64,(3, 3), activation='elu', padding='same' )(c5)
 p5 = MaxPooling2D(pool_size=(2, 2))(c5)
 c55 = Conv2D(128,(3, 3), activation='elu', padding='same' )(p5)
 c55 = Conv2D(128,(3, 3), activation='elu', padding='same' )(c55)
 u6 = Conv2DTranspose(64,(2, 2), strides=(2, 2), padding='same' )(c55)
 u6 = concatenate([u6, c5])
 c6 = Conv2D(64,(3, 3), activation='elu', padding='same' )(u6)
 c6 = Conv2D(64,(3, 3), activation='elu', padding='same' )(c6)
 u71 = Conv2DTranspose(32,(2, 2), strides=(2, 2), padding='same' )(c6)
 u71 = concatenate([u71, c4])
 c71 = Conv2D(32,(3, 3), activation='elu', padding='same' )(u71)
 c61 = Conv2D(32,(3, 3), activation='elu', padding='same' )(c71)
 u7 = Conv2DTranspose(32,(2, 2), strides=(2, 2), padding='same' )(c61)
 u7 = concatenate([u7, c3])
 c7 = Conv2D(32,(3, 3), activation='elu', padding='same' )(u7)
 c7 = Conv2D(32,(3, 3), activation='elu', padding='same' )(c7)
 u8 = Conv2DTranspose(16,(2, 2), strides=(2, 2), padding='same' )(c7)
 u8 = concatenate([u8, c2])
 c8 = Conv2D(16,(3, 3), activation='elu', padding='same' )(u8)
 c8 = Conv2D(16,(3, 3), activation='elu', padding='same' )(c8)
 u9 = Conv2DTranspose(8,(2, 2), strides=(2, 2), padding='same' )(c8)
 u9 = concatenate([u9, c1], axis=3)
 c9 = Conv2D(8,(3, 3), activation='elu', padding='same' )(u9)
 c9 = Conv2D(8,(3, 3), activation='elu', padding='same' )(c9)
 outputs = Conv2D(4,(1, 1), activation='sigmoid' )(c9)
 model = Model(inputs=[inputs], outputs=[outputs])
 model.compile(optimizer='adam', loss=bce_dice_loss, metrics=[dice_coef])
 return model
def train(fold):
training_data_path = ".. /input/siic-isic-224x224-images/train/"
df = pd.read_csv("/kaggle/working/train_folds.csv")
 device = "cuda"
epochs = 50
train_bs = 32
valid_bs = 16
df_train = df[df.kfold != fold].reset_index(drop=True)
 df_valid = df[df.kfold == fold].reset_index(drop=True)
 model = EfficientNet()
model.to(device)
 mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225)
 train_aug = albumentations.Compose(
 [
albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True),
albumentations.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=15),
albumentations.Flip(p=0.5)
 ] 
)
 valid_aug = albumentations.Compose(
 [
albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True)
 ] 
)
 train_images = df_train.image_name.values.tolist()
train_images = [os.path.join(training_data_path, i + ".png")for i in train_images]
train_targets = df_train.target.values
valid_images = df_valid.image_name.values.tolist()
valid_images = [os.path.join(training_data_path, i + ".png")for i in valid_images]
valid_targets = df_valid.target.values
train_dataset = ClassificationLoader(
 image_paths=train_images,
targets=train_targets,
resize=None,
augmentations=train_aug, 
)
 train_loader = torch.utils.data.DataLoader(
 train_dataset, batch_size=train_bs, shuffle=True, num_workers=4 
)
 valid_dataset = ClassificationLoader(
 image_paths=valid_images,
targets=valid_targets,
resize=None,
augmentations=valid_aug, 
)
 valid_loader = torch.utils.data.DataLoader(
 valid_dataset, batch_size=valid_bs, shuffle=False, num_workers=4 
)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-4)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
 optimizer,
patience=3,
threshold=0.001,
mode="max" 
)
 es = EarlyStopping(patience=5, mode="max")
 for epoch in range(epochs):
train_loss = Engine.train(train_loader, model, optimizer, device=device)
 predictions, valid_loss = Engine.evaluate(
 valid_loader, model, device=device 
)
 predictions = np.vstack(( predictions)).ravel()
auc = metrics.roc_auc_score(valid_targets, predictions)
 print(f"Epoch = {epoch}, AUC = {auc}")
 scheduler.step(auc)
 es(auc, model, model_path=f"model_fold_{fold}.bin")
 if es.early_stop:
print("Early stopping")
 break<load_from_csv>
model.load_weights('.. /input/modelunet3/model.h5')
 test_df = []
for i in range(0, test_imgs.shape[0], 500):
batch_idx = list(
 range(i, min(test_imgs.shape[0], i + 500)) 
)
 test_generator = DataGenerator(
 batch_idx,
df=test_imgs,
shuffle=False,
mode='predict',
base_path='.. /input/severstal-steel-defect-detection/test_images',
target_df=sub_df,
batch_size=1,
n_classes=4 
)
 batch_pred_masks = model.predict_generator(
 test_generator,
workers=1,
verbose=1,
use_multiprocessing=False 
)
 for j, b in tqdm(enumerate(batch_idx)) :
filename = test_imgs['ImageId'].iloc[b]
image_df = sub_df[sub_df['ImageId'] == filename].copy()
pred_masks = batch_pred_masks[j, ].round().astype(int)
 pred_rles = build_rles(pred_masks)
 image_df['EncodedPixels'] = pred_rles
test_df.append(image_df )
def predict(fold):
test_data_path = ".. /input/siic-isic-224x224-images/test/"
df = pd.read_csv(".. /input/siim-isic-melanoma-classification/test.csv")
 device = "cuda"
model_path=f"model_fold_{fold}.bin"
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225)
 aug = albumentations.Compose(
 [
albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True)
 ] 
)
 images = df.image_name.values.tolist()
images = [os.path.join(test_data_path, i + ".png")for i in images]
targets = np.zeros(len(images))
test_dataset = ClassificationLoader(
 image_paths=images,
targets=targets,
resize=None,
augmentations=aug, 
)
 test_loader = torch.utils.data.DataLoader(
 test_dataset, batch_size=16, shuffle=False, num_workers=4 
)
 model = EfficientNet()
model.load_state_dict(torch.load(model_path))
model.to(device)
 predictions = Engine.predict(test_loader, model, device=device)
 predictions = np.vstack(( predictions)).ravel()
return predictions<train_model>
subb = pd.concat(test_df )
train(0)
 train(1)
 train(2)
 train(3)
 train(4 )<predict_on_test>
sub.to_csv('submission.csv', index=False )
p1 = predict(0)
 p2 = predict(1)
 p3 = predict(2)
 p4 = predict(3)
 p5 = predict(4 )<save_to_csv>
sub.to_csv('submission.csv', index=False )