Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<define_variables>
!pip install -q efficientnet >> /dev/null
Conv2D,
BatchNormalization,
Dense,
GlobalAveragePooling2D,
Input,
Activation,
Lambda,
multiply)
 sns.set_style("darkgrid" )
corr.corrections = {
275: 5, 927: 6, 3080: 0, 3162: 3, 3485: 8, 3700: 5, 3740: 9,
4680: 1, 5215: 1, 5276: 1, 5928: 5, 6789: 3, 7026: 4, 9040: 2,
9202: 7, 9744: 3, 9814: 4, 9924: 2, 10344: 9, 11370: 8, 11746: 8,
11862: 8, 12620: 8, 12864: 9, 14579: 9, 14742: 0, 16204: 9,
16281: 5, 16452: 4, 17589: 8, 17931: 5, 18166: 1, 19351: 1,
19896: 8, 20052: 9, 20509: 5, 20665: 1, 20946: 3, 21388: 1,
22253: 8, 22465: 4, 22527: 1, 22632: 8, 22823: 4, 23874: 1,
24015: 4, 24155: 4, 24662: 8, 25527: 6, 25915: 8, 26913: 3,
27372: 3, 27402: 2
}
corr.show_corrections(9, figsize=(14.5, 11))<save_to_csv>
DEVICE = "TPU"
SEED = 42
FOLDS = 5
IMG_SIZES = [384,384,384,384,384]
INC2019 = [0,0,0,0,0]
INC2018 = [1,1,1,1,1]
BATCH_SIZES = [32]*FOLDS
EPOCHS = [12]*FOLDS
EFF_NETS = [6,6,6,6,6]
WGTS = [1/FOLDS]*FOLDS
TTA = 11
preds = corr.corrected_labels()
submission = pd.DataFrame(
 preds, columns=('Label',),
index=pd.Index(( 1 + int(p.stem)for p in data.test_ds.x.items),
name='ImageId')
 ).sort_index()
submission.to_csv('submission.csv')
 submission.head(10 )<import_modules>
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )
! echo 64e2b22ef2bf4e7f8b179c497a81aeea11761ce1e242083a44c42a20c8a52c65 \
submission.csv | sha256sum -c<load_from_csv>
GCS_PATH = [None]*FOLDS; GCS_PATH2 = [None]*FOLDS
for i,k in enumerate(IMG_SIZES):
GCS_PATH[i] = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(k,k))
GCS_PATH2[i] = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(k,k))
files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/train*.tfrec')))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec')) )
class Data_Clean(object):
def __init__(self, trainPath, testPath):
self.train = pd.read_csv(trainPath)
 self.test = pd.read_csv(testPath)
 self.data_assemble = []
self.column = self.train.columns.drop('Image')
 def seperate_data(self):
for column_name in self.train.columns.drop('Image'):
data = self.train[[column_name, 'Image']].dropna()
self.data_assemble.append([data[column_name], data['Image']])
 self.test = self.test['Image']
def reshape_image(self):
for data in self.data_assemble:
data[1] = data[1].apply(lambda im: np.fromstring(im, sep=' ', dtype=float))
values = data[1].values / 255.0
values = np.vstack(values ).reshape(-1, 96, 96, 1)
 data[1] = values
self.test = self.test.apply(lambda img: np.fromstring(img, sep=' ', dtype=float))
self.test = self.test.values / 255.0
self.test = np.vstack(self.test ).reshape(-1, 96, 96, 1)
 def run(self):
self.seperate_data()
self.reshape_image()
return self.data_assemble, self.test, self.column
class CNN(object):
def __init__(self, train, test, columns_list):
self.train = train
self.test = test
self.columns_list = columns_list
def define_CNN(self):
model = Sequential()
model.add(Convolution2D(filters=32, kernel_size=(3, 3), padding='same', use_bias=False, input_shape=(96, 96, 1)))
 model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(Convolution2D(filters=32, kernel_size=(3, 3), padding='same', use_bias=False))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2, 2)))
 model.add(Convolution2D(filters=64, kernel_size=(3, 3), padding='same', use_bias=False))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(Convolution2D(filters=64, kernel_size=(3, 3), padding='same', use_bias=False))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2, 2)))
 model.add(Convolution2D(filters=96, kernel_size=(3, 3), padding='same', use_bias=False))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(Convolution2D(filters=96, kernel_size=(3, 3), padding='same', use_bias=False))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2, 2)))
 model.add(Convolution2D(filters=128, kernel_size=(3, 3), padding='same', use_bias=False))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(Convolution2D(filters=128, kernel_size=(3, 3), padding='same', use_bias=False))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2, 2)))
 model.add(Convolution2D(filters=256, kernel_size=(3, 3), padding='same', use_bias=False))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(Convolution2D(filters=256, kernel_size=(3, 3), padding='same', use_bias=False))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2, 2)))
 model.add(Convolution2D(filters=512, kernel_size=(3, 3), padding='same', use_bias=False))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(Convolution2D(filters=512, kernel_size=(3, 3), padding='same', use_bias=False))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(Flatten())
 model.add(Dense(512, activation='relu'))
model.add(Dropout(0.1))
model.add(Dense(1))
self.model = model
def Adam(self, epochs, batchSize, xtrain, xvalidation, ytrain, yvalidation):
self.model.compile(optimizer='adam', loss='mean_squared_error', metrics=['mae'])
 self.model.fit(xtrain, ytrain, batch_size=batchSize, epochs=epochs, validation_data=(xvalidation, yvalidation),
verbose=0)
 def show_a_result(self, ypred, ytest):
plt.imshow(ytest.reshape(96, 96))
ypred = copy.deepcopy(ypred)
 xy = np.split(ypred, 30)
 predx, predy = [], []
for i in range(0, 30, 2):
predx.append(xy[i])
 predy.append(xy[i+1])
 plt.plot(predx, predy, 'o', color='red')
 plt.show()
def make_output(self, ypred):
pred = ypred
dataPath = '.. /input/IdLookupTable.csv'
lookid_data = pd.read_csv(dataPath)
 lookid_list = list(lookid_data['FeatureName'])
 imageID = list(lookid_data['ImageId'] - 1)
 rowid = lookid_data['RowId']
rowid = list(rowid)
 feature = []
for f in lookid_list:
feature.append(f)
 preded = []
for x, y in zip(imageID, feature):
preded.append(pred[y].loc[x])
 rowid = pd.Series(rowid, name='RowId')
 loc = pd.Series(preded, name='Location')
 submission = pd.concat([rowid, loc], axis=1)
 submission.to_csv('Utkarsh.csv', index=False)
 def run(self):
ypred = pd.DataFrame(index = [i for i in range(1783)] ,columns=self.columns_list)
 for index, data in enumerate(self.train):
label = data[0]
columns_name = self.columns_list[index]
train = data[1]
xtrain, xvalidation, ytrain, yvalidation = train_test_split(train, label, test_size=0.1, random_state=9)
 self.define_CNN()
print(columns_name, ' training started:')
 self.Adam(epochs=100, batchSize=128, xtrain=xtrain, xvalidation=xvalidation, ytrain=ytrain, yvalidation=yvalidation)
 ypred[columns_name] = self.model.predict(self.test)
 self.make_output(ypred)
 self.show_a_result(ypred.loc[159], self.test[159])
 if __name__ == '__main__':
trainPath = '.. /input/training/training.csv'
testPath = '.. /input/test/test.csv'
data_clean = Data_Clean(trainPath, testPath)
 train, test, columns_list = data_clean.run()
cnn = CNN(train, test, columns_list)
 cnn.run() 
<load_from_csv>
ROT_ = 180.0
SHR_ = 2.0
HZOOM_ = 8.0
WZOOM_ = 8.0
HSHIFT_ = 8.0
WSHIFT_ = 8.0
sample_submission = pd.read_csv(".. /input/digit-recognizer/sample_submission.csv")
 test = pd.read_csv(".. /input/digit-recognizer/test.csv")
 train = pd.read_csv(".. /input/digit-recognizer/train.csv" )<prepare_x_and_y>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, DIM=256):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )
X_train = train / 255.0
X_test = test / 255.0<data_type_conversions>
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img, augment=True, dim=256):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img,DIM=dim)
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 img = tf.reshape(img, [dim,dim, 3])
 return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )
X_train =(X_train.iloc[:,1:].values ).astype('float32')
 y_train = train['label'].astype('float32')
 X_test = X_test.values.astype('float32' )<data_type_conversions>
def get_dataset(files, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True, batch_size=16, dim=256):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, imgname_or_label:(prepare_image(img, augment=augment, dim=dim),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(batch_size * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds
mean_px = X_train.mean().astype(np.float32)
 std_px = X_train.std().astype(np.float32)
 def standardize(x):
return(x - mean_px)/std_px
[mean_px, std_px]<categorify>
EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3,
efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6]
def build_model(dim = 128, ef = 0):
inp = Input(shape =(dim, dim, 3))
base = EFNS[ef](
 input_shape =(dim, dim, 3),
weights = "imagenet",
include_top = False,
drop_connect_rate = 0.3 
)
 pt_depth = base.get_output_shape_at(0)[-1]
pt_features = base(inp)
 bn_features = BatchNormalization()(pt_features)
 attn_layer = Conv2D(64, kernel_size =(1, 1), padding = "same", activation = "relu" )(Dropout(0.5 )(bn_features))
attn_layer = Conv2D(16, kernel_size =(1, 1), padding = "same", activation = "relu" )(attn_layer)
 attn_layer = Conv2D(8, kernel_size =(1, 1), padding = "same", activation = "relu" )(attn_layer)
 attn_layer = Conv2D(1, kernel_size =(1, 1), padding = "valid", activation = "sigmoid" )(attn_layer)
 up_c2_w = np.ones(( 1, 1, 1, pt_depth))
up_c2 = Conv2D(
 pt_depth, kernel_size =(1, 1),
padding = "same",
activation = "linear",
use_bias = False,
weights = [up_c2_w] 
)
 up_c2.trainable = False
attn_layer = up_c2(attn_layer)
 mask_features = multiply([attn_layer, bn_features])
 gap_features = GlobalAveragePooling2D()(mask_features)
 gap_mask = GlobalAveragePooling2D()(attn_layer)
 gap = Lambda(lambda x: x[0] / x[1], name = "RescaleGAP" )([gap_features, gap_mask])
 gap_dr = Dropout(0.25 )(gap)
 dr_steps = Dropout(0.25 )(Dense(128, activation = "relu" )(gap_dr))
x = Dense(1, activation = "sigmoid" )(dr_steps)
 model = Model(inputs = inp, outputs = x)
 opt = tf.keras.optimizers.Adam(learning_rate = 0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.05)
 model.compile(optimizer = opt, loss = loss, metrics = ["AUC"])
 return model
print(y_train)
 y_train= to_categorical(y_train)
 num_classes = y_train.shape[1]<define_variables>
def get_lr_callback(batch_size=8):
lr_start = 0.000005
lr_max = 0.00000125 * REPLICAS * batch_size
lr_min = 0.000001
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.8
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback
seed = 43
np.random.seed(seed)
 seed<split>
VERBOSE = 0
DISPLAY_PLOT = True
skf = KFold(n_splits = FOLDS,shuffle = True,random_state = SEED)
 oof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = []
preds = np.zeros(( count_data_items(files_test),1))
for fold,(idxT,idxV)in enumerate(skf.split(np.arange(15))):
if DEVICE=='TPU':
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 print('
print(' 
(IMG_SIZES[fold],EFF_NETS[fold],BATCH_SIZES[fold]*REPLICAS))
files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxT])
 if INC2019[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1])
 print('
if INC2018[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2])
 print('
np.random.shuffle(files_train); print('
files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxV])
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/test*.tfrec')))
 K.clear_session()
with strategy.scope() :
model = build_model(dim=IMG_SIZES[fold],ef=EFF_NETS[fold])
 sv = tf.keras.callbacks.ModelCheckpoint(
 'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min', save_freq='epoch')
 print('Training...')
 history = model.fit(
 get_dataset(
 files_train,
augment = True,
shuffle = True,
repeat = True,
dim = IMG_SIZES[fold],
batch_size = BATCH_SIZES[fold] 
),
epochs = EPOCHS[fold],
callbacks = [sv,get_lr_callback(BATCH_SIZES[fold])],
steps_per_epoch = count_data_items(files_train)/ BATCH_SIZES[fold]//REPLICAS,
validation_data = get_dataset(
 files_valid,augment = False,
shuffle = False,
repeat = False,
dim = IMG_SIZES[fold] 
),
verbose = VERBOSE 
)
 print('Loading best model...')
 model.load_weights('fold-%i.h5'%fold)
 print('Predicting OOF with TTA...')
 ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_valid = count_data_items(files_valid); STEPS = TTA * ct_valid/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,]
oof_pred.append(np.mean(pred.reshape(( ct_valid,TTA),order='F'),axis=1))
ds_valid = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=True, return_image_names=True)
 oof_tar.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]))
oof_folds.append(np.ones_like(oof_tar[-1],dtype='int8')*fold)
 ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 oof_names.append(np.array([img_name.numpy().decode("utf-8")for img, img_name in iter(ds.unbatch())]))
print('Predicting Test with TTA...')
 ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,]
preds[:,0] += np.mean(pred.reshape(( ct_test,TTA),order='F'),axis=1)* WGTS[fold]
auc = roc_auc_score(oof_tar[-1],oof_pred[-1])
 oof_val.append(np.max(history.history['val_auc']))
print('
if DISPLAY_PLOT:
plt.figure(figsize=(15,5))
plt.plot(np.arange(EPOCHS[fold]),history.history['auc'],'-o',label='Train AUC',color='
plt.plot(np.arange(EPOCHS[fold]),history.history['val_auc'],'-o',label='Val AUC',color='
x = np.argmax(history.history['val_auc']); y = np.max(history.history['val_auc'])
 xdist = plt.xlim() [1] - plt.xlim() [0]; ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
%.2f'%y,size=14)
 plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)
 plt.legend(loc=2)
 plt2 = plt.gca().twinx()
plt2.plot(np.arange(EPOCHS[fold]),history.history['loss'],'-o',label='Train Loss',color='
plt2.plot(np.arange(EPOCHS[fold]),history.history['val_loss'],'-o',label='Val Loss',color='
x = np.argmin(history.history['val_loss']); y = np.min(history.history['val_loss'])
 ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
plt.ylabel('Loss',size=14)
 plt.title('FOLD %i - Image Size %i, EfficientNet B%i, inc2019=%i, inc2018=%i'% 
(fold+1,IMG_SIZES[fold],EFF_NETS[fold],INC2019[fold],INC2018[fold]),size=18)
 plt.legend(loc=3)
 plt.show()
gen = image.ImageDataGenerator()
X = X_train
y = y_train
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.10, random_state=42)
 batches = gen.flow(X_train, y_train, batch_size=64)
 val_batches=gen.flow(X_val, y_val, batch_size=64)
<define_variables>
oof = np.concatenate(oof_pred); true = np.concatenate(oof_tar);
names = np.concatenate(oof_names); folds = np.concatenate(oof_folds)
 auc = roc_auc_score(true,oof)
 print('Overall OOF AUC with TTA = %.3f'%auc)
 df_oof = pd.DataFrame(dict(
 image_name = names, target=true, pred = oof, fold=folds))
df_oof.to_csv('oof.csv',index=False)
 df_oof.head()
gen =ImageDataGenerator(rotation_range=8, width_shift_range=0.08, shear_range=0.3,
height_shift_range=0.08, zoom_range=0.08)
 batches = gen.flow(X_train, y_train, batch_size=64)
 val_batches = gen.flow(X_val, y_val, batch_size=64)
<choose_model_class>
ds = get_dataset(files_test, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )
<train_model><EOS> 
submission = pd.DataFrame(dict(image_name=image_names, target=preds[:,0]))
submission = submission.sort_values('image_name')
 submission.to_csv('submission.csv', index=False)
 submission.head()