Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<count_values>
!pip install tensorflow~=2.2.0 tensorflow_gcs_config~=2.2.0
data[cols_with_missing_values].dtypes.value_counts()<feature_engineering>
!pip install -q efficientnet
!pip install pandas_summary
!pip install tensorflow-addons
for col in cols_with_missing_values:
data[col + '_was_missing'] = data[col].isnull()<count_values>
import os
import re
import numpy as np
import pandas as pd
import random
import math
import matplotlib.pyplot as plt
from sklearn import metrics
from sklearn.model_selection import KFold, StratifiedKFold
import tensorflow as tf
from kaggle_datasets import KaggleDatasets
import efficientnet.tfkeras as efn
from tensorflow.keras import backend as K
import tensorflow_addons as tfa
for col in cols_with_missing_values:
data[col] = data[col].fillna(data[col].value_counts().index[0] )<define_variables>
def seed_all(seed):

 random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = str(seed)
 os.environ['TF_KERAS'] = str(seed)
 tf.random.set_seed(seed)
 seed_all(42 )
cols_for_one_hot_encdoing = []
cols_for_scale = []
for col in data.columns:
if data[col].dtypes == 'object':
cols_for_one_hot_encdoing.append(col)
 elif data[col].dtypes != 'bool':
cols_for_scale.append(col )<categorify>
DEVICE = 'TPU'
MIXED_PRECISION = True
XLA_ACCELERATE = True
if DEVICE == 'TPU':
print('Connecting to TPU...')
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print('Could not connect to TPU')
 tpu = None
if tpu:
try:
print('initializing TPU...')
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print('TPU initialized')
 if MIXED_PRECISION:
policy = tf.keras.mixed_precision.experimental.Policy('mixed_bfloat16')
 mixed_precision.set_policy(policy)
 print('Mixed precision enabled')
 if XLA_ACCELERATE:
tf.config.optimizer.set_jit(True)
 print('Accelerated Linear Algebra enabled')
 except _:
print('failed to initialize TPU')
 else:
DEVICE = 'GPU'
if DEVICE != 'TPU':
print('Using default strategy for CPU and single GPU')
 strategy = tf.distribute.get_strategy()
if DEVICE == 'GPU':
print('Num GPUs Available: ', len(tf.config.experimental.list_physical_devices('GPU')))
 if MIXED_PRECISION:
policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')
 mixed_precision.set_policy(policy)
 print('Mixed precision enabled')
 if XLA_ACCELERATE:
tf.config.optimizer.set_jit(True)
 print('Accelerated Linear Algebra enabled')
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )
data = pd.get_dummies(data, columns = cols_for_one_hot_encdoing, prefix = cols_for_one_hot_encdoing )<data_type_conversions>
CFG = dict(
 epochs = 30,
batch_size = 128,
lr = 0.00032,
inp_size = 256,
eff_B = 0,
sprinkles_mode = 'normal',
sprinkles_prob = 1,
num_holes = 5,
side_length = 256//10 
)
<prepare_x_and_y>
BASEPATH = '.. /input/siim-isic-melanoma-classification'
df_train = pd.read_csv(os.path.join(BASEPATH, 'train.csv'))
df_test = pd.read_csv(os.path.join(BASEPATH, 'test.csv'))
df_sub = pd.read_csv(os.path.join(BASEPATH, 'sample_submission.csv'))
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(CFG['inp_size'],CFG['inp_size']))
GCS_PATHE = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(CFG['inp_size'],CFG['inp_size']))
training_files = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')
 training_files_2018 = tf.io.gfile.glob(GCS_PATHE + '/train%.2i*.tfrec'%(2*x)for x in range(15))
training_files += training_files_2018
test_files = tf.io.gfile.glob(GCS_PATH + '/test*.tfrec' )
X_train = data.loc[train_data.index]
X_test = data.loc[test_data.index]<split>
def make_mask(num_holes,side_length,rows, cols, num_channels):

 row_range = tf.tile(tf.range(rows)[..., tf.newaxis], [1, num_holes])
 col_range = tf.tile(tf.range(cols)[..., tf.newaxis], [1, num_holes])
 r_idx = tf.random.uniform([num_holes], minval=0, maxval=rows-1,
dtype=tf.int32)
 c_idx = tf.random.uniform([num_holes], minval=0, maxval=cols-1,
dtype=tf.int32)
 r1 = tf.clip_by_value(r_idx - side_length // 2, 0, rows)
 r2 = tf.clip_by_value(r_idx + side_length // 2, 0, rows)
 c1 = tf.clip_by_value(c_idx - side_length // 2, 0, cols)
 c2 = tf.clip_by_value(c_idx + side_length // 2, 0, cols)
 row_mask =(row_range > r1)&(row_range < r2)
 col_mask =(col_range > c1)&(col_range < c2)
 mask = row_mask[:, tf.newaxis] & col_mask
mask = tf.reduce_any(mask, axis=-1)
 mask = mask[..., tf.newaxis]
mask = tf.tile(mask, [1, 1, num_channels])
 return mask
def sprinkles(image, cfg = CFG):

 num_holes = cfg['num_holes']
side_length = cfg['side_length']
mode = cfg['sprinkles_mode']
PROBABILITY = cfg['sprinkles_prob']
RandProb = tf.cast(tf.random.uniform([],0,1)< PROBABILITY, tf.int32)
 if(RandProb == 0)|(num_holes == 0): return image
img_shape = tf.shape(image)
 if mode is 'normal':
rejected = tf.zeros_like(image)
 elif mode is 'salt_pepper':
num_holes = num_holes // 2
rejected_high = tf.ones_like(image)
 rejected_low = tf.zeros_like(image)
 elif mode is 'gaussian':
rejected = tf.random.normal(img_shape, dtype=tf.float32)
 else:
raise ValueError(f'Unknown mode "{mode}" given.')
 rows = img_shape[0]
cols = img_shape[1]
num_channels = img_shape[-1]
if mode is 'salt_pepper':
mask1 = make_mask(num_holes,side_length,rows, cols, num_channels)
 mask2 = make_mask(num_holes,side_length,rows, cols, num_channels)
 filtered_image = tf.where(mask1, rejected_high, image)
 filtered_image = tf.where(mask2, rejected_low, filtered_image)
 else:
mask = make_mask(num_holes,side_length,rows, cols, num_channels)
 filtered_image = tf.where(mask, rejected, image)
 return filtered_image
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size = 0.1, random_state=2020 )<train_model>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
model_XGB = XGBRegressor(objective='reg:squarederror', n_estimators=1000)
 model_XGB.fit(X_train, y_train)
 y_val_pred = model_XGB.predict(X_val)
 np.sqrt(mean_squared_log_error(y_val, y_val_pred))<predict_on_test>
def transform(image, label):
DIM = CFG['inp_size']
XDIM = DIM%2
if 0.5 > tf.random.uniform([1], minval = 0, maxval = 1):
rot = 15.* tf.random.normal([1],dtype='float32')
 else:
rot = 180.* tf.random.normal([1],dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image['img_inp'],tf.transpose(idx3))
return {'img_inp': tf.reshape(d,[DIM,DIM,3]), 'meta_inp': image['meta_inp']}, label
y_test = model_XGB.predict(X_test ).astype(int )<save_to_csv>
def decode_image(image_data):

 image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [CFG['inp_size'], CFG['inp_size'], 3])
 return image
output = pd.DataFrame({'Id': samp_subm.index,
'SalePrice': y_test})
 output.to_csv('submission.csv', index=False )<set_options>
def data_augment(data, label):
data['img_inp'] = tf.image.random_flip_left_right(data['img_inp'])
 data['img_inp'] = tf.image.random_flip_up_down(data['img_inp'])
 data['img_inp'] = tf.image.random_hue(data['img_inp'], 0.01)
 data['img_inp'] = tf.image.random_saturation(data['img_inp'], 0.7, 1.3)
 data['img_inp'] = tf.image.random_contrast(data['img_inp'], 0.8, 1.2)
 data['img_inp'] = tf.image.random_brightness(data['img_inp'], 0.1)
 data['img_inp'] = sprinkles(data['img_inp'])
 return data, label
%matplotlib inline
py.init_notebook_mode(connected=True)
 @contextmanager
def timer(title):
t0 = time.time()
yield
print("{} - done in {:.0f}s".format(title, time.time() - t0))
warnings.filterwarnings('ignore' )<load_from_csv>
def read_labeled_tfrecord(example):

 LABELED_TFREC_FORMAT = {
'image': tf.io.FixedLenFeature([], tf.string),
'target': tf.io.FixedLenFeature([], tf.int64),
'age_approx': tf.io.FixedLenFeature([], tf.int64),
'sex': tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['target'], tf.int32)
 data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, label, data
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<categorify>
def read_unlabeled_tfrecord(example):

 UNLABELED_TFREC_FORMAT = {
'image': tf.io.FixedLenFeature([], tf.string),
'image_name': tf.io.FixedLenFeature([], tf.string),
'age_approx': tf.io.FixedLenFeature([], tf.int64),
'sex': tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 image_name = example['image_name']
data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, image_name, data
train['Type'] = 'train'
test['Type'] = 'test'
data = train.append(test)
 data.shape<feature_engineering>
def read_complete_tfrecord(example):

 LABELED_TFREC_FORMAT = {
'image': tf.io.FixedLenFeature([], tf.string),
'image_name': tf.io.FixedLenFeature([], tf.string),
'target': tf.io.FixedLenFeature([], tf.int64),
'age_approx': tf.io.FixedLenFeature([], tf.int64),
'sex': tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 image_name = example['image_name']
target = tf.cast(example['target'], tf.int32)
 data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, image_name, target, data
data['Title'] = data['Name']
for name_string in data['Name']:
data['Title'] = data['Name'].str.extract('([A-Za-z]+)\.', expand=True )<define_variables>
def load_dataset(filenames, labeled = True, ordered = False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO)
 return dataset
def load_complete_dataset(filenames):
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.map(read_complete_tfrecord, num_parallel_calls = AUTO)
 return dataset
mapping = {'Mlle': 'Miss',
'Ms': 'Miss',
'Mme': 'Mrs',
'Major': 'Other',
'Col': 'Other',
'Dr' : 'Other',
'Rev' : 'Other',
'Capt': 'Other',
'Jonkheer': 'Royal',
'Sir': 'Royal',
'Lady': 'Royal',
'Don': 'Royal',
'Countess': 'Royal',
'Dona': 'Royal'}
data.replace({'Title': mapping}, inplace=True)
 titles = ['Miss', 'Mr', 'Mrs', 'Royal', 'Other', 'Master']<feature_engineering>
def training_input(image, label, data):

 anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'img_inp': image, 'meta_inp': tabular}, label
for title in titles:
age_to_impute = data.groupby('Title')['Age'].median() [titles.index(title)]
data.loc[(data['Age'].isnull())&(data['Title'] == title), 'Age'] = age_to_impute<feature_engineering>
def test_input(image, image_name, data):

 anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'img_inp': image, 'meta_inp': tabular}, image_name
data['Family_Size'] = data['Parch'] + data['SibSp'] + 1
data.loc[:,'FsizeD']='Alone'
data.loc[(data['Family_Size']>1),'FsizeD']='Small'
data.loc[(data['Family_Size']>4),'FsizeD']='Big'<filter>
def validation_input(image, image_name, target, data):

 anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'img_inp': image, 'meta_inp': tabular}, image_name, target
data[data["Fare"].isnull() ]<data_type_conversions>
def get_training_dataset(filenames, labeled = True, ordered = False):

 dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(training_input, num_parallel_calls = AUTO)
 dataset = dataset.map(data_augment, num_parallel_calls = AUTO)
 dataset = dataset.map(transform, num_parallel_calls = AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(CFG['batch_size'])
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(filenames, labeled = True, ordered = True):

 dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(training_input, num_parallel_calls = AUTO)
 dataset = dataset.batch(CFG['batch_size'])
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(filenames, labeled = False, ordered = True):

 dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(test_input, num_parallel_calls = AUTO)
 dataset = dataset.batch(CFG['batch_size'])
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_complete_dataset(filenames):
dataset = load_complete_dataset(filenames)
 dataset = dataset.map(validation_input, num_parallel_calls = AUTO)
 dataset = dataset.batch(CFG['batch_size'])
 dataset = dataset.prefetch(AUTO)
 return dataset
fa = data[data["Pclass"]==3]
data['Fare'].fillna(fa['Fare'].median() , inplace = True )<feature_engineering>
NUM_TRAINING_IMAGES = int(count_data_items(training_files)* 0.8)
 NUM_VALIDATION_IMAGES = int(count_data_items(training_files)* 0.2)
 NUM_TEST_IMAGES = count_data_items(test_files)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // CFG['batch_size']
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))
data.loc[:,'Child']=1
data.loc[(data['Age']>=18),'Child']=0<feature_engineering>
class IncreaseSprinklesHoles(tf.keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs={}):

 if epoch <= 10:
CFG['num_holes'] = epoch + 5
CFG['side_length'] =(CFG['inp_size'] // 10)+(epoch // 2)
 if epoch >= 10:
CFG['num_holes'] = epoch + 5
CFG['side_length'] =(CFG['inp_size'] // 5)+(epoch // 2)
 if epoch >= 15:
CFG['num_holes'] = epoch + 5
CFG['side_length'] =(CFG['inp_size'] // 2)+(epoch // 2)
 sprinkles_cb = IncreaseSprinklesHoles()
data['Last_Name'] = data['Name'].apply(lambda x: str.split(x, ",")[0])
 DEFAULT_SURVIVAL_VALUE = 0.5
data['Family_Survival'] = DEFAULT_SURVIVAL_VALUE
for grp, grp_df in data[['Survived','Name', 'Last_Name', 'Fare', 'Ticket', 'PassengerId',
'SibSp', 'Parch', 'Age', 'Cabin']].groupby(['Last_Name', 'Fare']):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
data.loc[data['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
data.loc[data['PassengerId'] == passID, 'Family_Survival'] = 0
print("Number of passengers with family survival information:",
data.loc[data['Family_Survival']!=0.5].shape[0] )<groupby>
def get_model() :

 with strategy.scope() :
img_inp = tf.keras.layers.Input(shape =(CFG['inp_size'], CFG['inp_size'], 3), name = 'img_inp')
 meta_inp = tf.keras.layers.Input(shape =(9), name = 'meta_inp')
 effs = [0,1,2,3,4,5,6,7]
eff = effs[CFG['eff_B']]
constructor = getattr(efn, f'EfficientNetB{eff}')
 efnetb = constructor(weights = 'noisy-student', include_top = False)
 pt_depth = efnetb.get_output_shape_at(0)[-1]
pt_features = efnetb(img_inp)
 bn_features = tf.keras.layers.BatchNormalization()(pt_features)
 attn_layer = tf.keras.layers.Conv2D(64, kernel_size =(1, 1), padding = 'same', activation = 'swish' )(tf.keras.layers.Dropout(0.5 )(bn_features))
attn_layer = tf.keras.layers.Conv2D(16, kernel_size =(1, 1), padding = 'same', activation = 'swish' )(attn_layer)
 attn_layer = tf.keras.layers.Conv2D(8, kernel_size =(1,1), padding = 'same', activation = 'swish' )(attn_layer)
 attn_layer = tf.keras.layers.Conv2D(1, kernel_size =(1, 1), padding = 'valid', activation = 'sigmoid' )(attn_layer)
 up_c2_w = np.ones(( 1, 1, 1, pt_depth))
up_c2 = tf.keras.layers.Conv2D(pt_depth, kernel_size =(1, 1), padding = 'same', activation = 'linear', use_bias = False, weights = [up_c2_w])
 up_c2.trainable = False
attn_layer = up_c2(attn_layer)
 mask_features = tf.keras.layers.multiply([attn_layer, bn_features])
 gap_features = tf.keras.layers.GlobalAveragePooling2D()(mask_features)
 gap_mask = tf.keras.layers.GlobalAveragePooling2D()(attn_layer)
 gap = tf.keras.layers.Lambda(lambda x: x[0] / x[1], name = 'RescaleGAP' )([gap_features, gap_mask])
 gap_dr = tf.keras.layers.Dropout(0.5 )(gap)
 dr_steps = tf.keras.layers.Dropout(0.25 )(tf.keras.layers.Dense(128, activation = 'swish' )(gap_dr))
meta_layer = tf.keras.layers.Dense(16 )(meta_inp)
 meta_layer = tf.keras.layers.BatchNormalization()(meta_layer)
 meta_layer = tf.keras.layers.Activation('swish' )(meta_layer)
 meta_layer = tf.keras.layers.Dropout(0.2 )(meta_layer)
 meta_layer = tf.keras.layers.Dense(8 )(meta_inp)
 meta_layer = tf.keras.layers.BatchNormalization()(meta_layer)
 meta_layer = tf.keras.layers.Activation('swish' )(meta_layer)
 meta_layer = tf.keras.layers.Dropout(0.2 )(meta_layer)
 concat = tf.keras.layers.concatenate([dr_steps, meta_layer])
 concat = tf.keras.layers.BatchNormalization()(concat)
 concat = tf.keras.layers.Dense(512, activation = 'swish' )(concat)
 concat = tf.keras.layers.Dropout(0.15 )(concat)
 output = tf.keras.layers.Dense(1, activation = 'sigmoid',dtype='float32' )(concat)
 model = tf.keras.models.Model(inputs = [img_inp, meta_inp], outputs = [output])
 opt = tf.keras.optimizers.Adam(learning_rate = CFG['lr'])
 model.compile(
 optimizer = opt,
loss = [tfa.losses.SigmoidFocalCrossEntropy(reduction=tf.keras.losses.Reduction.AUTO,gamma = 2.0, alpha = 0.90)],
metrics = [tf.keras.metrics.AUC() ] 
)
 return model
def train_validate_predict(df_sub, folds = 3):

 models = []
oof_image_name = []
oof_target = []
oof_prediction = []
kfold = KFold(folds, shuffle = True, random_state = 42)
 for fold,(trn_ind, val_ind)in enumerate(kfold.split(training_files)) :
print('
')
 print('-'*50)
 print(f'Training fold {fold + 1}')
 CFG['num_holes'] = 5
CFG['side_length'] = CFG['inp_size']//10
train_dataset = get_training_dataset([training_files[x] for x in trn_ind], labeled = True, ordered = False)
 val_dataset = get_validation_dataset([training_files[x] for x in val_ind], labeled = True, ordered = True)
 K.clear_session()
model = get_model()
early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_auc', mode = 'max', patience = 5,
verbose = 2, min_delta = 0.00025, restore_best_weights = True)
 cb_lr_schedule = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_auc', factor = 0.6, patience = 2 , verbose = 2, min_delta = 0.0005, min_lr=0.000001, mode = 'max')
 history = model.fit(train_dataset,
steps_per_epoch = STEPS_PER_EPOCH,
epochs = CFG['epochs'],
callbacks = [sprinkles_cb, cb_lr_schedule, early_stopping],
validation_data = val_dataset,
verbose = 1)
 models.append(model)
 number_of_files = count_data_items([training_files[x] for x in val_ind])
 dataset = get_complete_dataset([training_files[x] for x in val_ind])
 image_name = dataset.map(lambda image, image_name, target: image_name ).unbatch()
image_name = next(iter(image_name.batch(number_of_files)) ).numpy().astype('U')
 target = dataset.map(lambda image, image_name, target: target ).unbatch()
target = next(iter(target.batch(number_of_files)) ).numpy()
image = dataset.map(lambda image, image_name, target: image)
 probabilities = model.predict(image)
 oof_image_name.extend(list(image_name))
oof_target.extend(list(target))
oof_prediction.extend(list(np.concatenate(probabilities)))
 print('
')
 print('-'*50)
 oof_df = pd.DataFrame({'image_name': oof_image_name, 'target': oof_target, 'predictions': oof_prediction})
 oof_df.to_csv('OOF_EfficientNetB%i_%i.csv'%(CFG['eff_B'],CFG['inp_size']), index = False)
 test_ds = get_test_dataset(test_files, labeled = False, ordered = True)
 test_images_ds = test_ds.map(lambda image, image_name: image)
 print('Computing predictions...')
 probabilities = np.average([np.concatenate(models[i].predict(test_images_ds)) for i in range(folds)], axis = 0)
 print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, image_name: image_name ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 pred_df = pd.DataFrame({'image_name': test_ids, 'target': probabilities})
 df_sub.drop('target', inplace = True, axis = 1)
 df_sub = df_sub.merge(pred_df, on = 'image_name')
 df_sub.to_csv('sub_EfficientNetB%i_%i.csv'%(CFG['eff_B'],CFG['inp_size']), index = False)
 return oof_target, oof_prediction
oof_target, oof_prediction = train_validate_predict(df_sub )
<drop_column><EOS> 
roc_auc = metrics.roc_auc_score(oof_target, oof_prediction)
 print('Final OOF Roc Auc Score: ', roc_auc )