Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<compute_test_metric>
sns.set()
py.init_notebook_mode(connected=True)
 os.listdir(".. /input/" )
print('The highest accuracy score is:', np.max(np.array(scores)) )<define_variables>
basepath = ".. /input/siim-isic-melanoma-classification/"
modelspath = ".. /input/pytorch-pretrained-image-models/"
imagestatspath = ".. /input/siimisic-melanoma-classification-image-stats/"
number_of_features = list(range(1,13))<compute_test_metric>
train_info = pd.read_csv(basepath + "train.csv")
 train_info.head()
print("Maximum accuracy score is :", max(scores_k))<define_variables>
test_info = pd.read_csv(basepath + "test.csv")
 test_info.head()
print("Optimal number of features :", np.argmax(np.array(scores_k)) + 1 )<define_variables>
missing_vals_train = train_info.isnull().sum() / train_info.shape[0]
missing_vals_train[missing_vals_train > 0].sort_values(ascending=False )
print("Optimal number of features : %d" % selector.n_features_ )<compute_test_metric>
missing_vals_test = test_info.isnull().sum() / test_info.shape[0]
missing_vals_test[missing_vals_test > 0].sort_values(ascending=False )
print("Maximum accuracy score is :", np.max(selector.grid_scores_))<define_search_space>
train_info.image_name.value_counts().max()
threshold = [0.001, 0.0025, 0.005, 0.01, 0.025 ,0.05, 0.1, 0.15]<compute_test_metric>
test_info.image_name.value_counts().max()
print("Maximum accuracy score is :", np.max(np.array(scores_sfm)) )<compute_test_metric>
train_info.patient_id.value_counts().max()
print("Optimal threshold :", threshold[np.argmax(np.array(scores_sfm)) ] )<define_search_space>
test_info.patient_id.value_counts().max()
ada_params = {'n_estimators': [90, 100, 110], 'base_estimator': [None, svm],
'learning_rate': [0.09 ,0.1, 0.11]}<choose_model_class>
200/test_info.shape[0] * 100
rs_ada = RandomizedSearchCV(ada, param_distributions= ada_params,
scoring='accuracy', cv= StratifiedKFold(7), refit=True, n_iter= 500 )<train_model>
train_info.groupby("benign_malignant" ).target.nunique()
rs_ada.fit(x_train, y_train )<train_model>
patient_ages_table_train = train_info.groupby(["patient_id", "age_approx"] ).size() / train_info.groupby("patient_id" ).size()
patient_ages_table_train = patient_ages_table_train.unstack().transpose()
patient_ages_table_test = test_info.groupby(["patient_id", "age_approx"] ).size() / test_info.groupby("patient_id" ).size()
patient_ages_table_test = patient_ages_table_test.unstack().transpose()
patient_with_known_ages_train = train_info[train_info.patient_id.isin(patient_ages_table_train.columns.values)]
sorted_patients_train = patient_with_known_ages_train.patient_id.value_counts().index.values
patient_with_known_ages_test = test_info[test_info.patient_id.isin(patient_ages_table_test.columns.values)]
sorted_patients_test = patient_with_known_ages_test.patient_id.value_counts().index.values
fig, ax = plt.subplots(2,1, figsize=(20,20))
sns.heatmap(patient_ages_table_train[sorted_patients_train], cmap="Reds", ax=ax[0], cbar=False);
ax[0].set_title("Image coverage in % per patient and age in train data");
sns.heatmap(patient_ages_table_test[sorted_patients_test], cmap="Blues", ax=ax[1], cbar=False);
ax[1].set_title("Image coverage in % per patient and age in test data");
ax[0].set_xlabel("")
 ax[1].set_xlabel("");
print('Best Parameters are:
', rs_ada.best_params_,
'
Training accuracy score is:
', rs_ada.best_score_ )<compute_test_metric>
patient_gender_train = train_info.groupby("patient_id" ).sex.unique().apply(lambda l: l[0])
 patient_gender_test = test_info.groupby("patient_id" ).sex.unique().apply(lambda l: l[0])
 train_patients = pd.DataFrame(index=patient_gender_train.index.values, data=patient_gender_train.values, columns=["sex"])
 test_patients = pd.DataFrame(index=patient_gender_test.index.values, data=patient_gender_test.values, columns=["sex"])
 train_patients.loc[:, "num_images"] = train_info.groupby("patient_id" ).size()
test_patients.loc[:, "num_images"] = test_info.groupby("patient_id" ).size()
train_patients.loc[:, "min_age"] = train_info.groupby("patient_id" ).age_approx.min()
train_patients.loc[:, "max_age"] = train_info.groupby("patient_id" ).age_approx.max()
test_patients.loc[:, "min_age"] = test_info.groupby("patient_id" ).age_approx.min()
test_patients.loc[:, "max_age"] = test_info.groupby("patient_id" ).age_approx.max()
train_patients.loc[:, "age_span"] = train_patients["max_age"] - train_patients["min_age"]
test_patients.loc[:, "age_span"] = test_patients["max_age"] - test_patients["min_age"]
train_patients.loc[:, "benign_cases"] = train_info.groupby(["patient_id", "benign_malignant"] ).size().loc[:, "benign"]
train_patients.loc[:, "malignant_cases"] = train_info.groupby(["patient_id", "benign_malignant"] ).size().loc[:, "malignant"]
train_patients["min_age_malignant"] = train_info.groupby(["patient_id", "benign_malignant"] ).age_approx.min().loc[:, "malignant"]
train_patients["max_age_malignant"] = train_info.groupby(["patient_id", "benign_malignant"] ).age_approx.max().loc[:, "malignant"]
print('Validation accuracy score is:
', rs_ada.score(x_valid, y_valid))<train_model>
train_patients.sort_values(by="malignant_cases", ascending=False ).head()
ada = ens.AdaBoostClassifier(n_estimators= 110, learning_rate= 0.09)
 ada.fit(features, target )<compute_train_metric>
example_files = os.listdir(basepath + "train/")[0:2]
example_files
y_scores_ada = ada.predict_proba(x_test)[:, 1]
ada_fpr, ada_tpr, ada_thresholds = sklearn.metrics.roc_curve(y_test, y_scores_ada)
 ada_auc = sklearn.metrics.auc(x=ada_fpr, y=ada_tpr )<compute_test_metric>
train_info["dcm_path"] = basepath + "train/" + train_info.image_name + ".dcm"
test_info["dcm_path"] = basepath + "test/" + test_info.image_name + ".dcm"
ada_acc = ada.score(x_test, y_test )<train_model>
example_dcm = pydicom.dcmread(train_info.dcm_path[2])
 example_dcm
print('Area Under Curve: {}, Accuracy: {}'.format(ada_auc, ada_acc))<choose_model_class>
train_info["image_path"] = basepath + "jpeg/train/" + train_info.image_name + ".jpg"
test_info["image_path"] = basepath + "jpeg/test/" + test_info.image_name + ".jpg"
gb = ens.GradientBoostingClassifier()<define_search_space>
test_image_stats = pd.read_csv(imagestatspath + "test_image_stats.csv")
 test_image_stats.head(1 )
threshold = np.arange(1, 10, 0.5)*1e-1<compute_train_metric>
train_image_stats_1 = pd.read_csv(imagestatspath + "train_image_stats_10000.csv")
 train_image_stats_2 = pd.read_csv(imagestatspath + "train_image_stats_20000.csv")
 train_image_stats_3 = pd.read_csv(imagestatspath + "train_image_stats_toend.csv")
 train_image_stats_4 = train_image_stats_1.append(train_image_stats_2)
 train_image_stats = train_image_stats_4.append(train_image_stats_3)
 train_image_stats.shape
scores = []
for i in threshold:
selector = sklearn.feature_selection.VarianceThreshold(threshold= i)
 selected_features = selector.fit_transform(features)
 gb.fit(selected_features, target)
 y_pred = gb.predict(features.loc[:, selector.get_support() ])
 scores.append(sklearn.metrics.accuracy_score(target, y_pred))
plt.figure()
plt.xlabel("variance threshold")
 plt.ylabel("Cross validated accuracy score")
 plt.plot(threshold, np.array(scores))<compute_test_metric>
plot_test = True
print('The highest accuracy score is obtained after execluding features whose variance is less than: ',
np.round(threshold[np.argmax(np.array(scores)) ],3))<compute_test_metric>
test_image_stats.groupby(["rows", "columns"] ).size().sort_values(ascending=False ).iloc[0:10] / test_image_stats.shape[0]
print('The highest accuracy score is:', np.max(np.array(scores)) )<define_variables>
train_image_stats.groupby(["rows", "columns"] ).size().sort_values(ascending=False ).iloc[0:10] / train_image_stats.shape[0]
number_of_features = list(range(1,13))<compute_test_metric>
examples1 = {"rows": 1080, "columns": 1920}
examples2 = {"rows": 4000, "columns": 6000}
print("Maximum accuracy score is :", max(scores_k))<define_variables>
train = pd.read_csv('.. /input/siim-isic-melanoma-classification/train.csv')
 print('Examples WITH Melanoma')
 imgs = train.loc[train.target==1].sample(10 ).image_name.values
plt.figure(figsize=(20,8))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()
print('Examples WITHOUT Melanoma')
 imgs = train.loc[train.target==0].sample(10 ).image_name.values
plt.figure(figsize=(20,8))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()
print("Optimal number of features :", np.argmax(np.array(scores_k)) + 1 )<define_variables>
!pip install -q efficientnet >> /dev/null
print("Optimal number of features : %d" % selector.n_features_ )<compute_test_metric>
import pandas as pd, numpy as np
from kaggle_datasets import KaggleDatasets
import tensorflow as tf, re, math
import tensorflow.keras.backend as K
import efficientnet.tfkeras as efn
from sklearn.model_selection import KFold
from sklearn.metrics import roc_auc_score
import matplotlib.pyplot as plt
print("Maximum accuracy score is :", np.max(selector.grid_scores_))<define_search_space>
DEVICE = "TPU"
SEED = 42
FOLDS = 5
IMG_SIZES = [384,384,384,384,384]
INC2019 = [0,0,0,0,0]
INC2018 = [1,1,1,1,1]
BATCH_SIZES = [32]*FOLDS
EPOCHS = [12]*FOLDS
EFF_NETS = [6,6,6,6,6]
WGTS = [1/FOLDS]*FOLDS
TTA = 11
threshold = [0.001, 0.0025, 0.005, 0.01, 0.025 ,0.05, 0.1, 0.15]<compute_test_metric>
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )
print("Maximum accuracy score is :", np.max(np.array(scores_sfm)) )<compute_test_metric>
GCS_PATH = [None]*FOLDS; GCS_PATH2 = [None]*FOLDS
for i,k in enumerate(IMG_SIZES):
GCS_PATH[i] = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(k,k))
GCS_PATH2[i] = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(k,k))
files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/train*.tfrec')))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec')) )
print("Optimal threshold :", threshold[np.argmax(np.array(scores_sfm)) ] )<train_on_grid>
ROT_ = 180.0
SHR_ = 2.0
HZOOM_ = 8.0
WZOOM_ = 8.0
HSHIFT_ = 8.0
WSHIFT_ = 8.0
selector = sklearn.feature_selection.SelectKBest(k= 11)
 selector.fit(features, target)
 gb_selected_features = selector.get_support()<define_search_space>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, DIM=256):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )
gb_params = {'n_estimators': [150, 160, 170], 'loss': ['deviance', 'exponential'],
'subsample': [0.7, 0.8, 0.9], 'max_features': ['auto', 'log2', None]}<choose_model_class>
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img, augment=True, dim=256):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img,DIM=dim)
 img = tf.image.random_flip_up_down(img)
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 img = tf.reshape(img, [dim,dim, 3])
 return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )
rs_gb = RandomizedSearchCV(gb, param_distributions= gb_params,
scoring='accuracy', cv= StratifiedKFold(7), refit=True, n_iter= 2000 )<train_model>
def get_dataset(files, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True, batch_size=16, dim=256):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, imgname_or_label:(prepare_image(img, augment=augment, dim=dim),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(batch_size * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds
rs_gb.fit(x_train.loc[:,gb_selected_features], y_train )<train_model>
EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3,
efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6]
def build_model(dim=128, ef=0):
inp = tf.keras.layers.Input(shape=(dim,dim,3))
base = EFNS[ef](input_shape=(dim,dim,3),weights='imagenet',include_top=False)
 x = base(inp)
 x = tf.keras.layers.GlobalAveragePooling2D()(x)
 x = tf.keras.layers.Dense(256, activation = 'relu',kernel_regularizer = tf.keras.regularizers.l2() )(x)
 x = tf.keras.layers.Dropout(0.2 )(x)
 x = tf.keras.layers.Dense(128, activation = 'relu' )(x)
 x = tf.keras.layers.Dropout(0.1 )(x)
 x = tf.keras.layers.Dense(1,activation='sigmoid' )(x)
 model = tf.keras.Model(inputs=inp,outputs=x)
 opt = tf.keras.optimizers.Adam(learning_rate=0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05)
 model.compile(optimizer=opt,loss=loss,metrics=['AUC'])
 return model
print('Best Parameters are:
', rs_gb.best_params_,
'
Training accuracy score is:
', rs_gb.best_score_ )<compute_test_metric>
def get_lr_callback(batch_size=8):
lr_start = 0.000005
lr_max = 0.00000125 * REPLICAS * batch_size
lr_min = 0.000001
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.8
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback
print('Validation accuracy score is:
',
rs_gb.score(x_valid.loc[:,gb_selected_features], y_valid))<train_on_grid>
VERBOSE = 0
DISPLAY_PLOT = True
skf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)
 oof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = []
preds = np.zeros(( count_data_items(files_test),1))
for fold,(idxT,idxV)in enumerate(skf.split(np.arange(15))):
if DEVICE=='TPU':
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 print('
print(' 
(IMG_SIZES[fold],EFF_NETS[fold],BATCH_SIZES[fold]*REPLICAS))
files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxT])
 if INC2019[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1])
 print('
if INC2018[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2])
 print('
np.random.shuffle(files_train); print('
files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxV])
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/test*.tfrec')))
 K.clear_session()
with strategy.scope() :
model = build_model(dim=IMG_SIZES[fold],ef=EFF_NETS[fold])
 sv = tf.keras.callbacks.ModelCheckpoint(
 'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min', save_freq='epoch')
 print('Training...')
 history = model.fit(
 get_dataset(files_train, augment=True, shuffle=True, repeat=True,
dim=IMG_SIZES[fold], batch_size = BATCH_SIZES[fold]),
epochs=EPOCHS[fold], callbacks = [sv,get_lr_callback(BATCH_SIZES[fold])],
steps_per_epoch=count_data_items(files_train)/BATCH_SIZES[fold]//REPLICAS,
validation_data=get_dataset(files_valid,augment=False,shuffle=False,
repeat=False,dim=IMG_SIZES[fold]),
verbose=VERBOSE 
)
 print('Loading best model...')
 model.load_weights('fold-%i.h5'%fold)
 print('Predicting OOF with TTA...')
 ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_valid = count_data_items(files_valid); STEPS = TTA * ct_valid/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,]
oof_pred.append(np.mean(pred.reshape(( ct_valid,TTA),order='F'),axis=1))
ds_valid = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=True, return_image_names=True)
 oof_tar.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]))
oof_folds.append(np.ones_like(oof_tar[-1],dtype='int8')*fold)
 ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 oof_names.append(np.array([img_name.numpy().decode("utf-8")for img, img_name in iter(ds.unbatch())]))
print('Predicting Test with TTA...')
 ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,]
preds[:,0] += np.mean(pred.reshape(( ct_test,TTA),order='F'),axis=1)* WGTS[fold]
auc = roc_auc_score(oof_tar[-1],oof_pred[-1])
 oof_val.append(np.max(history.history['val_auc']))
print('
if DISPLAY_PLOT:
plt.figure(figsize=(15,5))
plt.plot(np.arange(EPOCHS[fold]),history.history['auc'],'-o',label='Train AUC',color='
plt.plot(np.arange(EPOCHS[fold]),history.history['val_auc'],'-o',label='Val AUC',color='
x = np.argmax(history.history['val_auc']); y = np.max(history.history['val_auc'])
 xdist = plt.xlim() [1] - plt.xlim() [0]; ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
%.2f'%y,size=14)
 plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)
 plt.legend(loc=2)
 plt2 = plt.gca().twinx()
plt2.plot(np.arange(EPOCHS[fold]),history.history['loss'],'-o',label='Train Loss',color='
plt2.plot(np.arange(EPOCHS[fold]),history.history['val_loss'],'-o',label='Val Loss',color='
x = np.argmin(history.history['val_loss']); y = np.min(history.history['val_loss'])
 ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
plt.ylabel('Loss',size=14)
 plt.title('FOLD %i - Image Size %i, EfficientNet B%i, inc2019=%i, inc2018=%i'% 
(fold+1,IMG_SIZES[fold],EFF_NETS[fold],INC2019[fold],INC2018[fold]),size=18)
 plt.legend(loc=3)
 plt.show()
param_name = 'max_depth'
param_range = np.arange(1, 31)
 train_score, valid_score = [], []
for depth in param_range:
gb = ens.GradientBoostingClassifier(n_estimators= 170,
subsample= 0.9, max_features= 'auto', loss= 'exponential',max_depth= depth)
 gb.fit(x_train.loc[:,gb_selected_features], y_train)
 train_score.append(gb.score(x_train.loc[:,gb_selected_features], y_train))
valid_score.append(gb.score(x_valid.loc[:,gb_selected_features], y_valid))<train_model>
oof = np.concatenate(oof_pred); true = np.concatenate(oof_tar);
names = np.concatenate(oof_names); folds = np.concatenate(oof_folds)
 auc = roc_auc_score(true,oof)
 print('Overall OOF AUC with TTA = %.3f'%auc)
 df_oof = pd.DataFrame(dict(
 image_name = names, target=true, pred = oof, fold=folds))
df_oof.to_csv('oof.csv',index=False)
 df_oof.head()
gb = ens.GradientBoostingClassifier(n_estimators= 170, subsample= 0.9, max_features= 'auto',
loss= 'exponential',max_depth= 4)
 gb.fit(features.loc[:, gb_selected_features],target )<compute_train_metric>
ds = get_dataset(files_test, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )
y_scores_gb = gb.predict_proba(x_test.loc[:, gb_selected_features])[:, 1]
gb_fpr, gb_tpr, gb_thresholds = sklearn.metrics.roc_curve(y_test, y_scores_gb)
 gb_auc = sklearn.metrics.auc(x=gb_fpr, y=gb_tpr )<compute_test_metric>
submission = pd.DataFrame(dict(image_name=image_names, target=preds[:,0]))
submission = submission.sort_values('image_name')
 submission.to_csv('submission.csv', index=False)
 submission.head()