Kaggle competition: Plant Pathology 2020 - FGVC7
DEVICE = "TPU"
SEED = 42
FOLDS = 5
IMG_SIZES = [384] * FOLDS
INC2019 = [0] * FOLDS
INC2018 = [1] * FOLDS
INC2020 = [1] * FOLDS
BATCH_SIZES = [64]*FOLDS
EPOCHS = [11]*FOLDS
EFF_NETS = [6] * FOLDS
WGTS = [1/FOLDS]*FOLDS
TTA = 11<choose_model_class>
!pip install -q efficientnet
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<define_variables>
tqdm.pandas()
np.random.seed(0)
 tf.random.set_seed(0)
 warnings.filterwarnings("ignore" )
GCS_PATH = [None]*FOLDS; GCS_PATH2 = [None]*FOLDS; GCS_PATH3 = [None]*FOLDS
for i,k in enumerate(IMG_SIZES):
GCS_PATH[i] = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(k,k))
GCS_PATH2[i] = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(k,k))
GCS_PATH3[i] = KaggleDatasets().get_gcs_path('malignant-v2-%ix%i'%(k,k))
files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/train*.tfrec')))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec')) )<define_variables>
EPOCHS = 20
SAMPLE_LEN = 100
IMAGE_PATH = ".. /input/plant-pathology-2020-fgvc7/images/"
TEST_PATH = ".. /input/plant-pathology-2020-fgvc7/test.csv"
TRAIN_PATH = ".. /input/plant-pathology-2020-fgvc7/train.csv"
SUB_PATH = ".. /input/plant-pathology-2020-fgvc7/sample_submission.csv"
sub = pd.read_csv(SUB_PATH)
 test_data = pd.read_csv(TEST_PATH)
 train_data = pd.read_csv(TRAIN_PATH )
ROT_ = 180.0
SHR_ = 2.0
HZOOM_ = 8.0
WZOOM_ = 8.0
HSHIFT_ = 8.0
WSHIFT_ = 8.0<normalization>
red_values = [np.mean(train_images[idx][:, :, 0])for idx in range(len(train_images)) ]
green_values = [np.mean(train_images[idx][:, :, 1])for idx in range(len(train_images)) ]
blue_values = [np.mean(train_images[idx][:, :, 2])for idx in range(len(train_images)) ]
values = [np.mean(train_images[idx])for idx in range(len(train_images)) ]
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, DIM=256):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )<prepare_x_and_y>
invert(train_images[3])
 invert(train_images[4])
 invert(train_images[5] )
@tf.function
def transform_grid_mark(image, inv_mat, image_shape):
h, w, c = image_shape
cx, cy = w//2, h//2
new_xs = tf.repeat(tf.range(-cx, cx, 1), h)
 new_ys = tf.tile(tf.range(-cy, cy, 1), [w])
 new_zs = tf.ones([h*w], dtype=tf.int32)
 old_coords = tf.matmul(inv_mat, tf.cast(tf.stack([new_xs, new_ys, new_zs]), tf.float32))
old_coords_x, old_coords_y = tf.round(old_coords[0, :] + tf.cast(w, tf.float32)//2.) , tf.round(old_coords[1, :] + tf.cast(h, tf.float32)//2.)
old_coords_x = tf.cast(old_coords_x, tf.int32)
 old_coords_y = tf.cast(old_coords_y, tf.int32)
 clip_mask_x = tf.logical_or(old_coords_x<0, old_coords_x>w-1)
 clip_mask_y = tf.logical_or(old_coords_y<0, old_coords_y>h-1)
 clip_mask = tf.logical_or(clip_mask_x, clip_mask_y)
 old_coords_x = tf.boolean_mask(old_coords_x, tf.logical_not(clip_mask))
old_coords_y = tf.boolean_mask(old_coords_y, tf.logical_not(clip_mask))
new_coords_x = tf.boolean_mask(new_xs+cx, tf.logical_not(clip_mask))
new_coords_y = tf.boolean_mask(new_ys+cy, tf.logical_not(clip_mask))
old_coords = tf.cast(tf.stack([old_coords_y, old_coords_x]), tf.int32)
 new_coords = tf.cast(tf.stack([new_coords_y, new_coords_x]), tf.int64)
 rotated_image_values = tf.gather_nd(image, tf.transpose(old_coords))
rotated_image_channel = list()
for i in range(c):
vals = rotated_image_values[:,i]
sparse_channel = tf.SparseTensor(tf.transpose(new_coords), vals, [h, w])
 rotated_image_channel.append(tf.sparse.to_dense(sparse_channel, default_value=0, validate_indices=False))
return tf.transpose(tf.stack(rotated_image_channel), [1,2,0])
 @tf.function
def random_rotate(image, angle, image_shape):
def get_rotation_mat_inv(angle):
angle = math.pi * angle / 180
cos_val = tf.math.cos(angle)
 sin_val = tf.math.sin(angle)
 one = tf.constant([1], tf.float32)
 zero = tf.constant([0], tf.float32)
 rot_mat_inv = tf.concat([cos_val, sin_val, zero,
-sin_val, cos_val, zero,
zero, zero, one], axis=0)
 rot_mat_inv = tf.reshape(rot_mat_inv, [3,3])
 return rot_mat_inv
angle = float(angle)* tf.random.normal([1],dtype='float32')
 rot_mat_inv = get_rotation_mat_inv(angle)
 return transform_grid_mark(image, rot_mat_inv, image_shape)
 @tf.function
def grid_mask(DIM=256):
h = tf.constant(DIM, dtype=tf.float32)
 w = tf.constant(DIM, dtype=tf.float32)
 image_height, image_width =(h, w)
 d1 = int(DIM / 6)
 d2 = int(DIM / 4)
 rotate_angle = 45
ratio = 0.4
hh = tf.math.ceil(tf.math.sqrt(h*h+w*w))
hh = tf.cast(hh, tf.int32)
 hh = hh+1 if hh%2==1 else hh
d = tf.random.uniform(shape=[], minval=d1, maxval=d2, dtype=tf.int32)
 l = tf.cast(tf.cast(d,tf.float32)*ratio+0.5, tf.int32)
 st_h = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
 st_w = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
 y_ranges = tf.range(-1 * d + st_h, -1 * d + st_h + l)
 x_ranges = tf.range(-1 * d + st_w, -1 * d + st_w + l)
 for i in range(0, hh//d+1):
s1 = i * d + st_h
s2 = i * d + st_w
y_ranges = tf.concat([y_ranges, tf.range(s1,s1+l)], axis=0)
 x_ranges = tf.concat([x_ranges, tf.range(s2,s2+l)], axis=0)
 x_clip_mask = tf.logical_or(x_ranges <0 , x_ranges > hh-1)
 y_clip_mask = tf.logical_or(y_ranges <0 , y_ranges > hh-1)
 clip_mask = tf.logical_or(x_clip_mask, y_clip_mask)
 x_ranges = tf.boolean_mask(x_ranges, tf.logical_not(clip_mask))
y_ranges = tf.boolean_mask(y_ranges, tf.logical_not(clip_mask))
hh_ranges = tf.tile(tf.range(0,hh), [tf.cast(tf.reduce_sum(tf.ones_like(x_ranges)) , tf.int32)])
 x_ranges = tf.repeat(x_ranges, hh)
 y_ranges = tf.repeat(y_ranges, hh)
 y_hh_indices = tf.transpose(tf.stack([y_ranges, hh_ranges]))
x_hh_indices = tf.transpose(tf.stack([hh_ranges, x_ranges]))
y_mask_sparse = tf.SparseTensor(tf.cast(y_hh_indices, tf.int64), tf.zeros_like(y_ranges), [hh, hh])
 y_mask = tf.sparse.to_dense(y_mask_sparse, 1, False)
 x_mask_sparse = tf.SparseTensor(tf.cast(x_hh_indices, tf.int64), tf.zeros_like(x_ranges), [hh, hh])
 x_mask = tf.sparse.to_dense(x_mask_sparse, 1, False)
 mask = tf.expand_dims(tf.clip_by_value(x_mask + y_mask, 0, 1), axis=-1)
 mask = random_rotate(mask, rotate_angle, [hh, hh, 1])
 mask = tf.image.crop_to_bounding_box(mask,(hh-tf.cast(h, tf.int32)) //2,(hh-tf.cast(w, tf.int32)) //2, tf.cast(image_height, tf.int32), tf.cast(image_width, tf.int32))
return mask
@tf.function
def apply_grid_mask(image, DIM=256):
mask = grid_mask(DIM=DIM)
 mask = tf.concat([mask, mask, mask], axis=-1)
 return image * tf.cast(mask, 'float32' )<prepare_x_and_y>
AUTO = tf.data.experimental.AUTOTUNE
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 BATCH_SIZE = 16 * strategy.num_replicas_in_sync
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img, augment=True, dim=256):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img,DIM=dim)
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 img = apply_grid_mask(img,DIM=dim)
 img = tf.reshape(img, [dim,dim, 3])
 return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )<create_dataframe>
def format_path(st):
return GCS_DS_PATH + '/images/' + st + '.jpg'
test_paths = test_data.image_id.apply(format_path ).values
train_paths = train_data.image_id.apply(format_path ).values
train_labels = np.float32(train_data.loc[:, 'healthy':'scab'].values)
 train_paths, valid_paths, train_labels, valid_labels =\
train_test_split(train_paths, train_labels, test_size=0.15, random_state=2020 )
def get_dataset(files, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True, batch_size=16, dim=256):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, imgname_or_label:(prepare_image(img, augment=augment, dim=dim),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(batch_size * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds<choose_model_class>
def decode_image(filename, label=None, image_size=(512, 512)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if label is None:
return image
else:
return image, label
EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3,
efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6]
def build_model(dim=128, ef=0):
inp = tf.keras.layers.Input(shape=(dim,dim,3))
base = EFNS[ef](input_shape=(dim,dim,3),weights='imagenet',include_top=False)
 x = base(inp)
 x = tf.keras.layers.GlobalAveragePooling2D()(x)
 x = tf.keras.layers.Dense(1,activation='sigmoid' )(x)
 model = tf.keras.Model(inputs=inp,outputs=x)
 opt = tfa.optimizers.RectifiedAdam()
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05)
 model.compile(optimizer=opt,loss=loss,metrics=['AUC'])
 return model<init_hyperparams>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( valid_paths, valid_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )
def get_lr_callback(batch_size=8):
lr_start = 0.00001
lr_max = 0.0000013 * REPLICAS * batch_size
lr_min = 0.000001
lr_ramp_ep = 4
lr_sus_ep = 0
lr_decay = 0.85
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback<define_variables>
def build_lrfn(lr_start=0.00001, lr_max=0.00005,
lr_min=0.00001, lr_rampup_epochs=5,
lr_sustain_epochs=0, lr_exp_decay=.8):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)*\
lr_exp_decay**(epoch - lr_rampup_epochs\
- lr_sustain_epochs)+ lr_min
return lr
return lrfn
VERBOSE = 1
DISPLAY_PLOT = True
skf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)
 oof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = []
preds = np.zeros(( count_data_items(files_test),1))
for fold,(idxT,idxV)in enumerate(skf.split(np.arange(15))):
if DEVICE=='TPU':
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 print('
print(' 
(IMG_SIZES[fold],EFF_NETS[fold],BATCH_SIZES[fold]*REPLICAS))
files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxT])
 if INC2019[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1])
 print('
if INC2018[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2])
 print('
if INC2020[fold]:
files_train += tf.io.gfile.glob([GCS_PATH3[fold] + '/train%.2i*.tfrec'%x for x in idxT + 15])
 print('
np.random.shuffle(files_train); print('
files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxV])
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/test*.tfrec')))
 K.clear_session()
with strategy.scope() :
model = build_model(dim=IMG_SIZES[fold],ef=EFF_NETS[fold])
 sv = tf.keras.callbacks.ModelCheckpoint(
 'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min', save_freq='epoch')
 print('Training...')
 history = model.fit(
 get_dataset(files_train, augment=True, shuffle=True, repeat=True,
dim=IMG_SIZES[fold], batch_size = BATCH_SIZES[fold]),
epochs=EPOCHS[fold], callbacks = [sv,get_lr_callback(BATCH_SIZES[fold])],
steps_per_epoch=count_data_items(files_train)/BATCH_SIZES[fold]//REPLICAS,
validation_data=get_dataset(files_valid,augment=False,shuffle=False,
repeat=False,dim=IMG_SIZES[fold]),
verbose=VERBOSE 
)
 print('Loading best model...')
 model.load_weights('fold-%i.h5'%fold)
 print('Save whole model...')
 model.save('fold-%i-all.h5'%fold)
 gc.collect()
print('Predicting OOF with TTA...')
 ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*3)
 ct_valid = count_data_items(files_valid)
 STEPS = TTA * ct_valid/BATCH_SIZES[fold]/3/REPLICAS
pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,]
oof_pred.append(np.mean(pred.reshape(( ct_valid,TTA),order='F'),axis=1))
ds_valid = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=True, return_image_names=True)
 oof_tar.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]))
oof_folds.append(np.ones_like(oof_tar[-1],dtype='int8')*fold)
 ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 oof_names.append(np.array([img_name.numpy().decode("utf-8")for img, img_name in iter(ds.unbatch())]))
gc.collect()
print('Predicting Test with TTA...')
 ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*3)
 ct_test = count_data_items(files_test)
 STEPS = TTA * ct_test/BATCH_SIZES[fold]/3/REPLICAS
pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,]
preds[:,0] += np.mean(pred.reshape(( ct_test,TTA),order='F'),axis=1)* WGTS[fold]
gc.collect()
auc = roc_auc_score(oof_tar[-1],oof_pred[-1])
 oof_val.append(np.max(history.history['val_auc']))
print('
if DISPLAY_PLOT:
plt.figure(figsize=(15,5))
plt.plot(np.arange(EPOCHS[fold]),history.history['auc'],'-o',label='Train AUC',color='
plt.plot(np.arange(EPOCHS[fold]),history.history['val_auc'],'-o',label='Val AUC',color='
x = np.argmax(history.history['val_auc']); y = np.max(history.history['val_auc'])
 xdist = plt.xlim() [1] - plt.xlim() [0]; ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
%.2f'%y,size=14)
 plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)
 plt.legend(loc=2)
 plt2 = plt.gca().twinx()
plt2.plot(np.arange(EPOCHS[fold]),history.history['loss'],'-o',label='Train Loss',color='
plt2.plot(np.arange(EPOCHS[fold]),history.history['val_loss'],'-o',label='Val Loss',color='
x = np.argmin(history.history['val_loss']); y = np.min(history.history['val_loss'])
 ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
plt.ylabel('Loss',size=14)
 plt.title('FOLD %i - Image Size %i, EfficientNet B%i, inc2020=%i, inc2019=%i, inc2018=%i'% 
(fold+1,IMG_SIZES[fold],EFF_NETS[fold],INC2020[fold],INC2019[fold],INC2018[fold]),size=18)
 plt.legend(loc=3)
 plt.show()
gc.collect()<save_to_csv>
with strategy.scope() :
model = tf.keras.Sequential([DenseNet121(input_shape=(512, 512, 3),
weights='imagenet',
include_top=False),
L.GlobalAveragePooling2D() ,
L.Dense(train_labels.shape[1],
activation='softmax')])
 model.compile(optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'])
 model.summary()
oof = np.concatenate(oof_pred); true = np.concatenate(oof_tar);
names = np.concatenate(oof_names); folds = np.concatenate(oof_folds)
 auc = roc_auc_score(true,oof)
 print('Overall OOF AUC with TTA = %.3f'%auc)
 df_oof = pd.DataFrame(dict(
 image_name = names, target=true, pred = oof, fold=folds))
df_oof.to_csv('oof.csv',index=False)
 df_oof.head()<create_dataframe>
history = model.fit(train_dataset,
epochs=EPOCHS,
callbacks=[lr_schedule],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset )
ds = get_dataset(files_test, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )<save_to_csv>
def process(img):
return cv2.resize(img/255.0,(512, 512)).reshape(-1, 512, 512, 3)
 def predict(img):
return model.layers[2](model.layers[1](model.layers[0](process(img)))).numpy() [0]
fig = make_subplots(rows=4, cols=2)
 preds = predict(train_images[2])
 colors = {"Healthy":px.colors.qualitative.Plotly[0], "Scab":px.colors.qualitative.Plotly[0], "Rust":px.colors.qualitative.Plotly[0], "Multiple diseases":px.colors.qualitative.Plotly[0]}
if list.index(preds.tolist() , max(preds)) == 0:
pred = "Healthy"
if list.index(preds.tolist() , max(preds)) == 1:
pred = "Scab"
if list.index(preds.tolist() , max(preds)) == 2:
pred = "Rust"
if list.index(preds.tolist() , max(preds)) == 3:
pred = "Multiple diseases"
colors[pred] = px.colors.qualitative.Plotly[1]
colors["Healthy"] = "seagreen"
colors = [colors[val] for val in colors.keys() ]
fig.add_trace(go.Image(z=cv2.resize(train_images[2],(205, 136))), row=1, col=1)
 fig.add_trace(go.Bar(x=["Healthy", "Multiple diseases", "Rust", "Scab"], y=preds, marker=dict(color=colors)) , row=1, col=2)
 fig.update_layout(height=1200, width=800, title_text="DenseNet Predictions", showlegend=False)
 preds = predict(train_images[0])
 colors = {"Healthy":px.colors.qualitative.Plotly[0], "Scab":px.colors.qualitative.Plotly[0], "Rust":px.colors.qualitative.Plotly[0], "Multiple diseases":px.colors.qualitative.Plotly[0]}
if list.index(preds.tolist() , max(preds)) == 0:
pred = "Healthy"
if list.index(preds.tolist() , max(preds)) == 1:
pred = "Multiple diseases"
if list.index(preds.tolist() , max(preds)) == 2:
pred = "Rust"
if list.index(preds.tolist() , max(preds)) == 3:
pred = "Scab"
colors[pred] = px.colors.qualitative.Plotly[1]
colors["Multiple diseases"] = "seagreen"
colors = [colors[val] for val in colors.keys() ]
fig.add_trace(go.Image(z=cv2.resize(train_images[0],(205, 136))), row=2, col=1)
 fig.add_trace(go.Bar(x=["Healthy", "Multiple diseases", "Rust", "Scab"], y=preds, marker=dict(color=colors)) , row=2, col=2)
 preds = predict(train_images[3])
 colors = {"Healthy":px.colors.qualitative.Plotly[0], "Scab":px.colors.qualitative.Plotly[0], "Rust":px.colors.qualitative.Plotly[0], "Multiple diseases":px.colors.qualitative.Plotly[0]}
if list.index(preds.tolist() , max(preds)) == 0:
pred = "Healthy"
if list.index(preds.tolist() , max(preds)) == 1:
pred = "Multiple diseases"
if list.index(preds.tolist() , max(preds)) == 2:
pred = "Rust"
if list.index(preds.tolist() , max(preds)) == 3:
pred = "Scab"
colors[pred] = px.colors.qualitative.Plotly[1]
colors["Rust"] = "seagreen"
colors = [colors[val] for val in colors.keys() ]
fig.add_trace(go.Image(z=cv2.resize(train_images[3],(205, 136))), row=3, col=1)
 fig.add_trace(go.Bar(x=["Healthy", "Multiple diseases", "Rust", "Scab"], y=preds, marker=dict(color=colors)) , row=3, col=2)
 preds = predict(train_images[1])
 colors = {"Healthy":px.colors.qualitative.Plotly[0], "Scab":px.colors.qualitative.Plotly[0], "Rust":px.colors.qualitative.Plotly[0], "Multiple diseases":px.colors.qualitative.Plotly[0]}
if list.index(preds.tolist() , max(preds)) == 0:
pred = "Healthy"
if list.index(preds.tolist() , max(preds)) == 1:
pred = "Multiple diseases"
if list.index(preds.tolist() , max(preds)) == 2:
pred = "Rust"
if list.index(preds.tolist() , max(preds)) == 3:
pred = "Scab"
colors[pred] = px.colors.qualitative.Plotly[1]
colors["Scab"] = "seagreen"
colors = [colors[val] for val in colors.keys() ]
fig.add_trace(go.Image(z=cv2.resize(train_images[1],(205, 136))), row=4, col=1)
 fig.add_trace(go.Bar(x=["Healthy", "Multiple diseases", "Rust", "Scab"], y=preds, marker=dict(color=colors)) , row=4, col=2)
 fig.update_layout(template="plotly_white" )
submission = pd.DataFrame(dict(image_name=image_names, target=preds[:,0]))
submission = submission.sort_values('image_name')
 submission.to_csv('submission.csv', index=False)
 submission.head()<import_modules>
probs_dnn = model.predict(test_dataset, verbose=1)
 sub.loc[:, 'healthy':] = probs_dnn
sub.to_csv('submission_dnn.csv', index=False)
 sub.head()
import numpy as np
import pandas as pd<load_from_csv>
with strategy.scope() :
model = tf.keras.Sequential([efn.EfficientNetB7(input_shape=(512, 512, 3),
weights='imagenet',
include_top=False),
L.GlobalAveragePooling2D() ,
L.Dense(train_labels.shape[1],
activation='softmax')])
 model.compile(optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'])
 model.summary()
sub_image = pd.read_csv('/kaggle/input/melanoma-submissions2/submissionImage.csv')
 sub_tabular_70 = pd.read_csv('/kaggle/input/melanoma-submissions2/submissionTabular.csv')
 sub_multiple = pd.read_csv('/kaggle/input/melanoma-submissions2/submission_multiple_data_source.csv')
 sub_public_merge = pd.read_csv('/kaggle/input/submission-9/submission_935.csv')
 sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 sub_tabular_78 = pd.read_csv('/kaggle/input/tabular-prediction/submission_tabular_78.csv')
 sub_mean = pd.read_csv('/kaggle/input/siim-isic-multiple-model-training-stacking-923/submission_mean.csv' )<define_variables>
history = model.fit(train_dataset,
epochs=EPOCHS,
callbacks=[lr_schedule],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset )
sub.target = sub_mean.target *0.29 + sub_public_merge.target *0.69 + sub_tabular_78.target *0.02<save_to_csv>
def process(img):
return cv2.resize(img/255.0,(512, 512)).reshape(-1, 512, 512, 3)
 def predict(img):
return model.layers[2](model.layers[1](model.layers[0](process(img)))).numpy() [0]
fig = make_subplots(rows=4, cols=2)
 preds = predict(train_images[2])
 colors = {"Healthy":px.colors.qualitative.Plotly[0], "Scab":px.colors.qualitative.Plotly[0], "Rust":px.colors.qualitative.Plotly[0], "Multiple diseases":px.colors.qualitative.Plotly[0]}
if list.index(preds.tolist() , max(preds)) == 0:
pred = "Healthy"
if list.index(preds.tolist() , max(preds)) == 1:
pred = "Scab"
if list.index(preds.tolist() , max(preds)) == 2:
pred = "Rust"
if list.index(preds.tolist() , max(preds)) == 3:
pred = "Multiple diseases"
colors[pred] = px.colors.qualitative.Plotly[1]
colors["Healthy"] = "seagreen"
colors = [colors[val] for val in colors.keys() ]
fig.add_trace(go.Image(z=cv2.resize(train_images[2],(205, 136))), row=1, col=1)
 fig.add_trace(go.Bar(x=["Healthy", "Multiple diseases", "Rust", "Scab"], y=preds, marker=dict(color=colors)) , row=1, col=2)
 fig.update_layout(height=1200, width=800, title_text="EfficientNet Predictions", showlegend=False)
 preds = predict(train_images[0])
 colors = {"Healthy":px.colors.qualitative.Plotly[0], "Scab":px.colors.qualitative.Plotly[0], "Rust":px.colors.qualitative.Plotly[0], "Multiple diseases":px.colors.qualitative.Plotly[0]}
if list.index(preds.tolist() , max(preds)) == 0:
pred = "Healthy"
if list.index(preds.tolist() , max(preds)) == 1:
pred = "Multiple diseases"
if list.index(preds.tolist() , max(preds)) == 2:
pred = "Rust"
if list.index(preds.tolist() , max(preds)) == 3:
pred = "Scab"
colors[pred] = px.colors.qualitative.Plotly[1]
colors["Multiple diseases"] = "seagreen"
colors = [colors[val] for val in colors.keys() ]
fig.add_trace(go.Image(z=cv2.resize(train_images[0],(205, 136))), row=2, col=1)
 fig.add_trace(go.Bar(x=["Healthy", "Multiple diseases", "Rust", "Scab"], y=preds, marker=dict(color=colors)) , row=2, col=2)
 preds = predict(train_images[3])
 colors = {"Healthy":px.colors.qualitative.Plotly[0], "Scab":px.colors.qualitative.Plotly[0], "Rust":px.colors.qualitative.Plotly[0], "Multiple diseases":px.colors.qualitative.Plotly[0]}
if list.index(preds.tolist() , max(preds)) == 0:
pred = "Healthy"
if list.index(preds.tolist() , max(preds)) == 1:
pred = "Multiple diseases"
if list.index(preds.tolist() , max(preds)) == 2:
pred = "Rust"
if list.index(preds.tolist() , max(preds)) == 3:
pred = "Scab"
colors[pred] = px.colors.qualitative.Plotly[1]
colors["Rust"] = "seagreen"
colors = [colors[val] for val in colors.keys() ]
fig.add_trace(go.Image(z=cv2.resize(train_images[3],(205, 136))), row=3, col=1)
 fig.add_trace(go.Bar(x=["Healthy", "Multiple diseases", "Rust", "Scab"], y=preds, marker=dict(color=colors)) , row=3, col=2)
 preds = predict(train_images[1])
 colors = {"Healthy":px.colors.qualitative.Plotly[0], "Scab":px.colors.qualitative.Plotly[0], "Rust":px.colors.qualitative.Plotly[0], "Multiple diseases":px.colors.qualitative.Plotly[0]}
if list.index(preds.tolist() , max(preds)) == 0:
pred = "Healthy"
if list.index(preds.tolist() , max(preds)) == 1:
pred = "Multiple diseases"
if list.index(preds.tolist() , max(preds)) == 2:
pred = "Rust"
if list.index(preds.tolist() , max(preds)) == 3:
pred = "Scab"
colors[pred] = px.colors.qualitative.Plotly[1]
colors["Scab"] = "seagreen"
colors = [colors[val] for val in colors.keys() ]
fig.add_trace(go.Image(z=cv2.resize(train_images[1],(205, 136))), row=4, col=1)
 fig.add_trace(go.Bar(x=["Healthy", "Multiple diseases", "Rust", "Scab"], y=preds, marker=dict(color=colors)) , row=4, col=2)
 fig.update_layout(template="plotly_white" )
sub.to_csv('submission.csv', index = False )<set_options>
probs_efn = model.predict(test_dataset, verbose=1)
 sub.loc[:, 'healthy':] = probs_efn
sub.to_csv('submission_efn.csv', index=False)
 sub.head()
sns.set()
py.init_notebook_mode(connected=True)
 os.listdir(".. /input/" )<define_variables>
with strategy.scope() :
model = tf.keras.Sequential([efn.EfficientNetB7(input_shape=(512, 512, 3),
weights='noisy-student',
include_top=False),
L.GlobalAveragePooling2D() ,
L.Dense(train_labels.shape[1],
activation='softmax')])
 model.compile(optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'])
 model.summary()
basepath = ".. /input/siim-isic-melanoma-classification/"
modelspath = ".. /input/pytorch-pretrained-image-models/"
imagestatspath = ".. /input/siimisic-melanoma-classification-image-stats/"
foldspath = ".. /input/irndropout/"<load_from_csv>
history = model.fit(train_dataset,
epochs=EPOCHS,
callbacks=[lr_schedule],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset )
train_info = pd.read_csv(basepath + "train.csv")
 train_info.head()<load_from_csv>
def process(img):
return cv2.resize(img/255.0,(512, 512)).reshape(-1, 512, 512, 3)
 def predict(img):
return model.layers[2](model.layers[1](model.layers[0](process(img)))).numpy() [0]
fig = make_subplots(rows=4, cols=2)
 preds = predict(train_images[2])
 colors = {"Healthy":px.colors.qualitative.Plotly[0], "Scab":px.colors.qualitative.Plotly[0], "Rust":px.colors.qualitative.Plotly[0], "Multiple diseases":px.colors.qualitative.Plotly[0]}
if list.index(preds.tolist() , max(preds)) == 0:
pred = "Healthy"
if list.index(preds.tolist() , max(preds)) == 1:
pred = "Scab"
if list.index(preds.tolist() , max(preds)) == 2:
pred = "Rust"
if list.index(preds.tolist() , max(preds)) == 3:
pred = "Multiple diseases"
colors[pred] = px.colors.qualitative.Plotly[1]
colors["Healthy"] = "seagreen"
colors = [colors[val] for val in colors.keys() ]
fig.add_trace(go.Image(z=cv2.resize(train_images[2],(205, 136))), row=1, col=1)
 fig.add_trace(go.Bar(x=["Healthy", "Multiple diseases", "Rust", "Scab"], y=preds, marker=dict(color=colors)) , row=1, col=2)
 fig.update_layout(height=1200, width=800, title_text="EfficientNet NoisyStudent Predictions", showlegend=False)
 preds = predict(train_images[0])
 colors = {"Healthy":px.colors.qualitative.Plotly[0], "Scab":px.colors.qualitative.Plotly[0], "Rust":px.colors.qualitative.Plotly[0], "Multiple diseases":px.colors.qualitative.Plotly[0]}
if list.index(preds.tolist() , max(preds)) == 0:
pred = "Healthy"
if list.index(preds.tolist() , max(preds)) == 1:
pred = "Multiple diseases"
if list.index(preds.tolist() , max(preds)) == 2:
pred = "Rust"
if list.index(preds.tolist() , max(preds)) == 3:
pred = "Scab"
colors[pred] = px.colors.qualitative.Plotly[1]
colors["Multiple diseases"] = "seagreen"
colors = [colors[val] for val in colors.keys() ]
fig.add_trace(go.Image(z=cv2.resize(train_images[0],(205, 136))), row=2, col=1)
 fig.add_trace(go.Bar(x=["Healthy", "Multiple diseases", "Rust", "Scab"], y=preds, marker=dict(color=colors)) , row=2, col=2)
 preds = predict(train_images[3])
 colors = {"Healthy":px.colors.qualitative.Plotly[0], "Scab":px.colors.qualitative.Plotly[0], "Rust":px.colors.qualitative.Plotly[0], "Multiple diseases":px.colors.qualitative.Plotly[0]}
if list.index(preds.tolist() , max(preds)) == 0:
pred = "Healthy"
if list.index(preds.tolist() , max(preds)) == 1:
pred = "Multiple diseases"
if list.index(preds.tolist() , max(preds)) == 2:
pred = "Rust"
if list.index(preds.tolist() , max(preds)) == 3:
pred = "Scab"
colors[pred] = px.colors.qualitative.Plotly[1]
colors["Rust"] = "seagreen"
colors = [colors[val] for val in colors.keys() ]
fig.add_trace(go.Image(z=cv2.resize(train_images[3],(205, 136))), row=3, col=1)
 fig.add_trace(go.Bar(x=["Healthy", "Multiple diseases", "Rust", "Scab"], y=preds, marker=dict(color=colors)) , row=3, col=2)
 preds = predict(train_images[1])
 colors = {"Healthy":px.colors.qualitative.Plotly[0], "Scab":px.colors.qualitative.Plotly[0], "Rust":px.colors.qualitative.Plotly[0], "Multiple diseases":px.colors.qualitative.Plotly[0]}
if list.index(preds.tolist() , max(preds)) == 0:
pred = "Healthy"
if list.index(preds.tolist() , max(preds)) == 1:
pred = "Multiple diseases"
if list.index(preds.tolist() , max(preds)) == 2:
pred = "Rust"
if list.index(preds.tolist() , max(preds)) == 3:
pred = "Scab"
colors[pred] = px.colors.qualitative.Plotly[1]
colors["Scab"] = "seagreen"
colors = [colors[val] for val in colors.keys() ]
fig.add_trace(go.Image(z=cv2.resize(train_images[1],(205, 136))), row=4, col=1)
 fig.add_trace(go.Bar(x=["Healthy", "Multiple diseases", "Rust", "Scab"], y=preds, marker=dict(color=colors)) , row=4, col=2)
 fig.update_layout(template="plotly_white" )
test_info = pd.read_csv(basepath + "test.csv")
 test_info.head()<sort_values>
probs_efnns = model.predict(test_dataset, verbose=1)
 sub.loc[:, 'healthy':] = probs_efnns
sub.to_csv('submission_efnns.csv', index=False)
 sub.head()
<sort_values><EOS> 
ensemble_1, ensemble_2, ensemble_3 = [sub]*3
ensemble_1.loc[:, 'healthy':] = 0.50*probs_dnn + 0.50*probs_efn
ensemble_2.loc[:, 'healthy':] = 0.25*probs_dnn + 0.75*probs_efn
ensemble_3.loc[:, 'healthy':] = 0.75*probs_dnn + 0.25*probs_efn
ensemble_1.to_csv('submission_ensemble_1.csv', index=False)
 ensemble_2.to_csv('submission_ensemble_2.csv', index=False)
 ensemble_3.to_csv('submission_ensemble_3.csv', index=False )