Kaggle competition: Plant Pathology 2020 - FGVC7
<SOS> metric: MCAUC Kaggle data source: plant-pathology-2020-fgvc7<categorify>
!pip install -q efficientnet
def transform(image, DIM=256):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )<prepare_x_and_y>
import numpy as np
import pandas as pd
import os
import tensorflow as tf
import keras
import efficientnet.tfkeras as efn
from kaggle_datasets import KaggleDatasets
from tensorflow.keras.models import Sequential
import tensorflow.keras.layers as L
from tensorflow.keras.applications import InceptionResNetV2
from tensorflow.keras.layers import Dense, GlobalAveragePooling2D
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
def dropout(image, DIM=256, PROBABILITY = 1, CT = 8, SZ = 0.2):
prob = tf.cast(tf.random.uniform([],0,1)<PROBABILITY, tf.int32)
 if(prob==0)|(CT==0)|(SZ==0): return image
for k in range(CT):
x = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 y = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 WIDTH = tf.cast(SZ*DIM,tf.int32)* prob
ya = tf.math.maximum(0,y-WIDTH//2)
 yb = tf.math.minimum(DIM,y+WIDTH//2)
 xa = tf.math.maximum(0,x-WIDTH//2)
 xb = tf.math.minimum(DIM,x+WIDTH//2)
 one = image[ya:yb,0:xa,:]
two = tf.zeros([yb-ya,xb-xa,3])
 three = image[ya:yb,xb:DIM,:]
middle = tf.concat([one,two,three],axis=1)
 image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM,:,:]],axis=0)
 image = tf.reshape(image,[DIM,DIM,3])
 return image<init_hyperparams>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
def seed_everything(seed=0):
np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
seed = 2048
seed_everything(seed)
 print("REPLICAS: ", strategy.num_replicas_in_sync)
 GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 40
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
AugParams = {
'd1' : 100,
'd2': 160,
'rotate' : 45,
'ratio' : 0.4
}<prepare_x_and_y>
def format_path(st):
return GCS_DS_PATH + '/images/' + st + '.jpg'
def transform2(image, inv_mat, image_shape):
h, w, c = image_shape
cx, cy = w//2, h//2
new_xs = tf.repeat(tf.range(-cx, cx, 1), h)
 new_ys = tf.tile(tf.range(-cy, cy, 1), [w])
 new_zs = tf.ones([h*w], dtype=tf.int32)
 old_coords = tf.matmul(inv_mat, tf.cast(tf.stack([new_xs, new_ys, new_zs]), tf.float32))
old_coords_x, old_coords_y = tf.round(old_coords[0, :] + w//2), tf.round(old_coords[1, :] + h//2)
 clip_mask_x = tf.logical_or(old_coords_x<0, old_coords_x>w-1)
 clip_mask_y = tf.logical_or(old_coords_y<0, old_coords_y>h-1)
 clip_mask = tf.logical_or(clip_mask_x, clip_mask_y)
 old_coords_x = tf.boolean_mask(old_coords_x, tf.logical_not(clip_mask))
old_coords_y = tf.boolean_mask(old_coords_y, tf.logical_not(clip_mask))
new_coords_x = tf.boolean_mask(new_xs+cx, tf.logical_not(clip_mask))
new_coords_y = tf.boolean_mask(new_ys+cy, tf.logical_not(clip_mask))
old_coords = tf.cast(tf.stack([old_coords_y, old_coords_x]), tf.int32)
 new_coords = tf.cast(tf.stack([new_coords_y, new_coords_x]), tf.int64)
 rotated_image_values = tf.gather_nd(image, tf.transpose(old_coords))
rotated_image_channel = list()
for i in range(c):
vals = rotated_image_values[:,i]
sparse_channel = tf.SparseTensor(tf.transpose(new_coords), vals, [h, w])
 rotated_image_channel.append(tf.sparse.to_dense(sparse_channel, default_value=0, validate_indices=False))
return tf.transpose(tf.stack(rotated_image_channel), [1,2,0])
 def random_rotate(image, angle, image_shape):
def get_rotation_mat_inv(angle):
angle = math.pi * angle / 180
cos_val = tf.math.cos(angle)
 sin_val = tf.math.sin(angle)
 one = tf.constant([1], tf.float32)
 zero = tf.constant([0], tf.float32)
 rot_mat_inv = tf.concat([cos_val, sin_val, zero,
-sin_val, cos_val, zero,
zero, zero, one], axis=0)
 rot_mat_inv = tf.reshape(rot_mat_inv, [3,3])
 return rot_mat_inv
angle = float(angle)* tf.random.normal([1],dtype='float32')
 rot_mat_inv = get_rotation_mat_inv(angle)
 return transform2(image, rot_mat_inv, image_shape)
 def GridMask(image_height, image_width, d1, d2, rotate_angle=1, ratio=0.5):
h, w = image_height, image_width
hh = int(np.ceil(np.sqrt(h*h+w*w)))
 hh = hh+1 if hh%2==1 else hh
d = tf.random.uniform(shape=[], minval=d1, maxval=d2, dtype=tf.int32)
 l = tf.cast(tf.cast(d,tf.float32)*ratio+0.5, tf.int32)
 st_h = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
 st_w = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
 y_ranges = tf.range(-1 * d + st_h, -1 * d + st_h + l)
 x_ranges = tf.range(-1 * d + st_w, -1 * d + st_w + l)
 for i in range(0, hh//d+1):
s1 = i * d + st_h
s2 = i * d + st_w
y_ranges = tf.concat([y_ranges, tf.range(s1,s1+l)], axis=0)
 x_ranges = tf.concat([x_ranges, tf.range(s2,s2+l)], axis=0)
 x_clip_mask = tf.logical_or(x_ranges <0 , x_ranges > hh-1)
 y_clip_mask = tf.logical_or(y_ranges <0 , y_ranges > hh-1)
 clip_mask = tf.logical_or(x_clip_mask, y_clip_mask)
 x_ranges = tf.boolean_mask(x_ranges, tf.logical_not(clip_mask))
y_ranges = tf.boolean_mask(y_ranges, tf.logical_not(clip_mask))
hh_ranges = tf.tile(tf.range(0,hh), [tf.cast(tf.reduce_sum(tf.ones_like(x_ranges)) , tf.int32)])
 x_ranges = tf.repeat(x_ranges, hh)
 y_ranges = tf.repeat(y_ranges, hh)
 y_hh_indices = tf.transpose(tf.stack([y_ranges, hh_ranges]))
x_hh_indices = tf.transpose(tf.stack([hh_ranges, x_ranges]))
y_mask_sparse = tf.SparseTensor(tf.cast(y_hh_indices, tf.int64), tf.zeros_like(y_ranges), [hh, hh])
 y_mask = tf.sparse.to_dense(y_mask_sparse, 1, False)
 x_mask_sparse = tf.SparseTensor(tf.cast(x_hh_indices, tf.int64), tf.zeros_like(x_ranges), [hh, hh])
 x_mask = tf.sparse.to_dense(x_mask_sparse, 1, False)
 mask = tf.expand_dims(tf.clip_by_value(x_mask + y_mask, 0, 1), axis=-1)
 mask = random_rotate(mask, rotate_angle, [hh, hh, 1])
 mask = tf.image.crop_to_bounding_box(mask,(hh-h)//2,(hh-w)//2, image_height, image_width)
 return mask
def apply_grid_mask(image, image_shape = IMAGE_SIZE):
mask = GridMask(image_shape[0],
image_shape[1],
AugParams['d1'],
AugParams['d2'],
AugParams['rotate'],
AugParams['ratio'])
 if image_shape[-1] == 3:
mask = tf.concat([mask, mask, mask], axis=-1)
 return tf.cast(image * tf.cast(mask, tf.float32), tf.float32 )<normalization>
train = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/train.csv')
 test = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/test.csv')
 sub = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/sample_submission.csv')
 train_paths = train.image_id.apply(format_path ).values
test_paths = test.image_id.apply(format_path ).values
train_labels = train.loc[:, 'healthy':].values
SPLIT_VALIDATION =True
if SPLIT_VALIDATION:
train_paths, valid_paths, train_labels, valid_labels =train_test_split(train_paths, train_labels, test_size=0.15, random_state=seed)
def apply_all_aug(img, no_grid = False, DIM = IMAGE_SIZE[0]):
if tf.random.uniform([],0,1)>.5:
if tf.random.uniform([],0,1)>.5:
img = transform(img)
 else:
img = dropout(img, PROBABILITY = 1)
 else:
if not no_grid:
if tf.random.uniform([],0,1)>.5:
img = apply_grid_mask(img)
 else:
img = img
else:
if tf.random.uniform([],0,1)>.5:
img = transform(img)
 else:
img = img
return tf.reshape(img,[DIM,DIM,3] )<init_hyperparams>
image_size = 800
STEPS_PER_EPOCH = count_data_items(train_files)// BATCH_SIZE
LR_START = 5e-6
LR_MAX = 5e-6 * 8
LR_MIN = 1e-5
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_DECAY =.8
def lr_schedule(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lr_schedule, verbose = True)
 rng = [i for i in range(EPOCHS)]
y = [lr_schedule(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<install_modules>
def decode_image(filename, label=None, image_size=(image_size, image_size)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if label is None:
return image
else:
return image, label
!pip install -q efficientnet 
<choose_model_class>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.cache() 
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 train_dataset_1 =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.cache() 
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(64)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( valid_paths, valid_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.map(data_augment, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )
def get_DenseNet201() :
with strategy.scope() :
dnet = DenseNet201(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'imagenet',
include_top = False 
)
 dnet.trainable = True
model = tf.keras.Sequential([
dnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(1, activation = 'sigmoid')
 ])
 opt = tf.keras.optimizers.Adam(learning_rate = 0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.05)
 model.compile(optimizer = opt,loss = loss,metrics = ['AUC'])
 return model
def get_Xception() :
with strategy.scope() :
xception = Xception(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'imagenet',
include_top = False 
)
 xception.trainable = True
model = tf.keras.Sequential([
xception,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(1, activation = 'sigmoid')
 ])
 opt = tf.keras.optimizers.Adam(learning_rate = 0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.05)
 model.compile(optimizer = opt,loss = loss,metrics = ['AUC'])
 return model
def get_InceptionV3() :
with strategy.scope() :
inception = InceptionV3(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'imagenet',
include_top = False 
)
 inception.trainable = True
model = tf.keras.Sequential([
inception,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(1, activation = 'sigmoid')
 ])
 opt = tf.keras.optimizers.Adam(learning_rate = 0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.05)
 model.compile(optimizer = opt,loss = loss,metrics = ['AUC'])
 return model
def get_InceptionResNetV2() :
with strategy.scope() :
inception_res = InceptionResNetV2(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'imagenet',
include_top = False 
)
 inception_res.trainable = True
model = tf.keras.Sequential([
inception_res,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(1, activation = 'sigmoid')
 ])
 opt = tf.keras.optimizers.Adam(learning_rate = 0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.05)
 model.compile(optimizer = opt,loss = loss,metrics = ['AUC'])
 return model
def get_EfficientNetB4() :
with strategy.scope() :
efficient = efn.EfficientNetB4(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'noisy-student',
include_top = False 
)
 efficient.trainable = True
model = tf.keras.Sequential([
efficient,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(1, activation = 'sigmoid')
 ])
 opt = tf.keras.optimizers.Adam(learning_rate = 0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.05)
 model.compile(optimizer = opt,loss = loss,metrics = ['AUC'])
 return model
def get_EfficientNetB5() :
with strategy.scope() :
efficient = efn.EfficientNetB5(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'noisy-student',
include_top = False 
)
 efficient.trainable = True
model = tf.keras.Sequential([
efficient,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(1, activation = 'sigmoid')
 ])
 opt = tf.keras.optimizers.Adam(learning_rate = 0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.05)
 model.compile(optimizer = opt,loss = loss,metrics = ['AUC'])
 return model
def get_EfficientNetB6() :
with strategy.scope() :
efficient = efn.EfficientNetB6(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'noisy-student',
include_top = False 
)
 efficient.trainable = True
model = tf.keras.Sequential([
efficient,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(1, activation = 'sigmoid')
 ])
 opt = tf.keras.optimizers.Adam(learning_rate = 0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.05)
 model.compile(optimizer = opt, loss = loss,metrics = ['AUC'])
 return model<choose_model_class>
LR_START = 0.0001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.0001
LR_RAMPUP_EPOCHS = 4
LR_SUSTAIN_EPOCHS = 6
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))
histories = []
preds = np.zeros(( count_data_items(test_files),1))
kfold = KFold(FOLDS, shuffle = True, random_state = SEED)
 for f,(train_index,val_index)in enumerate(kfold.split(np.arange(15))):
if DEVICE=='TPU':
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 print('
train_files = tf.io.gfile.glob([GCS_PATH+ '/train%.2i*.tfrec'%x for x in train_index])
 if UPSAMPLE:
train_files += tf.io.gfile.glob([GCS_PATH2 + '/train%.2i*.tfrec'%x for x in train_index])
 np.random.shuffle(train_files)
 val_files = tf.io.gfile.glob([GCS_PATH + '/train%.2i*.tfrec'%x for x in val_index])
 test_files = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')))
 train_ds = get_dataset(train_files,labeled = True, return_image_names = False, all_aug = True,
mat_aug = False, course_drop = False, repeat = True, shuffle = True)
 val_ds = get_dataset(val_files, mat_aug = False, course_drop = False, all_aug = False, grid_mask = False,
repeat = False, shuffle = False, labeled = True, return_image_names = False)
 sv = tf.keras.callbacks.ModelCheckpoint(
 f'fold-{f}.h5', monitor = 'val_loss', verbose = 0, save_best_only = True,
save_weights_only = True, mode = 'min', save_freq = 'epoch')
 print('Training...'); print('')
 model = get_EfficientNetB4()
history = model.fit(train_ds, validation_data = val_ds, callbacks = [sv, lr_callback],
verbose = VERBOSE, steps_per_epoch = STEPS_PER_EPOCH, epochs = EPOCHS 
)
 histories.append(history)
 print('Loading best model...')
 model.load_weights(f'fold-{f}.h5')
 if TTA > 0:
ds_test = get_dataset(test_files,labeled = False,return_image_names = False,mat_aug = True,
course_drop = False, repeat = True,shuffle = False)
 ct_test = count_data_items(test_files); STEPS = TTA * ct_test/BATCH_SIZE
pred = model.predict(ds_test,steps = STEPS,verbose = VERBOSE)[:TTA*ct_test,]
preds[:,0] += np.mean(pred.reshape(( ct_test,TTA),order = 'F'),axis = 1)* 1 / FOLDS
else:
ds_test = get_dataset(test_files,labeled = False,return_image_names = False, mat_aug = False,
course_drop = False, repeat = False, shuffle = False)
 pred = model.predict(ds_test,verbose = VERBOSE)
 preds += pred * 1 / FOLDS
del model; z = gc.collect()<save_to_csv>
ds = get_dataset(test_files, course_drop = False, repeat = False, mat_aug = False,
dim = IMAGE_SIZE[0], labeled = False, return_image_names = True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())])
 submission = pd.DataFrame(dict(image_name = image_names, target = preds[:,0]))
submission = submission.sort_values('image_name')
 submission.to_csv('submission.csv', index = False)
 submission.head(10 )<filter>
with strategy.scope() :
model = tf.keras.Sequential([
InceptionResNetV2(
 input_shape=(image_size, image_size, 3),
weights='imagenet',
include_top=False 
),
L.GlobalMaxPooling2D() ,
L.Dense(4, activation='softmax')
 ])
 model.compile(
 optimizer = 'adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 model.summary()
print(f"Predicted {len(submission[submission['target'] >.5])} images with melanoma" )<install_modules>
STEPS_PER_EPOCH = train_labels.shape[0] // BATCH_SIZE
history = model.fit(
 train_dataset,
epochs=EPOCHS,
callbacks=[lr_callback],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset if SPLIT_VALIDATION else None,
)
!pip install -q efficientnet >> /dev/null<import_modules>
with strategy.scope() :
model2 = tf.keras.Sequential([
efn.EfficientNetB7(
 input_shape=(image_size, image_size, 3),
weights='noisy-student',
include_top=False 
),
L.GlobalMaxPooling2D() ,
L.Dense(4, activation='softmax')
 ])
 model2.compile(
 optimizer = 'adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 model2.summary()
import pandas as pd
import numpy as np
from kaggle_datasets import KaggleDatasets
import tensorflow as tf
import re
import math
import tensorflow.keras.backend as K
import efficientnet.tfkeras as efn
from sklearn.model_selection import KFold
from sklearn.metrics import roc_auc_score
import matplotlib.pyplot as plt
import tensorflow_addons as tfa
import gc<import_modules>
STEPS_PER_EPOCH = train_labels.shape[0] // 64
history = model2.fit(
 train_dataset_1,
epochs=EPOCHS,
callbacks=[lr_callback],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset if SPLIT_VALIDATION else None,
)
print(tf.__version__)
 print(efficientnet.__version__)
 print(sklearn.__version__)
 print(matplotlib.__version__)
 print(tfa.__version__ )<define_variables>
probs1 = model.predict(test_dataset, verbose=1)
 probs2 = model2.predict(test_dataset, verbose=1)
 probs_avg =(probs1+probs2)/2
sub.loc[:, 'healthy':] = probs_avg
sub.to_csv('submission.csv', index=False)
 sub.head()