Kaggle competition: Plant Pathology 2020 - FGVC7
<SOS> metric: MCAUC Kaggle data source: plant-pathology-2020-fgvc7<choose_model_class>
!pip install knockknock
!pip install -q efficientnet
EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3,
efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6]
def build_model(dim=128, ef=0):
inp = tf.keras.layers.Input(shape=(dim,dim,3))
base = EFNS[ef](input_shape=(dim,dim,3),weights='noisy-student',include_top=False)
 x = base(inp)
 x = tf.keras.layers.GlobalAveragePooling2D()(x)
 x = tf.keras.layers.Dense(1,activation='sigmoid' )(x)
 model = tf.keras.Model(inputs=inp,outputs=x)
 opt = tf.keras.optimizers.Adam(learning_rate=0.00001)
 loss = binary_focal_loss()
model.compile(optimizer=opt,loss=loss,metrics=['AUC'])
 return model<init_hyperparams>
user_secrets = UserSecretsClient()
token = user_secrets.get_secret("token")
 chat_id = user_secrets.get_secret("chat_id")
 warnings.filterwarnings('ignore' )
def get_lr_callback(batch_size=8):
lr_start = 0.000005
lr_max = 0.00000125 * REPLICAS * batch_size
lr_min = 0.000001
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.8
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback<define_variables>
AUTO = tf.data.experimental.AUTOTUNE
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
 GCS_DS_PATH = KaggleDatasets().get_gcs_path()
VERBOSE = 2
DISPLAY_PLOT = True
skf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)
 oof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = []
preds = np.zeros(( count_data_items(files_test),1))
for fold,(idxT,idxV)in enumerate(skf.split(np.arange(15))):
if DEVICE=='TPU':
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 print('
print(' 
(IMG_SIZES[fold],EFF_NETS[fold],BATCH_SIZES[fold]*REPLICAS))
files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxT])
 if INC2019[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1])
 print('
if INC2018[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2])
 print('
for k in range(M1[fold]):
files_train += tf.io.gfile.glob([GCS_PATH3[fold] + '/train%.2i*.tfrec'%x for x in idxT])
 print('
for k in range(M2[fold]):
files_train += tf.io.gfile.glob([GCS_PATH3[fold] + '/train%.2i*.tfrec'%x for x in idxT+15])
 print('
for k in range(M3[fold]):
files_train += tf.io.gfile.glob([GCS_PATH3[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1+30])
 print('
for k in range(M4[fold]):
files_train += tf.io.gfile.glob([GCS_PATH3[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+30])
 print('
for k in range(MS[fold]):
files_train += tf.io.gfile.glob([GCS_PATH4[fold] + '/train%.2i*.tfrec'%x for x in idxT])
 print('
np.random.shuffle(files_train); print('
files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxV])
 if INC2019[fold]:
files_valid += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxV*2+1])
 print('
if INC2018[fold]:
files_valid += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxV*2])
 print('
for k in range(M1[fold]):
files_valid += tf.io.gfile.glob([GCS_PATH3[fold] + '/train%.2i*.tfrec'%x for x in idxV])
 print('
for k in range(M2[fold]):
files_valid += tf.io.gfile.glob([GCS_PATH3[fold] + '/train%.2i*.tfrec'%x for x in idxV+15])
 print('
for k in range(M3[fold]):
files_valid += tf.io.gfile.glob([GCS_PATH3[fold] + '/train%.2i*.tfrec'%x for x in idxV*2+1+30])
 print('
for k in range(M4[fold]):
files_valid += tf.io.gfile.glob([GCS_PATH3[fold] + '/train%.2i*.tfrec'%x for x in idxV*2+30])
 print('
for k in range(MS[fold]):
print('
files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/test*.tfrec')))
 K.clear_session()
with strategy.scope() :
model = build_model(dim=IMG_SIZES[fold],ef=EFF_NETS[fold])
 model.load_weights('.. /input/melanomaefnet6/fold-%i.h5'%fold)
 sv = tf.keras.callbacks.ModelCheckpoint(
 'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min', save_freq='epoch')
 print('Training...')
 history = model.fit(
 get_dataset(files_train, augment=True, shuffle=True, repeat=True,
dim=IMG_SIZES[fold], batch_size = BATCH_SIZES[fold]),
epochs=EPOCHS[fold], callbacks = [sv],
steps_per_epoch=count_data_items(files_train)/BATCH_SIZES[fold]//REPLICAS,
validation_data=get_dataset(files_valid,augment=False,shuffle=False,
repeat=False,dim=IMG_SIZES[fold]),
verbose=VERBOSE 
)
 print('Loading best model...')
 model.load_weights('fold-%i.h5'%fold)
 print('Predicting OOF with TTA...')
 ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_valid = count_data_items(files_valid); STEPS = TTA * ct_valid/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,]
print(pred.shape)
 oof_pred.append(np.mean(pred.reshape(( ct_valid,TTA),order='F'),axis=1))
ds_valid = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=True, return_image_names=True)
 oof_tar.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]))
oof_folds.append(np.ones_like(oof_tar[-1],dtype='int8')*fold)
 ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 oof_names.append(np.array([img_name.numpy().decode("utf-8")for img, img_name in iter(ds.unbatch())]))
print('Predicting Test with TTA...')
 ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,]
print(pred.shape)
 preds[:,0] += np.mean(pred.reshape(( ct_test,TTA),order='F'),axis=1)* WGTS[fold]
auc = roc_auc_score(oof_tar[-1],oof_pred[-1])
 oof_val.append(np.max(history.history['val_auc']))
print('
if DISPLAY_PLOT:
plt.figure(figsize=(15,5))
plt.plot(np.arange(EPOCHS[fold]),history.history['auc'],'-o',label='Train AUC',color='
plt.plot(np.arange(EPOCHS[fold]),history.history['val_auc'],'-o',label='Val AUC',color='
x = np.argmax(history.history['val_auc']); y = np.max(history.history['val_auc'])
 xdist = plt.xlim() [1] - plt.xlim() [0]; ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
%.2f'%y,size=14)
 plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)
 plt.legend(loc=2)
 plt2 = plt.gca().twinx()
plt2.plot(np.arange(EPOCHS[fold]),history.history['loss'],'-o',label='Train Loss',color='
plt2.plot(np.arange(EPOCHS[fold]),history.history['val_loss'],'-o',label='Val Loss',color='
x = np.argmin(history.history['val_loss']); y = np.min(history.history['val_loss'])
 ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
plt.ylabel('Loss',size=14)
 plt.title('FOLD %i - Image Size %i, EfficientNet B%i, inc2019=%i, inc2018=%i'% 
(fold+1,IMG_SIZES[fold],EFF_NETS[fold],INC2019[fold],INC2018[fold]),size=18)
 plt.legend(loc=3)
 plt.show() 
<save_to_csv>
SEED = 2020
N_FOLDS = 5
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
SIZE = [800,800]
LR = 0.0008
EPOCHS = 35
WARMUP = 15
oof = np.concatenate(oof_pred); true = np.concatenate(oof_tar);
names = np.concatenate(oof_names); folds = np.concatenate(oof_folds)
 auc = roc_auc_score(true,oof)
 print('Overall OOF AUC with TTA = %.3f'%auc)
 df_oof = pd.DataFrame(dict(
 image_name = names, target=true, pred = oof, fold=folds))
df_oof.to_csv('oof.csv',index=False)
 df_oof.head()<create_dataframe>
def seed_everything(SEED):
np.random.seed(SEED)
 tf.random.set_seed(SEED)
 seed_everything(SEED )
ds = get_dataset(files_test, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )<save_to_csv>
DIR_INPUT = '/kaggle/input/plant-pathology-2020-fgvc7'
train_df = pd.read_csv(DIR_INPUT + '/train.csv')
 test_df = pd.read_csv(DIR_INPUT + '/test.csv')
 cols = list(train_df.columns[1:] )
submission = pd.DataFrame(dict(image_name=image_names, target=preds[:,0]))
submission = submission.sort_values('image_name')
 submission.to_csv('submission.csv', index=False)
 submission.head()<install_modules>
def decode_image(filename, label=None, image_size=(SIZE[0], SIZE[1])) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)
 image = tf.image.per_image_standardization(image)
 image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None, seed=SEED):
image = tf.image.rot90(image,k=np.random.randint(4))
image = tf.image.random_flip_left_right(image, seed=seed)
 image = tf.image.random_flip_up_down(image, seed=seed)
 if label is None:
return image
else:
return image, label
def preprocess(df,test=False):
paths = df.image_id.apply(lambda x: GCS_DS_PATH + '/images/' + x + '.jpg' ).values
labels = df.loc[:,'healthy':].values
if test==False:
return paths,labels
else:
return paths
!pip install -q efficientnet
!pip install -q git+https://github.com/AmedeoBiolatti/dsqol<import_modules>
def get_cosine_schedule_with_warmup(lr,num_warmup_steps, num_training_steps, num_cycles=0.5):

 def lrfn(epoch):
if epoch < num_warmup_steps:
return float(epoch)/ float(max(1, num_warmup_steps)) * lr
progress = float(epoch - num_warmup_steps)/ float(max(1, num_training_steps - num_warmup_steps))
return max(0.0, 0.5 *(1.0 + math.cos(math.pi * float(num_cycles)* 2.0 * progress)))* lr
return tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 lr_schedule= get_cosine_schedule_with_warmup(lr=LR,num_warmup_steps=WARMUP,num_training_steps=EPOCHS )
import os, re, time, tqdm
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import metrics, model_selection
import tensorflow as tf
import tensorflow_addons as tfa
from tensorflow import keras
from tensorflow.keras import backend as K
from efficientnet import tfkeras as efnet
from kaggle_datasets import KaggleDatasets<import_modules>
def getmodel() :
with strategy.scope() :
model = tf.keras.Sequential([
efn.EfficientNetB7(input_shape=(SIZE[0], SIZE[1], 3),weights='imagenet',pooling='avg',include_top=False),
Dense(4, activation='softmax')])
 model.compile(
 optimizer='adam',loss = 'categorical_crossentropy',
metrics=['categorical_accuracy',])
 return model
from dsqol.tf import imgaug
from dsqol.tf.data import balance
from dsqol.tf.utils import average
from dsqol.tf import losses<define_variables>
@telegram_sender(token=token, chat_id=int(chat_id))
def train_one_fold(model,train_dataset,valid_dataset,steps):
history = model.fit(
 train_dataset,
epochs=EPOCHS,
callbacks=[lr_schedule],
steps_per_epoch=steps,
validation_data=valid_dataset)
 string = 'Train acc:{:.4f} Train loss:{:.4f},Val acc:{:.4f} Val loss:{:.4f}'.format(\
model.history.history['categorical_accuracy'][-1],model.history.history['loss'][-1],\
model.history.history['val_categorical_accuracy'][-1],model.history.history['val_loss'][-1])
 return string
SEED = 42
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 TIME_BUDGET = 2.99 * 3600
FOLDS = 5
INCLUDE_2019 = 0
INCLUDE_2018 = 1
INCLUDE_MALIGNANT = 1
IMG_READ_SIZE = 384
IMG_SIZE = 384
BALANCE_POS_RATIO = False
EFF_NET = 5
LOSS_TYPE = 'BCE'
LOSS_PARAMS = dict(label_smoothing=0.05)
 BATCH_SIZE = 32
EPOCHS = 14
TBM = 6
TTA = 20
VALID_FREQ = 1
N_SWA = 3
SWA_DECAY = 0.9<train_on_grid>
folds = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=SEED)
 train_y = np.argmax(train_df[cols].values,axis=1)
 test_pred = np.zeros(( len(test_df),4))
for i_fold,(train_idx, valid_idx)in enumerate(folds.split(train_df,train_y)) :
print("Fold {}/{}".format(i_fold + 1, N_FOLDS))
tf.tpu.experimental.initialize_tpu_system(tpu)
 train = train_df.iloc[train_idx]
train.reset_index(drop=True, inplace=True)
 valid = train_df.iloc[valid_idx]
valid.reset_index(drop=True, inplace=True)
 train_dataset =(tf.data.Dataset 
.from_tensor_slices(preprocess(train)) 
.map(decode_image, num_parallel_calls=AUTO)
.map(data_augment, num_parallel_calls=AUTO)
.shuffle(SEED)
.batch(BATCH_SIZE,drop_remainder=True)
.repeat() 
.prefetch(AUTO))
valid_dataset =(tf.data.Dataset 
.from_tensor_slices(preprocess(valid)) 
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO))
test_dataset =(tf.data.Dataset 
.from_tensor_slices(preprocess(test_df,test=True)) 
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE))
model = getmodel()
steps = train.shape[0] // BATCH_SIZE
string = train_one_fold(model,train_dataset,valid_dataset,steps)
 pred = model.predict(test_dataset)
 test_pred += pred
submission_df = pd.read_csv(DIR_INPUT + '/sample_submission.csv')
 submission_df[['healthy', 'multiple_diseases', 'rust', 'scab']] = pred
submission_df.to_csv(f'submission_{(i_fold+1)}.csv', index=False)
 print(pd.Series(np.argmax(submission_df[cols].values,axis=1)).value_counts())
 del model
gc.collect()
<define_variables><EOS> 
submission_df = pd.read_csv(DIR_INPUT + '/sample_submission.csv')
 submission_df[['healthy', 'multiple_diseases', 'rust', 'scab']] = test_pred/N_FOLDS
submission_df.to_csv('submission.csv', index=False)
 pd.Series(np.argmax(submission_df[cols].values,axis=1)).value_counts()