Kaggle competition: Plant Pathology 2020 - FGVC7
<SOS> metric: MCAUC Kaggle data source: plant-pathology-2020-fgvc7<define_variables>
!pip install efficientnet
def collect(df_list):
return pd.concat(df_list,axis=1)
 sub_df = collect(sub_list)
 oof_df = collect(oof_list)
 oof_df = oof_df.dropna()
ids = oof_df.index
oof = np.array(oof_df['pred'])
 labels = np.array(oof_df['target'])[:,0]
print('There are {} values overlapping'.format(len(oof_df)))
 idsO = sub_df.index
data_sub = np.array(sub_df )<load_pretrained>
import math, re, os
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from kaggle_datasets import KaggleDatasets
import tensorflow as tf
import tensorflow.keras.layers as L
import efficientnet.tfkeras as efn
from sklearn import metrics
from sklearn.model_selection import train_test_split
from keras.callbacks import ModelCheckpoint, EarlyStopping
DIM = 256; EFFN = 0; BATCH_SIZE = 128
PATH_TO_EMBEDDINGS = '.. /input/embeddingsmelanoma/'
embed = np.load(PATH_TO_EMBEDDINGS+'embed_train_%i_%i.npy'%(DIM,EFFN))
names = np.load(PATH_TO_EMBEDDINGS+'names_train.npy')
 embed_test = np.load(PATH_TO_EMBEDDINGS+'embed_test_%i_%i.npy'%(DIM,EFFN))
names_test = np.load(PATH_TO_EMBEDDINGS+'names_test.npy' )<load_from_csv>
GCS_DS_PATH = KaggleDatasets().get_gcs_path('plant-pathology-2020-fgvc7')
 EPOCHS = 200
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
IM_Z = 768
test = pd.read_csv('.. /input/siim-isic-melanoma-classification/test.csv' ).set_index('image_name',drop=True)
 test = test.loc[names_test]
test = pd.concat(( test,sub_df),axis=1 ).reset_index()
print('Test csv shape',test.shape)
 train = pd.read_csv('.. /input/melanoma-%ix%i/train.csv'%(DIM,DIM)).set_index('image_name',drop=True)
 train = train.loc[names]
train = pd.concat(( train,oof_df['pred']),axis=1 ).reset_index()
train.target = train.target.astype('float32')
 print('Train csv shape',train.shape)
 print('Displaying train.csv below...')
 train.head()<categorify>
def format_path(st):
return GCS_DS_PATH + '/images/' + st + '.jpg'
cat_enc = OneHotEncoder(drop='first')
 num_enc = StandardScaler()
embed_enc = MinMaxScaler()
numeric_features = ['age_approx']
cat_features = ['sex','anatom_site_general_challenge']
cats = cat_enc.fit_transform(train[cat_features].fillna('0')).toarray()
nums = num_enc.fit_transform(train[numeric_features].fillna(0))
embed_ENC = embed_enc.fit_transform(embed)
 Xtrain = np.concatenate(( cats,nums,embed_ENC),axis=1)
 Xpreds = np.array(train['pred'])
 cats = cat_enc.transform(test[cat_features].fillna('0')).toarray()
nums = num_enc.transform(test[numeric_features].fillna(0))
embed_encT = embed_enc.transform(embed_test)
 Xtest = np.concatenate(( cats,nums,embed_encT),axis=1)
 Xtargets = np.array(test['target'] )<define_variables>
train = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/train.csv')
 test = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/test.csv')
 sub = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/sample_submission.csv')
 train_paths = train.image_id.apply(format_path ).values
test_paths = test.image_id.apply(format_path ).values
N = len(train_paths)
 print(f"- Độ dài bộ train: {len(train_paths)}")
 print(f"- Độ dài bộ test: {len(test_paths)}")
 N_healthy = train['healthy'].mean() * N
N_multiple_diseases = train['multiple_diseases'].mean() * N
N_rust = train['rust'].mean() * N
N_scab = train['scab'].mean() * N
print(f'Healthy: {N_healthy} mẫu')
 print(f'Multiple diseases: {N_multiple_diseases} mẫu')
 print(f'Rust: {N_rust} mẫu')
 print(f'Scab: {N_scab} mẫu')
 train.describe()
weights = {0:1, 1:15}
FOLDS=5
SEED=42
DISPLAY_PLOT = 1
REPLICAS=1
EPOCHS=100
TTA=11
batch_size = 64
VERBOSE=0<define_variables>
train_labels = train.loc[:, 'healthy':].values
print(f"- Train labels:
{train_labels}")
 train_paths, valid_paths, train_labels, valid_labels = train_test_split(
 train_paths, train_labels, test_size=0.3, random_state=2020)
 print(f" - Độ dài bộ train: {len(train_paths)}")
 print(f" - Độ dài bộ validation: {len(valid_paths)}" )
tf.random.set_seed(5);<normalization>
def decode_image(filename, label=None, image_size=(IM_Z, IM_Z)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if label is None:
return image
else:
return image, label
def data_augment(data,mean=0.0,std1=embed_std/10,std2=0.03):
gauss = np.random.normal(loc=mean, scale=std1, size=Xtrain.shape[1])
 new_data = data[0] + gauss
gauss = np.random.normal(loc=mean, scale=std2, size=Xpreds.shape[1])
 new_preds = data[1] + gauss
return(new_data,new_preds )<create_dataframe>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.cache() 
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( valid_paths, valid_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )
def get_dataset(X,subs,y,augment=True,repeat=True,batch=batch_size):
ds = tf.data.Dataset.from_tensor_slices(((X,subs), y))
if repeat:
ds = ds.repeat()
if augment:
ds = ds.map(lambda elem,label:(data_augment(elem),label))
ds = ds.batch(batch)
 return ds<choose_model_class>
def build_lrfn(lr_start=0.00001, lr_max=0.000075,
lr_min=0.000001, lr_rampup_epochs=20,
lr_sustain_epochs=0, lr_exp_decay=.8):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn
ch_p = ModelCheckpoint(filepath="model_ef1.h5", monitor='val_loss', save_weights_only=True, save_best_only=True, verbose=1 )
dim = Xtrain.shape[1]
meta_dim = Xpreds.shape[1]
def build_model(ls=0.05):
inp = tf.keras.layers.Input(shape=(dim,))
x = L.Dropout(0.2 )(inp)
 x = L.Dense(int(1024), activation='relu' )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Dense(int(512), activation='relu' )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Dense(int(256), activation='relu' )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Dense(int(128), activation='relu' )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Dense(64, activation='relu' )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Dense(32, activation='relu' )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Dense(16, activation='relu' )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Dense(12, activation='relu' )(x)
 x = L.Dropout(0.3 )(x)
 meta_inp = tf.keras.layers.Input(shape=(meta_dim,))
xm = L.concatenate(( x,meta_inp))
xm = L.Dense(16, activation='relu' )(xm)
 xm = L.Dropout(0.2 )(xm)
 xm = L.Dense(12, activation='relu' )(xm)
 xm = L.Dropout(0.2 )(xm)
 xm = L.Dense(8, activation='relu' )(xm)
 xm = L.Dropout(0.1 )(xm)
 xm = L.Dense(1, activation='sigmoid' )(xm)
 model = tf.keras.Model(inputs=(inp,meta_inp), outputs=xm)
 opt = tf.keras.optimizers.Adam(learning_rate=0.00000125* REPLICAS * batch_size)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=ls)
 model.compile(optimizer=opt,loss=loss,metrics=['AUC'])
 return model<init_hyperparams>
lrs = []
lrnfn = build_lrfn()
for epoch in range(0, 100):
lr = lrnfn(epoch)
 lrs.append(lr)
 plt.plot(lrs )
def get_lr_callback(batch_size=8):
lr_start = 0.000005
lr_max = 0.00000125 * REPLICAS * batch_size
lr_min = 0.000001
lr_ramp_ep = 50
lr_sus_ep = 0
lr_decay = 0.99
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback<define_variables>
with strategy.scope() :
model = tf.keras.Sequential([
efn.EfficientNetB7(
 input_shape=(IM_Z, IM_Z, 3),
weights='imagenet',
include_top=False 
),
L.GlobalAveragePooling2D() ,
L.Dense(train_labels.shape[1], activation='softmax')
 ])
 model.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 model.summary()
ids = train['index']
labels = train.target.astype('int32')
 idsO = test['index']
data_sub = Xtest
strategy = tf.distribute.get_strategy()
skf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)
 oof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = []
preds = np.zeros(( test.shape[0],1))
preds_all = np.zeros(( test.shape[0],FOLDS))
for fold,(idxT2,idxV2)in enumerate(skf.split(np.arange(15))):
idxT = train.loc[train.tfrecord.isin(idxT2)].index.values
idxV = train.loc[train.tfrecord.isin(idxV2)].index.values
X = Xtrain[idxT]; y = labels[idxT]; XX=Xpreds[idxT]
X_val = Xtrain[idxV]; y_val = labels[idxV]; XX_val=Xpreds[idxV]
tf.keras.backend.clear_session()
with strategy.scope() :
model=build_model()
sv = tf.keras.callbacks.ModelCheckpoint(
 'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min', save_freq='epoch')
 history = model.fit(get_dataset(X,XX,y),epochs=EPOCHS,steps_per_epoch=X.shape[0]/batch_size//REPLICAS,
verbose=VERBOSE,class_weight=weights,
callbacks=[sv],
validation_data=get_dataset(X_val,XX_val, y_val,augment=False,repeat=False))
model.load_weights('fold-%i.h5'%fold)
 STEPS = TTA*X_val.shape[0]/(batch_size-1)/REPLICAS
pred = model.predict(get_dataset(X_val,XX_val,y_val), steps=STEPS)[:TTA*X_val.shape[0]]
oof_pred.append(np.mean(pred.reshape(( X_val.shape[0],TTA),order='F'),axis=1))
oof_tar.append(y_val)
 oof_names.append(ids[idxV])
 oof_folds.append(np.ones_like(oof_tar[-1],dtype='int8')*fold)
 STEPS = TTA*Xtest.shape[0]/(batch_size-1)/REPLICAS
psub = model.predict(get_dataset(Xtest,Xtargets,np.zeros(len(Xtest))), steps=STEPS)[:TTA*Xtest.shape[0]]
pstore = np.mean(psub.reshape(( len(Xtest),TTA),order='F'),axis=1)
 preds[:,0] += pstore*1/FOLDS
auc = roc_auc_score(oof_tar[-1],oof_pred[-1])
 oof_val.append(np.max(history.history['val_auc']))
print('
if DISPLAY_PLOT:
plt.figure(figsize=(15,5))
plt.plot(np.arange(EPOCHS),history.history['auc'],'-o',label='Train AUC',color='
plt.plot(np.arange(EPOCHS),history.history['val_auc'],'-o',label='Val AUC',color='
x = np.argmax(history.history['val_auc']); y = np.max(history.history['val_auc'])
 xdist = plt.xlim() [1] - plt.xlim() [0]; ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
%.2f'%y,size=14)
 plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)
 plt.legend(loc=2)
 plt2 = plt.gca().twinx()
plt2.plot(np.arange(EPOCHS),history.history['loss'],'-o',label='Train Loss',color='
plt2.plot(np.arange(EPOCHS),history.history['val_loss'],'-o',label='Val Loss',color='
x = np.argmin(history.history['val_loss']); y = np.min(history.history['val_loss'])
 ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
plt.ylabel('Loss',size=14)
 plt.legend(loc=3)
 plt.show()
oof_c = np.concatenate(oof_pred); true = np.concatenate(oof_tar);
names = np.concatenate(oof_names); folds = np.concatenate(oof_folds)
 auc = roc_auc_score(true,oof_c)
 print('Overall OOF AUC with TTA = %.5f'%auc)
 dd_oof = pd.DataFrame(dict(image_name = names, target=true, pred = oof_c, fold=folds))
submission = pd.DataFrame(dict(image_name=idsO, target=preds[:,0]))
submission = submission.sort_values('image_name')
 submission.to_csv('submission.csv', index=False)
 submission.head()<import_modules>
lrfn = build_lrfn()
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=0)
 STEPS_PER_EPOCH = train_labels.shape[0] // BATCH_SIZE
EarlyStop_callback = EarlyStopping(monitor='val_loss', patience=20, restore_best_weights=True )
import pandas as pd<import_modules>
history = model.fit(
 train_dataset,
epochs=EPOCHS,
callbacks=[ch_p, EarlyStop_callback],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset
)
import pandas as pd<load_from_csv>
model.load_weights("model_ef1.h5" )
df_1 = pd.read_csv('/kaggle/input/model-melanoma-2020/submission.csv')
 df_2a = pd.read_csv('/kaggle/input/model-metadata-melanoma-2020/submission_best_mean.csv')
 df_2b = pd.read_csv('/kaggle/input/model-metadata-melanoma-2020/submission_ensemble_mean.csv')
 df_2c = pd.read_csv('/kaggle/input/model-metadata-melanoma-2020/submission_weighted_ensemble_mean.csv')
 df_3a = pd.read_csv('/kaggle/input/melanoma-effnet-metdata/blended_effnets.csv')
 df_3b = pd.read_csv('/kaggle/input/melanoma-effnet-metdata/ensembled.csv')
 df_4 = pd.read_csv('/kaggle/input/minmax-melanoma-9619/submission.csv')
 df_5 = pd.read_csv('/kaggle/input/melanoma-2020-9648/submission_9648.csv' )<prepare_output>
def dummy_data_augment(image):
return image
data_augments = [dummy_data_augment, tf.image.flip_left_right, tf.image.flip_up_down]
df_submission1 = df_1.copy()
df_submission1['target'] = 0.41 * df_1['target'] + 0.04 * df_2c['target'] + 0.55 * df_3b['target']<save_to_csv>
TTA_test_datasets = [(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
.map(d_augment, num_parallel_calls=AUTO)
)for d_augment in data_augments]
df_submission1.to_csv('submission1.csv', index=False)
 df_submission1<prepare_output>
combined_probs_TTAs = [model.predict(dataset)for dataset in TTA_test_datasets]
df_submission2 = df_1.copy()
df_submission2['target'] = 0.075 * df_1['target'] + 0.1 * df_3b['target'] + 0.375 * df_4['target'] + 0.45 * df_5['target']<save_to_csv>
submission_name = 'submission_eff_TTA'
for d_aug in data_augments:
submission_name += "-"
submission_name += d_aug.__name__
submission_name += ".csv"
submission_name
<prepare_output><EOS> 
sub.loc[:, 'healthy':] = probs_TTA
sub.to_csv(submission_name, index=False)
 sub.head()