Kaggle competition: Plant Pathology 2020 - FGVC7
<SOS> metric: MCAUC Kaggle data source: plant-pathology-2020-fgvc7<import_modules>
!pip install -U pip
!pip install -q efficientnet
import lightgbm as lgb
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.metrics import mean_absolute_error<merge>
import pandas as pd
import os, math
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import efficientnet.tfkeras as efn
from kaggle_datasets import KaggleDatasets
import tensorflow.keras.backend as K
from tensorflow.keras.callbacks import LearningRateScheduler
from tensorflow.keras.models import load_model
from sklearn.model_selection import train_test_split
def merge_matchType(x):
if x in {'normal-squad-fpp', 'crashfpp', 'crashtpp', 'normal-duo-fpp',
'flarefpp', 'normal-solo-fpp', 'flaretpp', 'normal-duo',
'normal-squad', 'normal-solo'}:
return 'others'
else:
return x<feature_engineering>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
 GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 100
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
AUG_BATCH = BATCH_SIZE
IMG_SIZE = 512
AUTO = tf.data.experimental.AUTOTUNE
data['matchType'] = data.matchType.apply(merge_matchType)
 data.matchType.unique()<categorify>
def seed_everything(seed=0):
np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
def format_path(s):
return GCS_DS_PATH + "/images/" +s+ ".jpg"
seed_everything(2048 )
data_dumm = pd.get_dummies(data, columns=['matchType'])
 data_dumm.head()<drop_column>
train_csv = pd.read_csv('.. /input/plant-pathology-2020-fgvc7/train.csv')
 test_csv = pd.read_csv('.. /input/plant-pathology-2020-fgvc7/test.csv')
 submission_csv = pd.read_csv('.. /input/plant-pathology-2020-fgvc7/sample_submission.csv')
 train_paths = train_csv.image_id.apply(format_path ).values
test_paths = test_csv.image_id.apply(format_path ).values
train_labels = train_csv.loc[:, 'healthy':].values
train_paths, valid_paths, train_labels, valid_labels = train_test_split(
 train_paths, train_labels, test_size=0.15, random_state=2048 )
data_dumm = data_dumm.drop('matchType_others', axis=1 )<drop_column>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))
def transform2(image, label, DIM = 512):
XDIM = DIM % 2
rot = 15.* tf.random.normal([1],dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image,tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3]),label
def cutmix(image, label, PROBABILITY = 1.0, DIM = None):
CLASSES = 4
imgs = []; labs = []
for j in range(AUG_BATCH):
P = tf.cast(tf.random.uniform([],0,1)<=PROBABILITY, tf.int32)
 k = tf.cast(tf.random.uniform([],0,AUG_BATCH),tf.int32)
 x = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 y = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 b = tf.random.uniform([],0,1)
 WIDTH = tf.cast(DIM * tf.math.sqrt(1-b),tf.int32)* P
ya = tf.math.maximum(0,y-WIDTH//2)
 yb = tf.math.minimum(DIM,y+WIDTH//2)
 xa = tf.math.maximum(0,x-WIDTH//2)
 xb = tf.math.minimum(DIM,x+WIDTH//2)
 one = image[j,ya:yb,0:xa,:]
two = image[k,ya:yb,xa:xb,:]
three = image[j,ya:yb,xb:DIM,:]
middle = tf.concat([one,two,three],axis=1)
 img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)
 imgs.append(img)
 a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)
 if len(label.shape)==1:
lab1 = tf.one_hot(label[j],CLASSES)
 lab2 = tf.one_hot(label[k],CLASSES)
 else:
lab1 = label[j,]
lab2 = label[k,]
labs.append(( 1-a)*lab1 + a*lab2)
 image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))
label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))
return image2,label2
def mixup(image, label, PROBABILITY = 1.0, DIM=None):
CLASSES = 4
imgs = []; labs = []
for j in range(AUG_BATCH):
P = tf.cast(tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)
 k = tf.cast(tf.random.uniform([],0,AUG_BATCH),tf.int32)
 a = tf.random.uniform([],0,1)*P
img1 = image[j,]
img2 = image[k,]
imgs.append(( 1-a)*img1 + a*img2)
 if len(label.shape)==1:
lab1 = tf.one_hot(label[j],CLASSES)
 lab2 = tf.one_hot(label[k],CLASSES)
 else:
lab1 = label[j,]
lab2 = label[k,]
labs.append(( 1-a)*lab1 + a*lab2)
 image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))
label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))
return image2,label2
def transform(image,label, DIM=None):
CLASSES = 4
SWITCH = 0.5
CUTMIX_PROB = 0.666
MIXUP_PROB = 0.666
image2, label2 = cutmix(image, label, CUTMIX_PROB, DIM)
 image3, label3 = mixup(image, label, MIXUP_PROB, DIM)
 imgs = []; labs = []
for j in range(AUG_BATCH):
P = tf.cast(tf.random.uniform([],0,1)<=SWITCH, tf.float32)
 imgs.append(P*image2[j,]+(1-P)*image3[j,])
 labs.append(P*label2[j,]+(1-P)*label3[j,])
 image4 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))
label4 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))
return image4,label4
data = data_dumm.loc[:,['Id', 'groupId', 'matchId', 'assists', 'boosts', 'damageDealt', 'DBNOs',
'headshotKills', 'heals', 'killPlace', 'killPoints', 'kills',
'killStreaks', 'longestKill', 'matchDuration', 'maxPlace', 'numGroups',
'rankPoints', 'revives', 'rideDistance', 'roadKills', 'swimDistance',
'teamKills', 'vehicleDestroys', 'walkDistance', 'weaponsAcquired',
'winPoints', 'matchType_duo', 'matchType_duo-fpp',
'matchType_solo', 'matchType_solo-fpp', 'matchType_squad',
'matchType_squad-fpp', 'winPlacePerc']]<count_unique_values>
def decode_image(filename, label=None, image_size=(512, 512)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None, image_size=(512, 512), seed=2048):
image = tf.image.random_flip_left_right(image,seed=seed)
 image = tf.image.random_flip_up_down(image,seed=seed)
 image = tf.image.random_brightness(image, 0.1, seed=seed)
 image = tf.image.random_hue(image, 0.01)
 image = tf.image.random_saturation(image, 0.7, 1.3)
 image = tf.image.random_contrast(image, 0.8, 1.2)
 image = tf.image.random_brightness(image, 0.1)
 image = tf.image.random_saturation(image, 0, 2)
 if label is None:
return image
else:
return image, label
def load_dataset(train_paths, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
return(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.with_options(ignore_order)
)
 def get_training_dataset(IMG_SIZE=512, do_aug=True, cutmixup=True, mataug=True):
dataset = load_dataset(train_paths)
 if do_aug:
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 if mataug:
dataset = dataset.map(transform2, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
if cutmixup:
dataset = dataset.batch(AUG_BATCH)
 dataset = dataset.map(lambda image, label: transform(image, tf.cast(label, tf.float32), DIM=IMG_SIZE), num_parallel_calls=AUTO)
 dataset = dataset.unbatch()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
print('There are {} unique maxPlace.'.format(len(data.maxPlace.unique())) )<feature_engineering>
valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( valid_paths, valid_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.cache() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )
data_store_by_maxPlace = {}
for x in data.maxPlace.unique() :
data_store_by_maxPlace[x] = data.loc[data.maxPlace==x]<predict_on_test>
with strategy.scope() :
model = tf.keras.Sequential([
efn.EfficientNetB5(include_top=False, weights='noisy-student', input_shape=[512, 512, 3]),
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(1000, use_bias=True, activation='relu'),
tf.keras.layers.Dropout(rate=0.5),
tf.keras.layers.Dense(train_labels.shape[1], activation='softmax')
 ])
 def custom_loss(x, y):
return tf.keras.losses.categorical_crossentropy(x, y, from_logits=False, label_smoothing=0.1)
 model.compile(optimizer='Adam',
loss=custom_loss,
metrics=['categorical_accuracy']
)
def adjust_pred(x, maxPlace):
space = 1/(maxPlace-1)
 return np.round(x / space)* space<split>
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.7
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))
def generate_lgb_model(data, for_eval):
X = data.iloc[:,3:33]
maxPlace = X.maxPlace.unique()
X = X.drop('maxPlace', axis=1)
 Y = data.iloc[:,33]
if for_eval == True:
seed = 7
test_size = 0.33
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=test_size, random_state=seed)
 else:
X_train, y_train = X, Y
d_train = lgb.Dataset(X_train, label=y_train)
 params = {}
params['objective'] = 'regression'
params['metric'] = 'mae'
model = lgb.train(params, d_train)
 if for_eval == True:
y_pred=model.predict(X_test)
 y_pred = adjust_pred(y_pred, maxPlace)
 mae = mean_absolute_error(y_test, y_pred)
 print("MAE for maxPlace = {}: {}".format(maxPlace, mae))
return [model,(list(y_test), y_pred)]
else:
return model<find_best_params>
STEPS_PER_EPOCH = train_labels.shape[0] // BATCH_SIZE
Checkpoint=tf.keras.callbacks.ModelCheckpoint(f"Effnet_best.h5", verbose=1, mode='min', save_best_only=True, monitor='val_loss', save_weights_only=True)
 history = model.fit(get_training_dataset(IMG_SIZE),
epochs=EPOCHS,
validation_data=valid_dataset,
steps_per_epoch = STEPS_PER_EPOCH,
callbacks=[Checkpoint] )
model_store = {}
maxPlace_set = set(data_store_by_maxPlace.keys())
 print('There are {} unique maxPlace.'.format(len(maxPlace_set)))
 for key, value in data_store_by_maxPlace.items() :
model_store[key] = generate_lgb_model(value,for_eval=True)
 maxPlace_set = maxPlace_set - {key}
print('There are {} more models to go.'.format(len(maxPlace_set)) )<define_variables>
probs = model.predict(test_dataset, verbose=1)
 submission_csv.loc[:, 'healthy':] = probs
submission_csv.to_csv('submission.csv', index=False)
 submission_csv.head()
<compute_test_metric><EOS> 
model.load_weights(f"./Effnet_best.h5")
 probs = model.predict(test_dataset, verbose=1)
 submission_csv.loc[:, 'healthy':] = probs
submission_csv.to_csv('submission1.csv', index=False)
 submission_csv.head()