Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<init_hyperparams>
')
 ')
 ')
def set_learning_rate(optimizer, lr):
for param_group in optimizer.param_groups:
param_group['lr'] = lr
def get_learning_rate(optimizer):
return optimizer.param_groups[0]['lr']
class LearningRate() :
def __init__(self, initial_lr, iteration_type):
self.initial_lr = initial_lr
self.iteration_type = iteration_type
def get_learning_rate(self, optimizer):
return optimizer.param_groups[0]['lr']
def set_learning_rate(self, optimizer, new_lr):
for param_group in optimizer.param_groups:
param_group['lr'] = new_lr
def adjust(self, optimizer, lr, iteration, params=None):
self.set_learning_rate(optimizer, lr)
 return lr
class FixedLR(LearningRate):
def __init__(self, initial_lr, iteration_type):
super().__init__(initial_lr, iteration_type)
 def adjust(self, optimizer, iteration, params=None):
new_lr = super().get_learning_rate(optimizer)
 return new_lr
class LinearLR(LearningRate):
def __init__(self, initial_lr, iteration_type, fixed_delta):
super().__init__(initial_lr, iteration_type)
 self.fixed_delta = fixed_delta
def adjust(self, optimizer, iteration, params=None):
lr = super().get_learning_rate(optimizer)
 new_lr = lr + self.fixed_delta
super().set_learning_rate(optimizer, new_lr)
 return new_lr
class SnapshotLR(LearningRate):

 def __init__(self, initial_lr, iteration_type,
max_lr, total_iters, n_cycles):

 super().__init__(initial_lr, iteration_type)
 self.max_lr = max_lr
self.total_iters = total_iters
self.cycles = n_cycles
def cosine_annealing(self, t):

 return self.max_lr/2 *(math.cos(
(math.pi *(t %(self.total_iters//self.cycles)))/ 
(self.total_iters//self.cycles)) + 1)
 def adjust(self, optimizer, iteration, params=None):
new_lr = self.cosine_annealing(iteration)
 self.set_learning_rate(optimizer, new_lr)
 return new_lr
class SnapshotParamsLR(LearningRate):

 def __init__(self, initial_lr, iteration_type,
total_iters, n_cycles):

 super().__init__(initial_lr, iteration_type)
 self.total_iters = total_iters
self.cycles = n_cycles
def cosine_annealing(self, t, max_lr):
return max_lr/2 *(math.cos(
(math.pi *(t %(self.total_iters//self.cycles)))/(
 self.total_iters//self.cycles)) + 1)
 def adjust(self, optimizer, iteration, params=None):
lrs = []
for param_group in optimizer.param_groups:
new_lr = self.cosine_annealing(iteration, param_group['max_lr'])
 param_group['lr'] = new_lr
lrs.append(new_lr)
 return new_lr
class DevDecayLR(LearningRate):

 def __init__(self, initial_lr, iteration_type,
decay_factor=0.9, decay_patience=1):
super().__init__(initial_lr, iteration_type)
 self.decay_factor = decay_factor
self.decay_patience = decay_patience
def adjust(self, optimizer, iteration, params):
lr = super().get_learning_rate(optimizer)
 best_iter = params['best_iter']
if(iteration - best_iter)> self.decay_patience:
print('Decaying learning rate by factor: {:.5f}'.format(
 self.decay_factor ).rstrip('0'))
lr *= self.decay_factor
super().set_learning_rate(optimizer, lr)
 return lr
class ScheduledLR(LearningRate):
def __init__(self, initial_lr, iteration_type, lr_schedule):
super().__init__(initial_lr, iteration_type)
 self.lr_schedule = lr_schedule
def adjust(self, optimizer, iteration, params=None):
if iteration in self.lr_schedule:
new_lr = self.lr_schedule[iteration]
else:
new_lr = self.get_learning_rate(optimizer)
 super().set_learning_rate(optimizer, new_lr)
 return new_lr
class DecayingLR(LearningRate):
def __init__(self, initial_lr, iteration_type, decay, n_epochs):
super().__init__(initial_lr, iteration_type)
 self.decay = decay
self.n_epochs = n_epochs
def exponential_decay(self, iteration, params=None):

 return self.initial_lr *(self.decay **(iteration // self.n_epochs))
def adjust(self, optimizer, iteration):
new_lr = self.exponential_decay(iteration)
 super().set_learning_rate(optimizer, new_lr)
 return new_lr
class CyclicalLR(LearningRate):

 def __init__(self, initial_lr, iteration_type, n_iters, cycle_length,
min_lr, max_lr):
assert initial_lr == min_lr
super(CyclicalLR, self ).__init__(initial_lr, iteration_type)
 self.n_iters = n_iters
self.cycle_length = cycle_length
self.min_lr = min_lr
self.max_lr = max_lr
def triangular(self, iteration):
iteration -= 1
cycle = math.floor(1 + iteration/self.cycle_length)
 x = abs(iteration/(self.cycle_length/2)- 2*cycle + 1)
 new_lr = self.min_lr +(self.max_lr - self.min_lr)* max(0,(1-x))
return new_lr
def adjust(self, optimizer, iteration, best_iter=1):
new_lr = self.triangular(iteration)
 super().set_learning_rate(optimizer, new_lr)
 return new_lr
def cosine_annealing(lr_max, T, M, t):

 return lr_max/2 *(math.cos(( math.pi *(t %(T//M)))/(T//M)) + 1 )<load_pretrained>
!pip install.. /input/cassava-models/Keras_Applications-1.0.8-py3-none-any.whl
!pip install.. /input/cassava-models/efficientnet-1.1.0-py3-none-any.whl
def save_checkpoint(checkpoint_path, model, optimizer):
state = {'state_dict': model.state_dict() ,
'optimizer' : optimizer.state_dict() }
torch.save(state, checkpoint_path)
 print('model saved to %s' % checkpoint_path)
 def load_checkpoint(checkpoint_path, model, optimizer):
state = torch.load(checkpoint_path)
 model.load_state_dict(state['state_dict'])
 optimizer.load_state_dict(state['optimizer'])
 print('model loaded from %s' % checkpoint_path )<choose_model_class>
AUTO = tf.data.experimental.AUTOTUNE
EPOCHS = 20
BATCH_SIZE = 32 * strategy.num_replicas_in_sync
IMAGE_SIZE = [512, 512]
SEED = 123
LR = 0.0001
TTA = 10
VERBOSE = 2
N_CLASSES = 5
TEST_FILENAMES = '.. /input/cassava-leaf-disease-classification/test_images/*.jpg'
def train_model(model,epochs,
learning_rate,loss_function,
optimizer, dataset, dataset_val,
batch_size 
):

 snapshot = SnapshotLR(
 initial_lr=0.000001,max_lr=0.0001,
total_iters=epochs,n_cycles=30,
iteration_type="epochs" 
)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode="max",
factor=0.97,
min_lr=0,
patience=10 
)
 lowest_train_loss = None
lowest_val_loss = None
highest_train_iou = None
highest_val_iou = None
for epoch in range(epochs):
train_loss = []
train_iou = []
for image, mask in tqdm_notebook(data.DataLoader(dataset,batch_size=batch_size,shuffle=True)) :
image = image.type(torch.FloatTensor ).cuda()
y_pred = model(Variable(image))
loss = loss_function(y_pred, Variable(mask.cuda()))
iou_train = iou_metric_batch(mask.cpu().numpy() ,y_pred.cpu().detach().numpy())
 optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm(model.parameters() , 1.0)
 optimizer.step()
train_loss.append(loss.data[0])
 train_iou.append(iou_train)
 val_loss = []
val_iou = []
for image, mask in data.DataLoader(dataset_val, batch_size = batch_size, shuffle = False):
image = image.cuda()
y_pred = model(Variable(image))
loss = loss_function(y_pred, Variable(mask.cuda()))
val_loss.append(loss.data[0])
 iou_val = iou_metric_batch(mask.cpu().numpy() ,y_pred.cpu().detach().numpy())
 val_iou.append(iou_val)
 if lowest_train_loss is None:
lowest_train_loss = np.mean(train_loss)
 save_checkpoint('low_train_loss_tgs.pth' ,model, optimizer)
 if lowest_val_loss is None:
lowest_val_loss = np.mean(val_loss)
 save_checkpoint('low_val_loss_tgs.pth' ,model, optimizer)
 if highest_train_iou is None:
highest_train_iou = np.mean(train_iou)
 save_checkpoint('high_train_iou_tgs.pth' ,model, optimizer)
 if highest_val_iou is None:
highest_val_iou = np.mean(val_iou)
 save_checkpoint('high_val_iou_tgs.pth' ,model, optimizer)
 print("Epoch: %d,T_Loss: %.7f, T_IOU: %.7f, V_Loss: %.7f, V_IOU: %.7f" %(
 epoch,
np.mean(train_loss),
np.mean(train_iou),
np.mean(val_loss),
np.mean(val_iou)
)
)
 if np.mean(train_loss)< lowest_train_loss:
lowest_train_loss = np.mean(train_loss)
 save_checkpoint('low_train_loss_tgs.pth' ,model, optimizer)
 if np.mean(val_loss)< lowest_val_loss:
lowest_val_loss = np.mean(val_loss)
 save_checkpoint('low_val_loss_tgs.pth' ,model, optimizer)
 if np.mean(train_iou)> highest_train_iou:
highest_train_iou = np.mean(train_iou)
 save_checkpoint('high_train_iou_tgs.pth' ,model, optimizer)
 if np.mean(val_iou)> highest_val_iou:
highest_val_iou = np.mean(val_iou)
 save_checkpoint('high_val_iou_tgs.pth' ,model, optimizer)
 scheduler.step(highest_val_iou)
 resulting_lr = snapshot.adjust(optimizer,epoch)
 print("LR",resulting_lr)
 save_checkpoint('tgs-%i.pth' % epochs, model, optimizer)
 print("Best T_L:",lowest_train_loss,
" Best T_I:",highest_train_iou,
" Best V_L:",lowest_val_loss,
" Best V_I:", highest_val_iou 
)
 return model<categorify>
def data_augment(image, image_name):
p_spatial = tf.random.uniform([], 0, 1.0, dtype = tf.float32)
 p_rotate = tf.random.uniform([], 0, 1.0, dtype = tf.float32)
 p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype = tf.float32)
 p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype = tf.float32)
 p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype = tf.float32)
 p_crop = tf.random.uniform([], 0, 1.0, dtype = tf.float32)
 image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if p_spatial > 0.75:
image = tf.image.transpose(image)
 if p_rotate > 0.75:
image = tf.image.rot90(image, k = 3)
 elif p_rotate > 0.5:
image = tf.image.rot90(image, k = 2)
 elif p_rotate > 0.25:
image = tf.image.rot90(image, k = 1)
 if p_pixel_1 >= 0.4:
image = tf.image.random_saturation(image, lower = 0.7, upper = 1.3)
 if p_pixel_2 >= 0.4:
image = tf.image.random_contrast(image, lower = 0.8, upper = 1.2)
 if p_pixel_3 >= 0.4:
image = tf.image.random_brightness(image, max_delta = 0.1)
 if p_crop > 0.7:
if p_crop > 0.9:
image = tf.image.central_crop(image, central_fraction = 0.7)
 elif p_crop > 0.8:
image = tf.image.central_crop(image, central_fraction = 0.8)
 else:
image = tf.image.central_crop(image, central_fraction = 0.9)
 elif p_crop > 0.4:
crop_size = tf.random.uniform([], int(IMAGE_SIZE[0] * 0.8), IMAGE_SIZE[0], dtype = tf.int32)
 image = tf.image.random_crop(image, size = [crop_size, crop_size, 3])
 image = tf.image.resize(image, size = IMAGE_SIZE)
 image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image, image_name
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels = 3)
 image = tf.image.resize(image, IMAGE_SIZE)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def get_image_name(file_path):
parts = tf.strings.split(file_path, os.path.sep)
 image_name = parts[-1]
return image_name
def read_image(file_path):
image_name = get_image_name(file_path)
 image = tf.io.read_file(file_path)
 image = decode_image(image)
 return image, image_name
def get_test_dataset(filenames, tta = False):
dataset = tf.data.Dataset.list_files(filenames, shuffle = False)
 dataset = dataset.map(read_image, num_parallel_calls = AUTO)
 dataset = dataset.map(data_augment, num_parallel_calls = AUTO)
 if tta:
dataset = dataset.repeat()
dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
NUM_TESTING_IMAGES = len(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
<categorify><EOS> 
def get_model() :
with strategy.scope() :
inp = tf.keras.layers.Input(shape =(*IMAGE_SIZE, 3))
x = efn.EfficientNetB5(weights = None, include_top = False )(inp)
 x = tf.keras.layers.GlobalAveragePooling2D()(x)
 x = tf.keras.layers.Dropout(0.2 )(x)
 output = tf.keras.layers.Dense(N_CLASSES, activation = 'softmax' )(x)
 model = tf.keras.models.Model(inputs = [inp], outputs = [output])
 opt = tf.keras.optimizers.Adam(learning_rate = LR)
 model.compile(
 optimizer = opt,
loss = [tf.keras.losses.CategoricalCrossentropy(label_smoothing = 0.4)],
metrics = [tf.keras.metrics.CategoricalAccuracy() ] 
)
 return model
def inference(model_paths):
prediction = np.zeros(( NUM_TESTING_IMAGES, N_CLASSES))
print('Extracting test image names...')
 test_dataset = get_test_dataset(TEST_FILENAMES, tta = False)
 image_name = test_dataset.map(lambda image, image_name: image_name ).unbatch()
image_name = next(iter(image_name.batch(NUM_TESTING_IMAGES)) ).numpy().astype('U')
 print('Test image names completed...')
 for fold, model_path in enumerate(model_paths):
print('
')
 print('-'*50)
 print(f'Predicting fold {fold + 1}')
 K.clear_session()
model = get_model()
model.load_weights(model_path)
 steps = TTA *(( NUM_TESTING_IMAGES / BATCH_SIZE)+ 1)
 test_dataset = get_test_dataset(TEST_FILENAMES, tta = True)
 image = test_dataset.map(lambda image, image_name: image)
 probabilities = model.predict(image, steps = steps)[: TTA * NUM_TESTING_IMAGES]
probabilities = np.mean(probabilities.reshape(( NUM_TESTING_IMAGES, TTA, N_CLASSES), order = 'F'), axis = 1)
 prediction += probabilities / len(model_paths)
 sub = pd.DataFrame({'image_id': image_name, 'label': np.argmax(prediction, axis = -1)})
 sub.to_csv('submission.csv', index = False)
 return image_name, prediction, sub
model_paths = glob.glob('.. /input/cassava-models/*.h5')
 image_name, prediction, sub = inference(model_paths)
 sub