Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<choose_model_class>
!pip install.. /input/validators
!cp -R.. /input/vit-keras/vit_keras./
class Edge_Regressor(tf.keras.layers.Layer):
def __init__(self, intermediate_dim):
super(Edge_Regressor, self ).__init__()
self.concat_layer = tf.keras.layers.Concatenate()
self.hidden_layer_1 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.hidden_layer_2 = tf.keras.layers.Dense(units=intermediate_dim, activation=tf.nn.relu)
 self.output_layer = tf.keras.layers.Dense(units=1, activation=None)
 def call(self, nodes, edges):
n_nodes = tf.shape(nodes)[1]
node_dim = tf.shape(nodes)[2]
state_i = tf.reshape(tf.tile(nodes, [1, 1, n_nodes]),[-1,n_nodes*n_nodes, node_dim ])
 state_j = tf.tile(nodes, [1, n_nodes, 1])
 concat = self.concat_layer([state_i, edges, state_j])
 activation_1 = self.hidden_layer_1(concat)
 activation_2 = self.hidden_layer_2(activation_1)
 return self.output_layer(activation_2 )<choose_model_class>
Flatten,GlobalAveragePooling2D,BatchNormalization, Activation
print(tf.__version__ )
class MP_Layer(tf.keras.layers.Layer):
def __init__(self, state_dim):
super(MP_Layer, self ).__init__(self)
 self.message_passers = Message_Passer_NNM(node_dim = state_dim)
 self.message_aggs = Message_Agg()
self.update_functions = Update_Func_GRU(state_dim = state_dim)
 self.state_dim = state_dim
def call(self, nodes, edges, mask):
n_nodes = tf.shape(nodes)[1]
node_dim = tf.shape(nodes)[2]
state_j = tf.tile(nodes, [1, n_nodes, 1])
 messages = self.message_passers(state_j, edges)
 masked = tf.math.multiply(messages, mask)
 masked = tf.reshape(masked, [tf.shape(messages)[0], n_nodes, n_nodes, node_dim])
 agg_m = self.message_aggs(masked)
 updated_nodes = self.update_functions(nodes, agg_m)
 nodes_out = updated_nodes
return nodes_out<choose_model_class>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print(f'Running on TPU {tpu.master() }')
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )
adj_input = tf.keras.Input(shape=(None,), name='adj_input')
 nod_input = tf.keras.Input(shape=(None,), name='nod_input')
 class MPNN(tf.keras.Model):
def __init__(self, out_int_dim, state_dim, T):
super(MPNN, self ).__init__(self)
 self.T = T
self.embed = tf.keras.layers.Dense(units=state_dim, activation=tf.nn.relu)
 self.MP = MP_Layer(state_dim)
 self.edge_regressor = Edge_Regressor(out_int_dim)
 def call(self, inputs = [adj_input, nod_input]):
nodes = inputs['nod_input']
edges = inputs['adj_input']
len_edges = tf.shape(edges)[-1]
_, x = tf.split(edges, [len_edges -1, 1], 2)
 mask = tf.where(tf.equal(x, 0), x, tf.ones_like(x))
nodes = self.embed(nodes)
 for mp in range(self.T):
nodes = self.MP(nodes, edges, mask)
 con_edges = self.edge_regressor(nodes, edges)
 return con_edges 
<compute_test_metric>
SEED = 100
DEBUG = False
WANDB = False
VALIDATION_SIZE = 0.2
BATCH_SIZE = 8 *REPLICAS
LEARNING_RATE = 3e-5 * REPLICAS
EPOCHS=40
MODEL_NAME = "VitL16"
N_FOLDS = 5
TTA = False
N_TTA = 3
T_1 = 0.2
T_2 = 1.2
SMOOTH_FRACTION = 0.01
N_ITER = 5
HEIGHT = 512
WIDTH = 512
HEIGHT_RS = 512
WIDTH_RS = 512
CHANNELS = 3
N_CLASSES = 5
MODEL_SAVE_PATH = ""
if DEBUG:
EPOCHS = 2
def mse(orig , preds):
mask = tf.where(tf.equal(orig, 0), orig, tf.ones_like(orig))
nums = tf.boolean_mask(orig, mask)
 preds = tf.boolean_mask(preds, mask)
 reconstruction_error = tf.reduce_mean(tf.square(tf.subtract(nums, preds)))
 return reconstruction_error<compute_test_metric>
def transform_rotation(image, height, rotation):
DIM = height
XDIM = DIM%2
rotation = rotation * tf.random.uniform([1],dtype='float32')
 rotation = math.pi * rotation / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(rotation_matrix,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3])
 def transform_shear(image, height, shear):
DIM = height
XDIM = DIM%2
shear = shear * tf.random.uniform([1],dtype='float32')
 shear = math.pi * shear / 180.
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(shear_matrix,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3])
 def data_augment_cutout(image, min_mask_size=(int(HEIGHT *.1), int(HEIGHT *.1)) ,
max_mask_size=(int(HEIGHT *.125), int(HEIGHT *.125))):
p_cutout = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 if p_cutout >.85:
n_cutout = tf.random.uniform([], 10, 15, dtype=tf.int32)
 image = random_cutout(image, HEIGHT, WIDTH,
min_mask_size=min_mask_size, max_mask_size=max_mask_size, k=n_cutout)
 elif p_cutout >.6:
n_cutout = tf.random.uniform([], 5, 10, dtype=tf.int32)
 image = random_cutout(image, HEIGHT, WIDTH,
min_mask_size=min_mask_size, max_mask_size=max_mask_size, k=n_cutout)
 elif p_cutout >.25:
n_cutout = tf.random.uniform([], 2, 5, dtype=tf.int32)
 image = random_cutout(image, HEIGHT, WIDTH,
min_mask_size=min_mask_size, max_mask_size=max_mask_size, k=n_cutout)
 else:
image = random_cutout(image, HEIGHT, WIDTH,
min_mask_size=min_mask_size, max_mask_size=max_mask_size, k=1)
 return image
def random_cutout(image, height, width, channels=3, min_mask_size=(10, 10), max_mask_size=(80, 80), k=1):
assert height > min_mask_size[0]
assert width > min_mask_size[1]
assert height > max_mask_size[0]
assert width > max_mask_size[1]
for i in range(k):
mask_height = tf.random.uniform(shape=[], minval=min_mask_size[0], maxval=max_mask_size[0], dtype=tf.int32)
 mask_width = tf.random.uniform(shape=[], minval=min_mask_size[1], maxval=max_mask_size[1], dtype=tf.int32)
 pad_h = height - mask_height
pad_top = tf.random.uniform(shape=[], minval=0, maxval=pad_h, dtype=tf.int32)
 pad_bottom = pad_h - pad_top
pad_w = width - mask_width
pad_left = tf.random.uniform(shape=[], minval=0, maxval=pad_w, dtype=tf.int32)
 pad_right = pad_w - pad_left
cutout_area = tf.zeros(shape=[mask_height, mask_width, channels], dtype=tf.uint8)
 cutout_mask = tf.pad([cutout_area], [[0,0],[pad_top, pad_bottom], [pad_left, pad_right], [0,0]], constant_values=1)
 cutout_mask = tf.squeeze(cutout_mask, axis=0)
 image = tf.multiply(tf.cast(image, tf.float32), tf.cast(cutout_mask, tf.float32))
return image
def log_mse(orig , preds):
mask = tf.where(tf.equal(orig, 0), orig, tf.ones_like(orig))
nums = tf.boolean_mask(orig, mask)
 preds = tf.boolean_mask(preds, mask)
 reconstruction_error = tf.math.log(tf.reduce_mean(tf.square(tf.subtract(nums, preds))))
return reconstruction_error<compute_test_metric>
def data_augment(image, label):
p_rotation = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_shear = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_cutout = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 flip_left = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 flip_right = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 if p_rotation >.2:
if p_rotation >.6:
image = transform_rotation(image, HEIGHT, rotation=10.)
else:
image = transform_rotation(image, HEIGHT, rotation=-10.)
if flip_left>0.5:
image = tf.image.random_flip_left_right(image)
 if flip_right>0.5 :
image = tf.image.random_flip_up_down(image)
 image = tf.image.resize(image, size=[HEIGHT, WIDTH])
 return image, label
def mae(orig , preds):
mask = tf.where(tf.equal(orig, 0), orig, tf.ones_like(orig))
nums = tf.boolean_mask(orig, mask)
 preds = tf.boolean_mask(preds, mask)
 reconstruction_error = tf.reduce_mean(tf.abs(tf.subtract(nums, preds)))
 return reconstruction_error<compute_test_metric>
copyfile(src = ".. /input/bitempered-logistic-loss-tensorflow-v2/bi_tempered_loss.py", dst = ".. /working/loss.py")
def log_mae(orig , preds):
mask = tf.where(tf.equal(orig, 0), orig, tf.ones_like(orig))
nums = tf.boolean_mask(orig, mask)
 preds = tf.boolean_mask(preds, mask)
 reconstruction_error = tf.math.log(tf.reduce_mean(tf.abs(tf.subtract(nums, preds))))
return reconstruction_error<init_hyperparams>
with strategy.scope() :
class BiTemperedLogisticLoss(tf.keras.losses.Loss):
def __init__(self, t1, t2, lbl_smth, n_iter):
super(BiTemperedLogisticLoss, self ).__init__()
self.t1 = t1
self.t2 = t2
self.lbl_smth = lbl_smth
self.n_iter = n_iter
def call(self, y_true, y_pred):
return bi_tempered_logistic_loss(y_pred, y_true, self.t1, self.t2, self.lbl_smth, self.n_iter )
learning_rate = 0.001
def step_decay(epoch):
initial_lrate = learning_rate
drop = 0.1
epochs_drop = 20.0
lrate = initial_lrate * np.power(drop,
np.floor(( epoch)/epochs_drop))
tf.print("Learning rate: ", lrate)
 return lrate
lrate = tf.keras.callbacks.LearningRateScheduler(step_decay)
 stop_early = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience = 15, restore_best_weights=True)
 opt = tf.optimizers.Adam(learning_rate=learning_rate)
<choose_model_class>
def get_vit_model(weights = None):
if weights == None:
return vit.vit_l16(
 image_size=HEIGHT,
activation='softmax',
pretrained=False,
include_top=True,
pretrained_top=False,
classes = 5, 
)
 else:
return vit.vit_l16(
 image_size=HEIGHT,
activation='softmax',
pretrained=False,
include_top=True,
pretrained_top=False,
classes = 5,
weights = weights 
)
mpnn = MPNN(out_int_dim = 512, state_dim = 128, T = 4)
 mpnn.compile(opt, log_mae, metrics = [mae, log_mse] )<define_variables>
files_path = '.. /input/cassava-leaf-disease-classification/test_images'
TEST_FILENAMES = tf.io.gfile.glob('.. /input/cassava-leaf-disease-classification/test_tfrecords/*')
 model_path_list = [
'.. /input/cassava-leaf-vit-models/ViTB16_best_fold_0__v3_.h5',
'.. /input/cassava-leaf-vit-models/ViTB16_best_fold_0_v4_.h5',
'.. /input/cassava-leaf-vit-models/ViTB16_best_fold_1_v4_.h5',
'.. /input/cassava-leaf-vit-models/ViTB16_best_fold_2_v4_.h5',
'.. /input/cassava-leaf-vit-models/ViTB16_best_fold_3_v4_.h5'
]
test_preds = np.zeros(( len(os.listdir(files_path)) , N_CLASSES))
for idx, model_path in enumerate(model_path_list):
print("Model: ", model_path)
 tf.keras.backend.clear_session()
model = get_vit_model('imagenet21k')
 model.load_weights(model_path)
 if TTA:
for step in range(N_TTA):
print(f"TTA step: {step+1}/{N_TTA}")
 test_ds = get_dataset(TEST_FILENAMES, labeled=False, ordered=True, augment = True)
 x_test = test_ds.map(lambda image, image_name: image)
 test_preds += model.predict(x_test)
 else:
test_ds = get_dataset(TEST_FILENAMES, labeled=False, ordered=True)
 x_test = test_ds.map(lambda image, image_name: image)
 test_preds += model.predict(x_test )
<split><EOS> 
test_preds = np.argmax(test_preds, axis=-1)
 image_names = [img_name.numpy().decode('utf-8')for img, img_name in iter(test_ds.unbatch())]
submission = pd.DataFrame({'image_id': image_names, 'label': test_preds})
 submission.to_csv('submission.csv', index=False)
 display(submission.head() )