Kaggle competition: Kannada MNIST
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))<categorify>
print(tf.__version__ )
def transform(image,label):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
rot = 15.* tf.random.normal([1],dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image,tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3]),label<categorify>
train = pd.read_csv("/kaggle/input/Kannada-MNIST/train.csv" )
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.one_hot(example['class'], 104)
 return image, label
def read_labeled_tfrecord_train(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.one_hot(example['class'], 104)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def load_dataset_train(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord_train if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def get_training_dataset(do_aug=False):
dataset = load_dataset_train(TRAINING_FILENAMES, labeled=True)
 dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 if do_aug: dataset = dataset.map(transform, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def data_augment(image, label):
image = tf.image.random_flip_left_right(image)
 return image, label
def get_validation_dataset(ordered=False):
dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)
 NUM_VALIDATION_IMAGES =(1 - SKIP_VALIDATION)* count_data_items(VALIDATION_FILENAMES)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES)) 
<train_model>
batch_size = 100
num_class = 10
epochs = 45
img_rows = 28
img_cols = 28
input_shape =(img_rows, img_cols, 1 )
def binary_focal_loss(gamma=2., alpha=.25):

 def binary_focal_loss_fixed(y_true, y_pred):

 pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
epsilon = K.epsilon()
pt_1 = K.clip(pt_1, epsilon, 1.- epsilon)
 pt_0 = K.clip(pt_0, epsilon, 1.- epsilon)
 return -K.sum(alpha * K.pow(1.- pt_1, gamma)* K.log(pt_1)) \
-K.sum(( 1 - alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0))
return binary_focal_loss_fixed
def categorical_focal_loss(gamma=2., alpha=.25):

 def categorical_focal_loss_fixed(y_true, y_pred):

 y_pred /= K.sum(y_pred, axis=-1, keepdims=True)
 epsilon = K.epsilon()
y_pred = K.clip(y_pred, epsilon, 1.- epsilon)
 cross_entropy = -y_true * K.log(y_pred)
 loss = alpha * K.pow(1 - y_pred, gamma)* cross_entropy
return K.sum(loss, axis=1)
 return categorical_focal_loss_fixed 
<choose_model_class>
y = train["label"]
x = train.drop(["label"], axis = 1)
 x_train, x_val, y_train, y_val = sklearn.model_selection.train_test_split(x.values, y.values, test_size = 0.10)
 print(y_train.shape)
 print(x_train.shape)
 print(y_val.shape)
 print(x_val.shape )
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 4
LR_SUSTAIN_EPOCHS = 4
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True )<choose_model_class>
print(x_train.shape[0])
 x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)
 x_train = x_train.astype('float32')
 x_train /= 255
print(x_val.shape[0])
 x_val = x_val.reshape(x_val.shape[0], img_rows, img_cols, 1)
 x_val = x_val.astype('float32')
 x_val /= 255
with strategy.scope() :
desnet=tf.keras.applications.DenseNet201(
 weights='imagenet',
include_top=False 
)
 model1 = tf.keras.Sequential([
desnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model1.compile(
 optimizer=tf.keras.optimizers.Adam(lr=0.0001),
loss = [categorical_focal_loss(gamma=2., alpha=.25)],
metrics=[tfa.metrics.F1Score(num_classes=104, average="macro")] 
)
 model1.summary()
model1.load_weights('.. /input/mytopmodel/weights_dense_22.h5' )<choose_model_class>
y_train = tf.keras.utils.to_categorical(y_train, num_class)
 y_val = tf.keras.utils.to_categorical(y_val, num_class)
 k_init = tf.initializers.TruncatedNormal(mean = 0.1, stddev = 0.05)
 b_init = tf.initializers.constant(value = 1e-4)
with strategy.scope() :
enet = efn.EfficientNetB4(
 input_shape=(img_size, img_size, 3),
weights='noisy-student',
include_top=False 
)
 model2 = tf.keras.Sequential([
enet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model2.compile(
 optimizer=tf.keras.optimizers.Adam(lr=0.0001),
loss = [categorical_focal_loss(gamma=2., alpha=.25)],
metrics=[tfa.metrics.F1Score(num_classes=104, average="macro")] 
)
 model2.summary()
model2.load_weights('.. /input/mytopmodel/weights_efficient_20.h5' )<choose_model_class>
datagen = tf.keras.preprocessing.image.ImageDataGenerator(
 rotation_range=15,
zoom_range = 0.20,
width_shift_range=0.20,
height_shift_range=0.20
)
with strategy.scope() :
enet = efn.EfficientNetB5(
 input_shape=(img_size, img_size, 3),
weights='noisy-student',
include_top=False 
)
 model3 = tf.keras.Sequential([
enet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model3.compile(
 optimizer=tf.keras.optimizers.Adam(lr=0.0001),
loss = [categorical_focal_loss(gamma=2., alpha=.25)],
metrics=[tfa.metrics.F1Score(num_classes=104, average="macro")] 
)
 model3.summary()
model3.load_weights('.. /input/mytopmodel/weights_efficient_24(1 ).h5' )<choose_model_class>
model = tf.keras.Sequential([
tf.keras.layers.Conv2D(32, kernel_size =(3, 3), activation='relu', input_shape = input_shape),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(32, kernel_size =(3, 3), activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(32, kernel_size =(3, 3), activation='relu', strides=2, padding='same'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Conv2D(64, kernel_size =(3, 3), activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(64, kernel_size =(3, 3), activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(64, kernel_size =(3, 3), activation='relu', strides=2, padding='same'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(128, activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(num_class, activation='softmax')
 ] )
with strategy.scope() :
enet = efn.EfficientNetB6(
 input_shape=(img_size, img_size, 3),
weights='noisy-student',
include_top=False 
)
 model4 = tf.keras.Sequential([
enet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model4.compile(
 optimizer=tf.keras.optimizers.Adam(lr=0.0001),
loss = [categorical_focal_loss(gamma=2., alpha=.25)],
metrics=[tfa.metrics.F1Score(num_classes=104, average="macro")] 
)
 model4.summary()
model4.load_weights('.. /input/mytopmodel/weights_efficient_23.h5' )<choose_model_class>
optimizer = tf.keras.optimizers.RMSprop(learning_rate=0.002,
rho=0.9,
momentum=0.1,
epsilon=1e-07,
centered=True,
name='RMSprop' )
model11 = tf.keras.Sequential()
for layer in model1.layers[:-2]:
model11.add(layer)
 for layer in model11.layers:
layer.trainable = False
model22 = tf.keras.Sequential()
for layer in model2.layers[:-2]:
model22.add(layer)
 for layer in model22.layers:
layer.trainable = False
model33 = tf.keras.Sequential()
for layer in model3.layers[:-2]:
model33.add(layer)
 for layer in model33.layers:
layer.trainable = False
model44 = tf.keras.Sequential()
for layer in model4.layers[:-2]:
model44.add(layer)
 for layer in model44.layers:
layer.trainable = False<choose_model_class>
lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='loss',
factor=0.25,
patience=2,
verbose=1,
mode="auto",
min_delta=0.0001,
cooldown=0,
min_lr=0.00001)
 es = tf.keras.callbacks.EarlyStopping(monitor='loss',
mode='min', verbose=1,
patience=300,
restore_best_weights=False )
with strategy.scope() :
x = tf.keras.Input(shape =(img_size, img_size, 3))
x1 = model11(x)
 x2 = model22(x)
 x3 = model33(x)
 x4 = model44(x)
 x5 = tf.keras.layers.concatenate([x1, x2, x3, x4], axis = 3)
 x6 = tf.keras.layers.GlobalAveragePooling2D()(x5)
 x6 = tf.keras.layers.Dropout(0.75 )(x6)
 x6 = tf.keras.layers.Dense(len(CLASSES), activation='softmax' )(x6)
 out = tf.keras.Model(inputs = x, outputs = x6)
 out.compile(
 optimizer=tf.keras.optimizers.Adam(lr=0.0001),
loss = [categorical_focal_loss(gamma=2., alpha=.25)],
metrics=[tfa.metrics.F1Score(num_classes=104, average="macro")] 
)
 out.summary()<save_to_csv>
model.compile(loss= "categorical_crossentropy", optimizer = optimizer, metrics=['accuracy'])
out.load_weights('.. /input/mytopmodel/weights_ensemble_12.h5')
 test_ds = get_test_dataset(ordered=True)
 test_images_ds = test_ds.map(lambda image, idnum: image)
 print('Computing predictions...')
 probabilities = out.predict(test_images_ds)
 predictions = np.argmax(probabilities, axis=-1)
 print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )<install_modules>
history = model.fit_generator(datagen.flow(x_train, y_train, batch_size = batch_size),
epochs = epochs,
steps_per_epoch = 100,
validation_data =(x_val, y_val),
validation_steps=50,
callbacks=[lr, es],
verbose=2 )
!pip install -q efficientnet<import_modules>
scores = model.evaluate(x_val, y_val )
import efficientnet.tfkeras as efn
from tensorflow.keras.applications import InceptionResNetV2
from tensorflow.keras.applications import ResNet152V2
from sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix 
<import_modules>
test=pd.read_csv('.. /input/Kannada-MNIST/test.csv')
 test_id=test.id
test=test.drop('id',axis=1)
 test=test/255
test=test.values.reshape(-1,28,28,1 )
import numpy as np
import pandas as pd
import os 
<import_modules>
y_pre=model.predict(test)
 y_pre=np.argmax(y_pre,axis=1 )
from IPython.display import display
from IPython.core.interactiveshell import InteractiveShell
from sklearn.model_selection import KFold 
<set_options>
sample_sub=pd.read_csv('.. /input/Kannada-MNIST/sample_submission.csv')
 sample_sub['label']=y_pre
sample_sub.to_csv('submission.csv',index=False)
 sample_sub.head()