Kaggle competition: Petals to the Metal - Flower Classification on TPU
x_train, x_test, y_train, y_test = train_test_split(train[train_x_calc], train[Target], test_size = 0.22, random_state = 0)
 x_train_bin, x_test_bin, y_train_bin, y_test_bin = train_test_split(train[train_x_bin], train[Target], test_size = 0.22, random_state = 0)
 x_train_dummy, x_test_dummy, y_train_dummy, y_test_dummy = \
train_test_split(train_dummy[train_x_dummy], train[Target], test_size = 0.22, random_state = 0)
 print("Original Train Shape: {}".format(train.shape))
print("Train Shape: {}".format(np.asarray(x_train ).shape))
print("Test Shape: {}".format(np.asarray(x_test ).shape))<normalization>
import os
import numpy as np
import torch
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
import pandas as pd
from kaggle_datasets import KaggleDatasets
import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from PIL import Image, ImageFile
import matplotlib.pyplot as plt
<train_model>
if 'google.colab' in sys.modules:
%tensorflow_version 2.x
print("Tensorflow version " + tf.__version__)
 AUTO = tf.data.experimental.AUTOTUNE
gaussian = GaussianNB()
gaussian.fit(x_train_dummy, y_train_dummy)
 y_hat_train = gaussian.predict(x_train_dummy)
 y_hat_test = gaussian.predict(x_test_dummy)
 acc_gaussian = gaussian.score(x_test_dummy, y_test_dummy)
 print('Gaussian Naive Bayes train accuracy score: ', gaussian.score(x_train_dummy, y_train_dummy))
print('Gaussian Naive Bayes test accuracy score: ', acc_gaussian)
 print('Gaussian Naive Bayes train accuracy score: ', round(accuracy_score(y_hat_train, y_train_dummy)* 100, 2))
print('Gaussian Naive Bayes test accuracy score: ', round(accuracy_score(y_hat_test, y_test_dummy)* 100, 2))<train_model>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.cluster_spec().as_dict() ['worker'])
 except ValueError:
tpu = None
gpus = tf.config.experimental.list_logical_devices("GPU")
 if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 elif len(gpus)> 1:
strategy = tf.distribute.MirroredStrategy(gpus)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
logreg = LogisticRegression()
logreg.fit(x_train_dummy, y_train_dummy)
 y_hat_train = logreg.predict(x_train_dummy)
 y_hat_test = logreg.predict(x_test_dummy)
 acc_logreg = logreg.score(x_test_dummy, y_test_dummy)
 print('Logistic Regression train accuracy score: ', logreg.score(x_train_dummy, y_train_dummy))
print('Logistic Regression test accuracy score: ', acc_logreg )<compute_train_metric>
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
svc = SVC()
svc.fit(x_train_dummy, y_train_dummy)
 y_hat_train = svc.predict(x_train_dummy)
 y_hat_test = svc.predict(x_test_dummy)
 cv_score = cross_val_score(svc, x_train_dummy, y_train_dummy, cv=10)
 plt.plot(cv_score)
 acc_svc = svc.score(x_test_dummy, y_test_dummy)
 print('Support Vector Machines train accuracy score: ', svc.score(x_train_dummy, y_train_dummy))
print('Support Vector Machines test accuracy score: ', acc_svc)
 print('Support Vector Machines cross validation train accuracy score: ', cv_score.mean() )<train_model>
EPOCHS = 50
IMAGE_SIZE = [512, 512]
FLOWERS_DATASETS = {
512: 'gs://kds-f0a1db95190f5af9d47fb82f7af36915a50096ee81e54178f8c49016/tfrecords-jpeg-512x512/*/*.tfrec',
}
assert IMAGE_SIZE[0] == IMAGE_SIZE[1], "only square images are supported"
assert IMAGE_SIZE[0] in FLOWERS_DATASETS, "this image size is not supported"
MIXED_PRECISION = False
if MIXED_PRECISION:
if tpu:
policy = tf.keras.mixed_precision.experimental.Policy('mixed_bfloat16')
 else:
policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')
 tf.config.optimizer.set_jit(True)
 tf.keras.mixed_precision.experimental.set_policy(policy)
 print('Mixed precision enabled')
 if strategy.num_replicas_in_sync == 8:
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
VALIDATION_BATCH_SIZE = 16 * strategy.num_replicas_in_sync
start_lr = 0.00001
min_lr = 0.00001
max_lr = 0.00005 * strategy.num_replicas_in_sync
rampup_epochs = 5
sustain_epochs = 0
exp_decay =.8
elif strategy.num_replicas_in_sync == 1:
BATCH_SIZE = 16
VALIDATION_BATCH_SIZE = 16
start_lr = 0.00001
min_lr = 0.00001
max_lr = 0.0002
rampup_epochs = 5
sustain_epochs = 0
exp_decay =.8
else:
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
VALIDATION_BATCH_SIZE = 8 * strategy.num_replicas_in_sync
start_lr = 0.00001
min_lr = 0.00001
max_lr = 0.00002 * strategy.num_replicas_in_sync
rampup_epochs = 7
sustain_epochs = 0
exp_decay =.8
def lrfn(epoch):
def lr(epoch, start_lr, min_lr, max_lr, rampup_epochs, sustain_epochs, exp_decay):
if epoch < rampup_epochs:
lr =(max_lr - start_lr)/rampup_epochs * epoch + start_lr
elif epoch < rampup_epochs + sustain_epochs:
lr = max_lr
else:
lr =(max_lr - min_lr)* exp_decay**(epoch-rampup_epochs-sustain_epochs)+ min_lr
return lr
return lr(epoch, start_lr, min_lr, max_lr, rampup_epochs, sustain_epochs, exp_decay)
 lr_callback = tf.keras.callbacks.LearningRateScheduler(lambda epoch: lrfn(epoch), verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, [lrfn(x)for x in rng])
 print(y[0], y[-1] )
linear_svc = LinearSVC()
linear_svc.fit(x_train_dummy, y_train_dummy)
 y_hat_train = linear_svc.predict(x_train_dummy)
 y_hat_test = linear_svc.predict(x_test_dummy)
 acc_linear_svc = linear_svc.score(x_test_dummy, y_test_dummy)
 print('Linear SVC train accuracy score: ', linear_svc.score(x_train_dummy, y_train_dummy))
print('Linear SVC test accuracy score: ', acc_linear_svc )<train_model>
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n )
perceptron = Perceptron()
perceptron.fit(x_train_dummy, y_train_dummy)
 y_hat_train = perceptron.predict(x_train_dummy)
 y_hat_test = perceptron.predict(x_test_dummy)
 acc_perceptron = perceptron.score(x_test_dummy, y_test_dummy)
 print('Perceptron train accuracy score: ', perceptron.score(x_train_dummy, y_train_dummy))
print('Perceptron test accuracy score: ', acc_perceptron )<train_model>
TRAIN_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH +'/tfrecords-jpeg-512x512/train/*.tfrec')
 TRAIN_STEPS = count_data_items(TRAIN_FILENAMES)// BATCH_SIZE
NUM_TEST_IMAGES = 7382
decisiontree = DecisionTreeClassifier()
decisiontree.fit(x_train_dummy, y_train_dummy)
 y_hat_train = decisiontree.predict(x_train_dummy)
 y_hat_test = decisiontree.predict(x_test_dummy)
 acc_decisiontree = decisiontree.score(x_test_dummy, y_test_dummy)
 print('Decision Tree train accuracy score: ', decisiontree.score(x_train_dummy, y_train_dummy))
print('Decision Tree test accuracy score: ', acc_decisiontree )<train_model>
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord)
 return dataset
randomforest = RandomForestClassifier()
randomforest.fit(x_train_dummy, y_train_dummy)
 y_hat_train = randomforest.predict(x_train_dummy)
 y_hat_test = randomforest.predict(x_test_dummy)
 acc_randomforest = randomforest.score(x_test_dummy, y_test_dummy)
 print('Random Forest train accuracy score: ', randomforest.score(x_train_dummy, y_train_dummy))
print('Random Forest test accuracy score: ', acc_randomforest )<train_model>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))
knn = KNeighborsClassifier()
knn.fit(x_train_dummy, y_train_dummy)
 y_hat_train = knn.predict(x_train_dummy)
 y_hat_test = knn.predict(x_test_dummy)
 acc_knn = knn.score(x_test_dummy, y_test_dummy)
 print('KNN train accuracy score: ', knn.score(x_train_dummy, y_train_dummy))
print('KNN test accuracy score: ', acc_knn )<train_model>
def data_transform(image,label):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
rot = 15.* tf.random.normal([1],dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image,tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3]),label
sgd = SGDClassifier()
sgd.fit(x_train_dummy, y_train_dummy)
 y_hat_train = sgd.predict(x_train_dummy)
 y_hat_test = sgd.predict(x_test_dummy)
 acc_sgd = sgd.score(x_test_dummy, y_test_dummy)
 print('SGD train accuracy score: ', sgd.score(x_train_dummy, y_train_dummy))
print('SGD test accuracy score: ', acc_sgd )<train_model>
def random_blockout(img, sl=0.1, sh=0.2, rl=0.4):
p=random.random()
if p>=0.25:
w, h, c = IMAGE_SIZE[0], IMAGE_SIZE[1], 3
origin_area = tf.cast(h*w, tf.float32)
 e_size_l = tf.cast(tf.round(tf.sqrt(origin_area * sl * rl)) , tf.int32)
 e_size_h = tf.cast(tf.round(tf.sqrt(origin_area * sh / rl)) , tf.int32)
 e_height_h = tf.minimum(e_size_h, h)
 e_width_h = tf.minimum(e_size_h, w)
 erase_height = tf.random.uniform(shape=[], minval=e_size_l, maxval=e_height_h, dtype=tf.int32)
 erase_width = tf.random.uniform(shape=[], minval=e_size_l, maxval=e_width_h, dtype=tf.int32)
 erase_area = tf.zeros(shape=[erase_height, erase_width, c])
 erase_area = tf.cast(erase_area, tf.uint8)
 pad_h = h - erase_height
pad_top = tf.random.uniform(shape=[], minval=0, maxval=pad_h, dtype=tf.int32)
 pad_bottom = pad_h - pad_top
pad_w = w - erase_width
pad_left = tf.random.uniform(shape=[], minval=0, maxval=pad_w, dtype=tf.int32)
 pad_right = pad_w - pad_left
erase_mask = tf.pad([erase_area], [[0,0],[pad_top, pad_bottom], [pad_left, pad_right], [0,0]], constant_values=1)
 erase_mask = tf.squeeze(erase_mask, axis=0)
 erased_img = tf.multiply(tf.cast(img,tf.float32), tf.cast(erase_mask, tf.float32))
return tf.cast(erased_img, img.dtype)
 else:
return tf.cast(img, img.dtype )
gbk = GradientBoostingClassifier()
gbk.fit(x_train_dummy, y_train_dummy)
 y_hat_train = gbk.predict(x_train_dummy)
 y_hat_test = gbk.predict(x_test_dummy)
 acc_gbk = gbk.score(x_test_dummy, y_test_dummy)
 print('SGD train accuracy score: ', gbk.score(x_train_dummy, y_train_dummy))
print('SGD test accuracy score: ', acc_gbk )<train_model>
def get_training_dataset() :
dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-512x512/train/*.tfrec'), labeled=True)
 dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.map(data_transform, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset() :
dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-512x512/val/*.tfrec'), labeled=True, ordered=False)
 dataset = dataset.batch(VALIDATION_BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 opt = tf.data.Options()
opt.experimental_distribute.auto_shard_policy = tf.data.experimental.AutoShardPolicy.DATA
dataset = dataset.with_options(opt)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-512x512/test/*.tfrec'), labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 return dataset
training_dataset = get_training_dataset()
validation_dataset = get_validation_dataset()
ada = AdaBoostClassifier()
ada.fit(x_train_dummy, y_train_dummy)
 y_hat_train = ada.predict(x_train_dummy)
 y_hat_test = ada.predict(x_test_dummy)
 acc_ada = ada.score(x_test_dummy, y_test_dummy)
 print('Ada Boost train accuracy score: ', ada.score(x_train_dummy, y_train_dummy))
print('Ada Boost accuracy score: ', acc_ada )<create_dataframe>
def dataset_to_numpy_util(dataset, N):
dataset = dataset.unbatch().batch(N)
 for images, labels in dataset:
numpy_images = images.numpy()
numpy_labels = labels.numpy()
break;
return numpy_images, numpy_labels
def title_from_label_and_target(label, correct_label):
label = np.argmax(label, axis=-1)
 correct_label = np.argmax(correct_label, axis=-1)
 correct =(label == correct_label)
 return "{} [{}{}{}]".format(label, str(correct), ', shoud be ' if not correct else '',
correct_label if not correct else ''), correct
def display_one_flower(image, title, subplot, red=False):
plt.subplot(subplot)
 plt.axis('off')
 plt.imshow(image)
 plt.title(title, fontsize=16, color='red' if red else 'black')
 return subplot+1
def display_9_images_from_dataset(dataset):
subplot=331
plt.figure(figsize=(13,13))
images, labels = dataset_to_numpy_util(dataset, 9)
 for i, image in enumerate(images):
title = labels[i]
subplot = display_one_flower(image, title, subplot)
 if i >= 8:
break;
plt.tight_layout()
plt.subplots_adjust(wspace=0.1, hspace=0.1)
 plt.show()
models = pd.DataFrame({
"Model": ["SVM", "KNN", "Logistic Regression", "Random Forest",
"Naive Bayes", "Perceptron", "Linear SVC",
"Decision Tree", "SGD", "GBK", "Ada Boost"],
"Score": [acc_svc, acc_knn, acc_logreg, acc_randomforest,
acc_gaussian, acc_perceptron, acc_linear_svc,
acc_decisiontree, acc_sgd, acc_gbk, acc_ada]
})
 models.sort_values(by="Score", ascending=False )<choose_model_class>
display_9_images_from_dataset(training_dataset )
eclf_hard = VotingClassifier(estimators = [("DecisionTree", decisiontree), 
("GBK", gbk), 
("Logistic", logreg), 
("Linear SVC", linear_svc), 
("Ada Boost", ada), 
("KNN", knn), 
('SVM', svc)], voting="hard", weights=[6, 2, 1, 0.8, 3.5, 5, 4])
 eclf_hard.fit(x_train_dummy, y_train_dummy)
 print('Emsemble Voting train accuracy score: ', eclf_hard.score(x_train_dummy, y_train_dummy))
print('Emsemble Voting accuracy score: ', eclf_hard.score(x_test_dummy, y_test_dummy))<train_on_grid>
!pip install -q efficientnet
mlp = MLPClassifier(max_iter=50)
 parameter_space = {
'hidden_layer_sizes': [(50,50,50),(50,100,50),(100,)],
'activation': ['tanh', 'relu'],
'solver': ['lbfgs', 'adam'],
'alpha': [0.0001, 0.05],
'learning_rate': ['constant','adaptive'],
}
gridsearch = GridSearchCV(mlp, parameter_space, n_jobs=-1, cv=3)
 gridsearch.fit(x_train_dummy, y_train_dummy)
 print("Best parameters found: ", gridsearch.best_params_)
 means = gridsearch.cv_results_['mean_test_score']
stds = gridsearch.cv_results_['std_test_score']
for mean, std, params in zip(means, stds, gridsearch.cv_results_['params']):
print("%0.3f(+/-%0.03f)for %r" %(mean, std * 2, params))
print('Results on the train set:')
 print(classification_report(y_train, gridsearch.predict(x_train_dummy)))
 print('Results on the test set:')
 print(classification_report(y_test, gridsearch.predict(x_test_dummy)))
<train_model>
with strategy.scope() :
pretrained_model = efn.EfficientNetB7(input_shape=[*IMAGE_SIZE, 3], weights='noisy-student', include_top=False)
 pretrained_model.trainable = True
model = tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(104, activation='softmax')
 ])
 model.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
xgbc = XGBClassifier()
xgbc.fit(x_train_dummy, y_train_dummy)
 y_hat_train = xgbc.predict(x_train_dummy)
 y_hat_test = xgbc.predict(x_test_dummy)
 print('Xgboost train accuracy score: ', xgbc.score(x_train_dummy, y_train_dummy))
print('Xgboost test accuracy score: ', xgbc.score(x_test_dummy, y_test_dummy))
print('Results on the test set: ')
 print(classification_report(y_test_dummy, y_hat_test))<save_to_csv>
historical = model.fit(training_dataset,
steps_per_epoch=TRAIN_STEPS,
epochs= EPOCHS,
callbacks=[lr_callback],
validation_data=validation_dataset )
<save_to_csv><EOS> 
test_ds = get_test_dataset(ordered=True)
 print('Computing predictions...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilities = model.predict(test_images_ds)
 predictions = np.argmax(probabilities, axis=-1)
 print(predictions)
 print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )