Kaggle competition: Petals to the Metal - Flower Classification on TPU
<SOS> metric: macrofscore Kaggle data source: petals-to-the-metal-flower-classification-on-tpu<count_values>
!pip install efficientnet
print("Tensorflow version " + tf.__version__ )
test.isAlone.value_counts()<drop_column>
AUTO = tf.data.experimental.AUTOTUNE
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu )
for data in df_all:
data.drop(columns=['PassengerId','Name','Cabin','Ticket','SibSp','Parch'],inplace=True,axis=1 )<count_missing_values>
SEED = 42
def seed_everything(seed):
os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(SEED )
for data in df_all:
print(f"
-------- {data.index } -------
")
 print(data.isnull().sum() )<categorify>
IMAGE_SIZE = [[512, 512]]
EPOCHS = 30
BATCH_SIZE = 32 * strategy.num_replicas_in_sync
AUG_BATCH = BATCH_SIZE
LEARNING_RATE = 1e-3
TTA_NUM = 3
RESUME_TRAINING = True
print("Batch size used: ", BATCH_SIZE )
test = pd.get_dummies(test,columns=['Sex','Embarked'])
 df = pd.get_dummies(df,columns=['Sex','Embarked'] )<prepare_x_and_y>
GCS_DS_PATH = KaggleDatasets().get_gcs_path('tpu-getting-started')
 MORE_IMAGES_GCS_DS_PATH = KaggleDatasets().get_gcs_path('tf-flower-photo-tfrec')
 GCS_PATH_SELECT = { 512: GCS_DS_PATH + '/tfrecords-jpeg-512x512' }
MOREIMAGES_PATH_SELECT = { 512: '/tfrecords-jpeg-512x512' }
y=df['Survived']
X=df.drop(columns=['Survived'],axis=1 )<split>
def decode_image(image_data, img_size):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*img_size, 3])
 return image
def read_labeled_tfrecord(example, img_size):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'], img_size)
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example, img_size):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'], img_size)
 idnum = example['id']
return image, idnum
def data_augment(image, label, seed=SEED):
image = tf.image.random_flip_left_right(image, seed=seed)
 image = tf.image.random_brightness(image, 0.1, seed=seed)
 image = tf.image.random_hue(image, 0.01)
 image = tf.image.random_saturation(image, 0.7, 1.3)
 image = tf.image.random_contrast(image, 0.8, 1.2)
 image = tf.image.random_brightness(image, 0.1)
 image = tf.image.random_saturation(image, 0, 2)
 return image, label
def get_training_dataset(IMG_SIZE=None, do_aug=True, cutmixup=True, mataug=False):
dataset = load_dataset(TRAINING_FILENAMES, labeled=True, IMG_SIZE= IMG_SIZE)
 if do_aug:
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 if mataug:
dataset = dataset.map(transform2, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
if cutmixup:
dataset = dataset.batch(AUG_BATCH)
 dataset = dataset.map(lambda image, label: transform(image, label, DIM=IMG_SIZE[0]), num_parallel_calls=AUTO)
 dataset = dataset.unbatch()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(ordered=False, IMG_SIZE=None):
dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered, IMG_SIZE=IMG_SIZE)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_train_valid_datasets(IMG_SIZE=None):
dataset = load_dataset(TRAINING_FILENAMES + VALIDATION_FILENAMES, labeled=True, IMG_SIZE=IMG_SIZE)
 dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False, IMG_SIZE=None):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered, IMG_SIZE=IMG_SIZE)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 def load_dataset(filenames, labeled=True, ordered=False, IMG_SIZE=None):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(lambda example: read_labeled_tfrecord(example, IMG_SIZE)if labeled else
read_unlabeled_tfrecord(example, IMG_SIZE), num_parallel_calls=AUTO)
 return dataset
def plot_training(H):
	
	with plt.xkcd() :
		plt.figure()
		plt.plot(H.history["loss"], label="train_loss")
 		plt.title("Training Loss")
 		plt.xlabel("Epoch
		plt.ylabel("Loss")
 		plt.legend(loc="lower left")
 		plt.show()
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.3 )<train_model>
GCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0][0]]
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')
 VALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')
 MOREIMAGES_PATH = MOREIMAGES_PATH_SELECT[IMAGE_SIZE[0][0]]
IMAGENET_FILES = tf.io.gfile.glob(MORE_IMAGES_GCS_DS_PATH + '/imagenet' + MOREIMAGES_PATH + '/*.tfrec')
 INATURELIST_FILES = tf.io.gfile.glob(MORE_IMAGES_GCS_DS_PATH + '/inaturalist' + MOREIMAGES_PATH + '/*.tfrec')
 OPENIMAGE_FILES = tf.io.gfile.glob(MORE_IMAGES_GCS_DS_PATH + '/openimage' + MOREIMAGES_PATH + '/*.tfrec')
 SKIP_VALIDATION = True
if SKIP_VALIDATION:
TRAINING_FILENAMES = TRAINING_FILENAMES + VALIDATION_FILENAMES + IMAGENET_FILES + INATURELIST_FILES + OPENIMAGE_FILES
NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)
 NUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH =(NUM_TRAINING_IMAGES + NUM_VALIDATION_IMAGES)// BATCH_SIZE
print('Dataset: {} training images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES+NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))
DT= DecisionTreeClassifier()
DT.fit(X_train, y_train)
 DT.score(X_test,y_test )<import_modules>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))
def transform2(image, label, DIM = IMAGE_SIZE[0][0]):
XDIM = DIM % 2
rot = 15.* tf.random.normal([1],dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image,tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3]),label
IFrame("https://scikit-learn.org/stable/modules/generated/sklearn.tree.DecisionTreeClassifier.html<define_search_space>
def cutmix(image, label, PROBABILITY = 1.0, DIM = None):
CLASSES = 104
imgs = []; labs = []
for j in range(AUG_BATCH):
P = tf.cast(tf.random.uniform([],0,1)<=PROBABILITY, tf.int32)
 k = tf.cast(tf.random.uniform([],0,AUG_BATCH),tf.int32)
 x = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 y = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 b = tf.random.uniform([],0,1)
 WIDTH = tf.cast(DIM * tf.math.sqrt(1-b),tf.int32)* P
ya = tf.math.maximum(0,y-WIDTH//2)
 yb = tf.math.minimum(DIM,y+WIDTH//2)
 xa = tf.math.maximum(0,x-WIDTH//2)
 xb = tf.math.minimum(DIM,x+WIDTH//2)
 one = image[j,ya:yb,0:xa,:]
two = image[k,ya:yb,xa:xb,:]
three = image[j,ya:yb,xb:DIM,:]
middle = tf.concat([one,two,three],axis=1)
 img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)
 imgs.append(img)
 a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)
 if len(label.shape)==1:
lab1 = tf.one_hot(label[j],CLASSES)
 lab2 = tf.one_hot(label[k],CLASSES)
 else:
lab1 = label[j,]
lab2 = label[k,]
labs.append(( 1-a)*lab1 + a*lab2)
 image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))
label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))
return image2,label2
parameters1 = [{'max_depth':np.linspace(1, 15, 15),'min_samples_split': np.linspace(0.1, 1.0, 5, endpoint=True)}]<train_on_grid>
def mixup(image, label, PROBABILITY = 1.0, DIM=None):
CLASSES = 104
imgs = []; labs = []
for j in range(AUG_BATCH):
P = tf.cast(tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)
 k = tf.cast(tf.random.uniform([],0,AUG_BATCH),tf.int32)
 a = tf.random.uniform([],0,1)*P
img1 = image[j,]
img2 = image[k,]
imgs.append(( 1-a)*img1 + a*img2)
 if len(label.shape)==1:
lab1 = tf.one_hot(label[j],CLASSES)
 lab2 = tf.one_hot(label[k],CLASSES)
 else:
lab1 = label[j,]
lab2 = label[k,]
labs.append(( 1-a)*lab1 + a*lab2)
 image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))
label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))
return image2,label2
Grid1 = GridSearchCV(DT, parameters1, cv=4,return_train_score=True,iid=True)
 Grid1.fit(X_train,y_train )<find_best_params>
def transform(image,label, DIM=None):
CLASSES = 104
SWITCH = 0.5
CUTMIX_PROB = 0.666
MIXUP_PROB = 0.666
image2, label2 = cutmix(image, label, CUTMIX_PROB, DIM)
 image3, label3 = mixup(image, label, MIXUP_PROB, DIM)
 imgs = []; labs = []
for j in range(AUG_BATCH):
P = tf.cast(tf.random.uniform([],0,1)<=SWITCH, tf.float32)
 imgs.append(P*image2[j,]+(1-P)*image3[j,])
 labs.append(P*label2[j,]+(1-P)*label3[j,])
 image4 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))
label4 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))
return image4,label4
scores = Grid1.cv_results_<compute_test_metric>
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.7
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))
for param, mean_train in zip(scores['params'],scores['mean_train_score']):
print(f"{param} accuracy on training data is {mean_train}" )<find_best_params>
def get_efficientnet() :
with strategy.scope() :
efficient = efn.EfficientNetB5(
 input_shape =(IMAGE_SIZE[0][0], IMAGE_SIZE[0][1], 3),
weights = 'noisy-student',
include_top = False 
)
 efficient.trainable = True
model = tf.keras.Sequential([
efficient,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(104, activation = 'softmax',dtype = 'float32')
 ])
 model.compile(optimizer='adam', loss = 'categorical_crossentropy', metrics=['categorical_accuracy'])
 return model
Grid1.best_estimator_<choose_model_class>
%%time
if RESUME_TRAINING:
with strategy.scope() :
model1 = load_model('.. /input/train-efficientnet/Effnet_save.h5')
 else:
model1 = get_efficientnet()
Checkpoint=tf.keras.callbacks.ModelCheckpoint(f"Effnet_30ep.h5", verbose=1, mode='max')
 train_history0 = model1.fit(
 get_training_dataset(IMAGE_SIZE[0], mataug=True, cutmixup=True, do_aug=True),
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
initial_epoch=20,
callbacks=[lr_callback, Checkpoint],
)
XGB = XGBClassifier()<define_search_space>
def predict(model, img_size, n_iter):
probs = []
data = get_test_dataset(ordered=True, IMG_SIZE=img_size)
 for i in range(n_iter):
test_images_ds = data.map(lambda image, idnum: image)
 probs.append(model.predict(test_images_ds,verbose=0))
return probs
parameters3 =[{"learning_rate": [0.05, 0.10, 0.15, 0.20] ,"max_depth": [ 3, 4, 5, 6], "min_child_weight": [3,5,7],"gamma": [ 0.0, 0.1, 0.2 ,0.3],"colsample_bytree" : [ 0.4, 0.5]}]<train_on_grid>
print('Calculating predictions...')
 probs1 = np.mean(predict(model1, IMAGE_SIZE[0], TTA_NUM), axis=0)
 predictions = np.argmax(probs1, axis=-1)
 print('Generating submission file...')
 test_ds = get_test_dataset(ordered=True, IMG_SIZE=IMAGE_SIZE[0])
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')