Kaggle competition: Flower Classification with TPUs
<SOS> metric: MacroFScore Kaggle data source: flower-classification-with-tpus<feature_engineering>
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
train_df['Alone'] = train_df['Family'].copy()
train_df['Alone'].loc[train_df['Alone'] == 0] = -1
train_df['Alone'].loc[train_df['Alone'] > 0] = 0
train_df['Alone'].loc[train_df['Alone'] == -1] = 1<sort_values>
print('TensorFlow version: %s' % tf.__version__)
 print('Keras version: %s' % tf.keras.__version__)
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("TPU is not available")
 tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
relation_alone_survived = train_df[['Alone','Survived']].groupby('Alone', as_index=False ).mean() \ 
.sort_values(by='Alone', ascending=False )<count_values>
IMAGE_SIZE = [512, 512]
IMAGE_CHANNELS = 3
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
GCS_PATH_SELECT = {
192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',
224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',
331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',
512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'
}
GCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')
 VALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec' )
train_df['Alone'].value_counts()<drop_column>
TENSOR_BOARD = False
VERBOSE = 0
IMAGE_AUGMENTATION = True
GRID_MASK = False
USE_CLASS_WEIGHTS = False
MODEL_NAME = 6 * ['InceptionV3']
WEIGHTS_NAME = 6 * ['imagenet']
selected_optimizer = 'adam'
selected_loss = 'sparse_categorical_crossentropy'
selected_metrics = 'sparse_categorical_accuracy'
EPOCHS = 30
FOLDS = len(MODEL_NAME)
 SEED = 92
MODEL_FILENAMES = TRAINING_FILENAMES + VALIDATION_FILENAMES
train_df.drop('Alone', 1, inplace=True)
 train_df.head()<count_values>
CLASSES = [
'pink primrose', 'hard-leaved pocket orchid',
'canterbury bells', 'sweet pea',
'wild geranium', 'tiger lily',
'moon orchid', 'bird of paradise',
'monkshood', 'globe thistle',
'snapdragon', "colt's foot",
'king protea', 'spear thistle',
'yellow iris', 'globe-flower',
'purple coneflower', 'peruvian lily',
'balloon flower', 'giant white arum lily',
'fire lily', 'pincushion flower',
'fritillary', 'red ginger',
'grape hyacinth', 'corn poppy',
'prince of wales feathers', 'stemless gentian',
'artichoke', 'sweet william',
'carnation', 'garden phlox',
'love in the mist', 'cosmos',
'alpine sea holly', 'ruby-lipped cattleya',
'cape flower', 'great masterwort',
'siam tulip', 'lenten rose',
'barberton daisy', 'daffodil',
'sword lily', 'poinsettia',
'bolero deep blue', 'wallflower',
'marigold', 'buttercup',
'daisy', 'common dandelion',
'petunia', 'wild pansy',
'primula', 'sunflower',
'lilac hibiscus', 'bishop of llandaff',
'gaura', 'geranium',
'orange dahlia', 'pink-yellow dahlia',
'cautleya spicata', 'japanese anemone',
'black-eyed susan', 'silverbush',
'californian poppy', 'osteospermum',
'spring crocus', 'iris',
'windflower', 'tree poppy',
'gazania', 'azalea',
'water lily', 'rose',
'thorn apple', 'morning glory',
'passion flower', 'lotus',
'toad lily', 'anthurium',
'frangipani', 'clematis',
'hibiscus', 'columbine',
'desert-rose', 'tree mallow',
'magnolia', 'cyclamen ',
'watercress', 'canna lily',
'hippeastrum ', 'bee balm',
'pink quill', 'foxglove',
'bougainvillea', 'camellia',
'mallow', 'mexican petunia',
'bromelia', 'blanket flower',
'trumpet creeper', 'blackberry lily',
'common tulip', 'wild rose']
print('There are {} null values in Embarked'.format(train_df['Embarked'].isnull().sum()))
train_df['Embarked'].value_counts()<count_values>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 rotation_matrix = tf.reshape(
 tf.concat([
c1, s1, zero,
-s1, c1, zero,
zero,zero, one],
axis=0),
[3,3] 
)
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(
 tf.concat([
one, s2, zero,
zero, c2, zero,
zero,zero, one],
axis=0),
[3,3] 
)
 zoom_matrix = tf.reshape(
 tf.concat([
one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one],
axis=0),
[3,3] 
)
 shift_matrix = tf.reshape(
 tf.concat([
one, zero, height_shift,
zero, one, width_shift,
zero, zero, one],
axis=0),
[3,3] 
)
 return K.dot(
 K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def data_rotate(image,label):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
rot = 15.* tf.random.normal([1],dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image,tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3]),label
embarked_most_common = train_df['Embarked'].value_counts().idxmax()
train_df['Embarked'].fillna(embarked_most_common, inplace=True)
 print('Now there are {} null values in Embarked'.format(train_df['Embarked'].isnull().sum()))<sort_values>
AUTO = tf.data.experimental.AUTOTUNE
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, IMAGE_CHANNELS])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else
read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def data_augment(image, label):
image = tf.image.random_flip_left_right(image)
 return image, label
def get_training_dataset(dataset, do_aug=False, do_grid=False):
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 if do_aug: dataset = dataset.map(data_rotate, num_parallel_calls=AUTO)
 if do_grid: dataset = dataset.map(data_gridmask, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(dataset):
dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)
 NUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.
 format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))
train_df[['Embarked','Survived']].groupby('Embarked', as_index=False ).mean().sort_values(by='Survived', ascending=False )<data_type_conversions>
def read_tfrecord_label(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 label = tf.cast(example['class'], tf.int32)
 return label
def get_class_weights(filenames):
count = count_data_items(filenames)
 dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.map(read_tfrecord_label, num_parallel_calls=AUTO)
 labels = next(iter(dataset.batch(count)) ).numpy()
return class_weight.compute_class_weight('balanced', [x for x in range(len(CLASSES)) ], labels )
train_df['Embarked'] = train_df['Embarked'].map({'S': 0, 'Q': 1, 'C': 2} ).astype(int )<data_type_conversions>
def model_select(name, weights = 'imagenet'):
if name == 'EfficientNetB7':
try:
except ImportError:
!pip install -q efficientnet
pretrained_model = efn.EfficientNetB7(
 weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, IMAGE_CHANNELS])
 pretrained_model.trainable = True
return tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 elif name == 'InceptionV3':
pretrained_model = tf.keras.applications.InceptionV3(
 weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, IMAGE_CHANNELS])
 pretrained_model.trainable = True
return tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(4096, activation='relu'),
tf.keras.layers.Dense(4096, activation='relu'),
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 elif name == 'DenseNet201':
pretrained_model = tf.keras.applications.DenseNet201(
 weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, IMAGE_CHANNELS])
 pretrained_model.trainable = True
return tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 elif name == 'ResNet152V2':
pretrained_model = tf.keras.applications.ResNet152V2(
 weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, IMAGE_CHANNELS])
 pretrained_model.trainable = True
return tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(4096, activation='relu'),
tf.keras.layers.Dense(4096, activation='relu'),
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 elif name == 'VGG16':
pretrained_model = tf.keras.applications.VGG16(
 weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, IMAGE_CHANNELS])
 pretrained_model.trainable = True
return tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(4096, activation='relu'),
tf.keras.layers.Dense(4096, activation='relu'),
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 else:
print('Unknown model: %s' % name)
 raise
def create_model(name, weights = 'imagenet'):
with strategy.scope() :
model = model_select(name, weights)
 model.compile(
 optimizer = selected_optimizer,
loss = selected_loss,
metrics=[selected_metrics] 
)
 return model
test_df['Fare'].fillna(test_df['Fare'].dropna().median() , inplace=True )<count_values>
def save_config() :
config = {
"IMAGE_SIZE" : IMAGE_SIZE,
"IMAGE_CHANNELS" : IMAGE_CHANNELS,
"BATCH_SIZE" : BATCH_SIZE,
"MODEL_NAME" : MODEL_NAME,
"WEIGHTS_NAME" : WEIGHTS_NAME,
"EPOCHS" : EPOCHS,
"FOLDS" : FOLDS,
"TENSOR_BOARD" : TENSOR_BOARD,
"VERBOSE" : VERBOSE,
"IMAGE_AUGMENTATION" : IMAGE_AUGMENTATION,
"GRID_MASK" : GRID_MASK,
"USE_CLASS_WEIGHTS" : USE_CLASS_WEIGHTS,
"SEED" : SEED
}
with open('config.json', 'w')as outfile:
json.dump(config, outfile)
 save_config()
train_df['FareCategorical'] = pd.qcut(train_df['Fare'], 4)
 train_df['FareCategorical'].value_counts()<sort_values>
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 4
LR_SUSTAIN_EPOCHS = 6
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch -
LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
!rm -rf./logs/
log_dir = "logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
 def callbacks(ifold, board = False):
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = False)
 if board:
tensorboard_callback = tf.keras.callbacks.TensorBoard(
 log_dir=log_dir + f"/fold{ifold}", histogram_freq=1)
 return [lr_callback, tensorboard_callback]
else:
return [lr_callback]
kfold = KFold(FOLDS, shuffle = True, random_state = SEED)
 CLASS_WEIGHTS = get_class_weights(MODEL_FILENAMES)if USE_CLASS_WEIGHTS else None
test_ds = get_test_dataset(ordered=True)
 def train_cross_validate_predict(filenames):
print(f'Start training {FOLDS} folds {EPOCHS} epochs, img_aug {IMAGE_AUGMENTATION} grid {GRID_MASK} class weights {USE_CLASS_WEIGHTS}')
 histories = []
for ifold,(trn_ind, val_ind)in enumerate(kfold.split(filenames)) :
tdf = list(pd.DataFrame({'FILENAMES': filenames} ).loc[trn_ind]['FILENAMES'])
 vdf = list(pd.DataFrame({'FILENAMES': filenames} ).loc[val_ind]['FILENAMES'])
 steps_per_epoch = count_data_items(tdf)// BATCH_SIZE
train_dataset = load_dataset(tdf, labeled = True)
 val_dataset = load_dataset(vdf, labeled = True, ordered = True)
 validation_data = get_validation_dataset(val_dataset)
 print_and_log(f'
model = create_model(MODEL_NAME[ifold], WEIGHTS_NAME[ifold])
 history = model.fit(
 get_training_dataset(train_dataset, IMAGE_AUGMENTATION, GRID_MASK),
steps_per_epoch = steps_per_epoch,
epochs = EPOCHS,
callbacks = callbacks(ifold, TENSOR_BOARD),
validation_data = validation_data,
class_weight = CLASS_WEIGHTS,
verbose = VERBOSE 
)
 test_images_ds = test_ds.map(lambda image, idnum: image)
 predict = model.predict(test_images_ds)
 np.save(f'fold{ifold+1}.npy', predict)
 cm_predict = model.predict(validation_data)
 np.save(f'cm_fold{ifold+1}.npy', cm_predict)
 histories.append(history)
 del model, train_dataset, val_dataset, validation_data, predict, cm_predict
gc.collect()
if tpu:
tf.tpu.experimental.initialize_tpu_system(tpu)
 return histories
train_df[['FareCategorical','Survived']].groupby('FareCategorical', as_index=False ).mean().\
sort_values(by='Survived',ascending=False )<feature_engineering>
histories = train_cross_validate_predict(MODEL_FILENAMES )
train_df.loc[train_df['Fare'] < 7.91, 'Fare'] = 0
train_df.loc[(train_df['Fare'] >= 7.91)&(train_df['Fare'] < 14.454), 'Fare'] = 1
train_df.loc[(train_df['Fare'] >= 14.454)&(train_df['Fare'] < 31), 'Fare'] = 2
train_df.loc[train_df['Fare'] >= 31, 'Fare'] = 3
train_df['Fare'] = train_df['Fare'].astype(int)
 train_df.drop('FareCategorical', 1, inplace=True )<categorify>
def get_predictions(preds):
print('Computing predictions...')
 probabilities = np.average([preds[i] for i in range(len(preds)) ], axis = 0)
 predictions = np.argmax(probabilities, axis=-1)
 print(predictions)
 return predictions
def save_prediction(predictions):
print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]),
fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )
train_df['Sex'] = train_df['Sex'].map({'male': 0, 'female': 1} )<feature_engineering>
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
def transform_df(df):
df.drop(['PassengerId','Ticket', 'Cabin'], 1, inplace=True)
 age_mean = df['Age'].mean()
age_std = df['Age'].std()
age_nan_count = df['Age'].isnull().sum()
rand_ages = np.random.randint(age_mean - age_std, age_mean + age_std, age_nan_count)
 df['Age'][np.isnan(df['Age'])] = rand_ages
df.loc[df['Age'] < 10.367, 'Age'] = 0
df.loc[(df['Age'] >= 10.367)&(df['Age'] < 20.315), 'Age'] = 1
df.loc[(df['Age'] >= 20.315)&(df['Age'] < 30.263), 'Age'] = 2
df.loc[(df['Age'] >= 30.263)&(df['Age'] < 40.21), 'Age'] = 3
df.loc[(df['Age'] >= 40.21)&(df['Age'] < 50.157), 'Age'] = 4
df.loc[(df['Age'] >= 50.157)&(df['Age'] < 60.105), 'Age'] = 5
df.loc[(df['Age'] >= 60.105)&(df['Age'] < 70.0525), 'Age'] = 6
df.loc[(df['Age'] >= 70.0525)&(df['Age'] <= 80), 'Age'] = 7
df['Age'] = df['Age'].astype(int)
 df['Name'] = df.Name.str.extract('([A-Za-z]+)\.', expand=False)
 df['Name'] = df['Name'].replace(['Capt', 'Dr','Rev','Mile','Col','Major','Countess','Jonkheer','Mme',\
'Don', 'Ms','Sir','Capt','Lady', 'Mlle'], 'Low Appearence')
 df['Name'] = df['Name'].map({'Mr':0, 'Low Appearence': 1, 'Master': 2, 'Miss': 3, 'Mrs': 4})
 df['Name'] = df['Name'].fillna(4 ).astype(int)
 df['Family'] = df['SibSp'] + df['Parch']
df.drop(['SibSp','Parch'], 1, inplace=True)

embarked_most_common = df['Embarked'].value_counts().idxmax()
df['Embarked'].fillna(embarked_most_common, inplace=True)
 df['Embarked'] = df['Embarked'].map({'S': 0, 'Q': 1, 'C': 2} ).astype(int)
 df['Fare'].fillna(df['Fare'].dropna().median() , inplace=True)
 df.loc[df['Fare'] < 7.91, 'Fare'] = 0
df.loc[(df['Fare'] >= 7.91)&(df['Fare'] < 14.454), 'Fare'] = 1
df.loc[(df['Fare'] >= 14.454)&(df['Fare'] < 31), 'Fare'] = 2
df.loc[df['Fare'] >= 31, 'Fare'] = 3
df['Fare'] = df['Fare'].astype(int)
 df['Sex'] = df['Sex'].map({'male': 0, 'female': 1})
 return df<load_from_csv>
print('TensorFlow version: %s' % tf.__version__)
 print('Keras version: %s' % tf.keras.__version__)
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("TPU is not available")
 tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )<categorify>
IMAGE_SIZE = [512, 512]
IMAGE_CHANNELS = 3
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
GCS_PATH_SELECT = {
192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',
224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',
331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',
512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'
}
GCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')
 VALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec' )
train_df = transform_df(train_df)
 test_df = transform_df(test_df )<import_modules>
TENSOR_BOARD = False
VERBOSE = 0
IMAGE_AUGMENTATION = True
GRID_MASK = False
USE_CLASS_WEIGHTS = False
MODEL_NAME = 6 * ['InceptionV3']
WEIGHTS_NAME = 6 * ['imagenet']
selected_optimizer = 'adam'
selected_loss = 'sparse_categorical_crossentropy'
selected_metrics = 'sparse_categorical_accuracy'
EPOCHS = 30
FOLDS = len(MODEL_NAME)
 SEED = 92
MODEL_FILENAMES = TRAINING_FILENAMES + VALIDATION_FILENAMES
from sklearn.model_selection import cross_val_score
from sklearn.dummy import DummyClassifier
from sklearn.naive_bayes import GaussianNB, MultinomialNB
from sklearn.svm import SVC, LinearSVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.cluster import KMeans
from xgboost import XGBClassifier<prepare_x_and_y>
CLASSES = [
'pink primrose', 'hard-leaved pocket orchid',
'canterbury bells', 'sweet pea',
'wild geranium', 'tiger lily',
'moon orchid', 'bird of paradise',
'monkshood', 'globe thistle',
'snapdragon', "colt's foot",
'king protea', 'spear thistle',
'yellow iris', 'globe-flower',
'purple coneflower', 'peruvian lily',
'balloon flower', 'giant white arum lily',
'fire lily', 'pincushion flower',
'fritillary', 'red ginger',
'grape hyacinth', 'corn poppy',
'prince of wales feathers', 'stemless gentian',
'artichoke', 'sweet william',
'carnation', 'garden phlox',
'love in the mist', 'cosmos',
'alpine sea holly', 'ruby-lipped cattleya',
'cape flower', 'great masterwort',
'siam tulip', 'lenten rose',
'barberton daisy', 'daffodil',
'sword lily', 'poinsettia',
'bolero deep blue', 'wallflower',
'marigold', 'buttercup',
'daisy', 'common dandelion',
'petunia', 'wild pansy',
'primula', 'sunflower',
'lilac hibiscus', 'bishop of llandaff',
'gaura', 'geranium',
'orange dahlia', 'pink-yellow dahlia',
'cautleya spicata', 'japanese anemone',
'black-eyed susan', 'silverbush',
'californian poppy', 'osteospermum',
'spring crocus', 'iris',
'windflower', 'tree poppy',
'gazania', 'azalea',
'water lily', 'rose',
'thorn apple', 'morning glory',
'passion flower', 'lotus',
'toad lily', 'anthurium',
'frangipani', 'clematis',
'hibiscus', 'columbine',
'desert-rose', 'tree mallow',
'magnolia', 'cyclamen ',
'watercress', 'canna lily',
'hippeastrum ', 'bee balm',
'pink quill', 'foxglove',
'bougainvillea', 'camellia',
'mallow', 'mexican petunia',
'bromelia', 'blanket flower',
'trumpet creeper', 'blackberry lily',
'common tulip', 'wild rose']
X_train = train_df.drop('Survived', 1)
 y_train = train_df['Survived']
cv_value = 4
model_results = {}
X_train.head()<data_type_conversions>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 rotation_matrix = tf.reshape(
 tf.concat([
c1, s1, zero,
-s1, c1, zero,
zero,zero, one],
axis=0),
[3,3] 
)
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(
 tf.concat([
one, s2, zero,
zero, c2, zero,
zero,zero, one],
axis=0),
[3,3] 
)
 zoom_matrix = tf.reshape(
 tf.concat([
one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one],
axis=0),
[3,3] 
)
 shift_matrix = tf.reshape(
 tf.concat([
one, zero, height_shift,
zero, one, width_shift,
zero, zero, one],
axis=0),
[3,3] 
)
 return K.dot(
 K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def data_rotate(image,label):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
rot = 15.* tf.random.normal([1],dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image,tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3]),label
X_train = X_train.as_matrix()
y_train = y_train.as_matrix()
test_df = test_df.as_matrix()<compute_train_metric>
AUTO = tf.data.experimental.AUTOTUNE
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, IMAGE_CHANNELS])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else
read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def data_augment(image, label):
image = tf.image.random_flip_left_right(image)
 return image, label
def get_training_dataset(dataset, do_aug=False, do_grid=False):
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 if do_aug: dataset = dataset.map(data_rotate, num_parallel_calls=AUTO)
 if do_grid: dataset = dataset.map(data_gridmask, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(dataset):
dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)
 NUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.
 format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))
clf_dummy = DummyClassifier(strategy='most_frequent')
 scores = cross_val_score(clf_dummy, X=X_train, y=y_train, cv=cv_value)
 model_results['DummyClassifier'] = np.mean(scores)
 np.mean(scores )<compute_train_metric>
def read_tfrecord_label(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 label = tf.cast(example['class'], tf.int32)
 return label
def get_class_weights(filenames):
count = count_data_items(filenames)
 dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.map(read_tfrecord_label, num_parallel_calls=AUTO)
 labels = next(iter(dataset.batch(count)) ).numpy()
return class_weight.compute_class_weight('balanced', [x for x in range(len(CLASSES)) ], labels )
clf_m_naive_bayes = MultinomialNB()
scores = cross_val_score(clf_m_naive_bayes, X=X_train, y=y_train, cv=cv_value)
 model_results['MultinomialNB'] = np.mean(scores)
 np.mean(scores )<compute_train_metric>
def model_select(name, weights = 'imagenet'):
if name == 'EfficientNetB7':
try:
except ImportError:
!pip install -q efficientnet
pretrained_model = efn.EfficientNetB7(
 weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, IMAGE_CHANNELS])
 pretrained_model.trainable = True
return tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 elif name == 'InceptionV3':
pretrained_model = tf.keras.applications.InceptionV3(
 weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, IMAGE_CHANNELS])
 pretrained_model.trainable = True
return tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(4096, activation='relu'),
tf.keras.layers.Dense(4096, activation='relu'),
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 elif name == 'DenseNet201':
pretrained_model = tf.keras.applications.DenseNet201(
 weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, IMAGE_CHANNELS])
 pretrained_model.trainable = True
return tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 elif name == 'ResNet152V2':
pretrained_model = tf.keras.applications.ResNet152V2(
 weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, IMAGE_CHANNELS])
 pretrained_model.trainable = True
return tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(4096, activation='relu'),
tf.keras.layers.Dense(4096, activation='relu'),
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 elif name == 'VGG16':
pretrained_model = tf.keras.applications.VGG16(
 weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, IMAGE_CHANNELS])
 pretrained_model.trainable = True
return tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(4096, activation='relu'),
tf.keras.layers.Dense(4096, activation='relu'),
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 else:
print('Unknown model: %s' % name)
 raise
def create_model(name, weights = 'imagenet'):
with strategy.scope() :
model = model_select(name, weights)
 model.compile(
 optimizer = selected_optimizer,
loss = selected_loss,
metrics=[selected_metrics] 
)
 return model
clf_svc = SVC()
scores = cross_val_score(clf_svc, X=X_train, y=y_train, cv=cv_value)
 model_results['SVC'] = np.mean(scores)
 np.mean(scores )<compute_train_metric>
def save_config() :
config = {
"IMAGE_SIZE" : IMAGE_SIZE,
"IMAGE_CHANNELS" : IMAGE_CHANNELS,
"BATCH_SIZE" : BATCH_SIZE,
"MODEL_NAME" : MODEL_NAME,
"WEIGHTS_NAME" : WEIGHTS_NAME,
"EPOCHS" : EPOCHS,
"FOLDS" : FOLDS,
"TENSOR_BOARD" : TENSOR_BOARD,
"VERBOSE" : VERBOSE,
"IMAGE_AUGMENTATION" : IMAGE_AUGMENTATION,
"GRID_MASK" : GRID_MASK,
"USE_CLASS_WEIGHTS" : USE_CLASS_WEIGHTS,
"SEED" : SEED
}
with open('config.json', 'w')as outfile:
json.dump(config, outfile)
 save_config()
clf_forest = RandomForestClassifier(n_estimators=200, max_depth=4, n_jobs=-1)
 scores = cross_val_score(clf_forest, X=X_train, y=y_train, cv=cv_value)
 model_results['RandomForest'] = np.mean(scores)
 np.mean(scores )<compute_train_metric>
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 4
LR_SUSTAIN_EPOCHS = 6
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch -
LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
!rm -rf./logs/
log_dir = "logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
 def callbacks(ifold, board = False):
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = False)
 if board:
tensorboard_callback = tf.keras.callbacks.TensorBoard(
 log_dir=log_dir + f"/fold{ifold}", histogram_freq=1)
 return [lr_callback, tensorboard_callback]
else:
return [lr_callback]
kfold = KFold(FOLDS, shuffle = True, random_state = SEED)
 CLASS_WEIGHTS = get_class_weights(MODEL_FILENAMES)if USE_CLASS_WEIGHTS else None
test_ds = get_test_dataset(ordered=True)
 def train_cross_validate_predict(filenames):
print(f'Start training {FOLDS} folds {EPOCHS} epochs, img_aug {IMAGE_AUGMENTATION} grid {GRID_MASK} class weights {USE_CLASS_WEIGHTS}')
 histories = []
for ifold,(trn_ind, val_ind)in enumerate(kfold.split(filenames)) :
tdf = list(pd.DataFrame({'FILENAMES': filenames} ).loc[trn_ind]['FILENAMES'])
 vdf = list(pd.DataFrame({'FILENAMES': filenames} ).loc[val_ind]['FILENAMES'])
 steps_per_epoch = count_data_items(tdf)// BATCH_SIZE
train_dataset = load_dataset(tdf, labeled = True)
 val_dataset = load_dataset(vdf, labeled = True, ordered = True)
 validation_data = get_validation_dataset(val_dataset)
 print_and_log(f'
model = create_model(MODEL_NAME[ifold], WEIGHTS_NAME[ifold])
 history = model.fit(
 get_training_dataset(train_dataset, IMAGE_AUGMENTATION, GRID_MASK),
steps_per_epoch = steps_per_epoch,
epochs = EPOCHS,
callbacks = callbacks(ifold, TENSOR_BOARD),
validation_data = validation_data,
class_weight = CLASS_WEIGHTS,
verbose = VERBOSE 
)
 test_images_ds = test_ds.map(lambda image, idnum: image)
 predict = model.predict(test_images_ds)
 np.save(f'fold{ifold+1}.npy', predict)
 cm_predict = model.predict(validation_data)
 np.save(f'cm_fold{ifold+1}.npy', cm_predict)
 histories.append(history)
 del model, train_dataset, val_dataset, validation_data, predict, cm_predict
gc.collect()
if tpu:
tf.tpu.experimental.initialize_tpu_system(tpu)
 return histories
clf_ada = AdaBoostClassifier()
scores = cross_val_score(clf_ada, X=X_train, y=y_train, cv=cv_value)
 model_results['AdaBoost'] = np.mean(scores)
 np.mean(scores )<compute_train_metric>
histories = train_cross_validate_predict(MODEL_FILENAMES )
clf_knn = KNeighborsClassifier(n_neighbors=5)
 scores = cross_val_score(clf_knn, X=X_train, y=y_train, cv=cv_value)
 model_results['KNN'] = np.mean(scores)
 np.mean(scores )<compute_train_metric>
def get_predictions(preds):
print('Computing predictions...')
 probabilities = np.average([preds[i] for i in range(len(preds)) ], axis = 0)
 predictions = np.argmax(probabilities, axis=-1)
 print(predictions)
 return predictions
def save_prediction(predictions):
print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]),
fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )