Kaggle competition: Petals to the Metal - Flower Classification on TPU
<SOS> metric: macrofscore Kaggle data source: petals-to-the-metal-flower-classification-on-tpu<count_values>
print('Tensorflow version ' + tf.__version__)
for dataset in [train, validation]:
dataset['Title'] = dataset['Name'].str.split(',' ).str[1].str.split('.' ).str[0].str.strip()
print(dataset['Title'].value_counts())
 print()<count_missing_values>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
REPLICAS = strategy.num_replicas_in_sync
print("REPLICAS: ", REPLICAS )
for df in [train, validation]:
print(df.shape)
 print()
print(df.isna().sum() )<drop_column>
GCS_DS_PATH = KaggleDatasets().get_gcs_path("tpu-getting-started")
 EXT_GCS = KaggleDatasets().get_gcs_path("tf-flower-photo-tfrec" )
for df in [train, validation]:
df.dropna(subset = ['Embarked'], inplace = True )<feature_engineering>
AUTO = tf.data.experimental.AUTOTUNE
IMAGE_SIZE = [331, 331]
EPOCHS = 25
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
SUBDIR = f"/tfrecords-jpeg-{IMAGE_SIZE[0]}x{IMAGE_SIZE[1]}"
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH + SUBDIR + "/train/*.tfrec")
 TEST_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH + SUBDIR + "/test/*.tfrec")
 VALIDATION_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH + SUBDIR + "/val/*.tfrec")
 imagenet_files = tf.io.gfile.glob(EXT_GCS + '/imagenet' + SUBDIR + '/*.tfrec')
 inaturelist_files = tf.io.gfile.glob(EXT_GCS + '/inaturalist' + SUBDIR + '/*.tfrec')
 openimage_files = tf.io.gfile.glob(EXT_GCS + '/openimage' + SUBDIR + '/*.tfrec')
 oxford_files = tf.io.gfile.glob(EXT_GCS + '/oxford_102' + SUBDIR + '/*.tfrec')
 tensorflow_files = tf.io.gfile.glob(EXT_GCS + '/tf_flowers' + SUBDIR + '/*.tfrec')
 TRAINING_FILENAMES.extend(imagenet_files + inaturelist_files + \
openimage_files + oxford_files + tensorflow_files )
def getTitle(series):
return series.str.split(',' ).str[1].str.split('.' ).str[0].str.strip()
print(getTitle(train[train.Age.isnull() ].Name ).value_counts())
 mr_mask = train['Title'] == 'Mr'
miss_mask = train['Title'] == 'Miss'
mrs_mask = train['Title'] == 'Mrs'
master_mask = train['Title'] == 'Master'
dr_mask = train['Title'] == 'Dr'
train.loc[mr_mask, 'Age'] = train.loc[mr_mask, 'Age'].fillna(train[train.Title == 'Mr'].Age.mean())
 train.loc[miss_mask, 'Age'] = train.loc[miss_mask, 'Age'].fillna(train[train.Title == 'Miss'].Age.mean())
 train.loc[mrs_mask, 'Age'] = train.loc[mrs_mask, 'Age'].fillna(train[train.Title == 'Mrs'].Age.mean())
 train.loc[master_mask, 'Age'] = train.loc[master_mask, 'Age'].fillna(train[train.Title == 'Master'].Age.mean())
 train.loc[dr_mask, 'Age'] = train.loc[dr_mask, 'Age'].fillna(train[train.Title == 'Dr'].Age.mean())
 print()
print(getTitle(train[train.Age.isnull() ].Name ).value_counts() )<feature_engineering>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))
print(getTitle(validation[validation.Age.isnull() ].Name ).value_counts())
 mr_mask = validation['Title'] == 'Mr'
miss_mask = validation['Title'] == 'Miss'
mrs_mask = validation['Title'] == 'Mrs'
master_mask = validation['Title'] == 'Master'
ms_mask = validation['Title'] == 'Ms'
validation.loc[mr_mask, 'Age'] = validation.loc[mr_mask, 'Age'].fillna(validation[validation.Title == 'Mr'].Age.mean())
 validation.loc[miss_mask, 'Age'] = validation.loc[miss_mask, 'Age'].fillna(validation[validation.Title == 'Miss'].Age.mean())
 validation.loc[mrs_mask, 'Age'] = validation.loc[mrs_mask, 'Age'].fillna(validation[validation.Title == 'Mrs'].Age.mean())
 validation.loc[master_mask, 'Age'] = validation.loc[master_mask, 'Age'].fillna(validation[validation.Title == 'Master'].Age.mean())
 validation.loc[ms_mask, 'Age'] = validation.loc[ms_mask, 'Age'].fillna(validation[validation.Title == 'Miss'].Age.mean())
 print(getTitle(validation[validation.Age.isnull() ].Name ).value_counts() )<count_missing_values>
def transform(image,label):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
rot = 15.* tf.random.normal([1],dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image,tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3]),label
print(train.isna().sum())
 print(validation.isna().sum() )<drop_column>
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def data_augment(image, label):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_brightness(image, 0.2)
 return image, label
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord)
 return dataset
def get_dataset(filenames, augment=True, shuffle=True, repeat=True, labeled=True, ordered=False):
dataset = load_dataset(filenames, labeled=labeled, ordered=ordered)
 if augment:
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.map(transform, num_parallel_calls=AUTO)
 if repeat:
dataset = dataset.repeat()
if shuffle:
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_training_dataset() :
dataset = load_dataset(TRAINING_FILENAMES, labeled=True)
 dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.map(transform, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(ordered=False):
dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
training_dataset = get_training_dataset()
validation_dataset = get_validation_dataset()
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = int(count_data_items(TRAINING_FILENAMES))
NUM_VALIDATION_IMAGES = int(count_data_items(VALIDATION_FILENAMES))
NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))
train.drop(columns=['PassengerId'], inplace = True)
 [df.drop(columns=['Ticket'], inplace = True)for df in [train, validation]]<categorify>
!pip install efficientnet
[train, validation] = [pd.get_dummies(data = df, columns = ['Pclass', 'Sex', 'Embarked'])for df in [train, validation]]<data_type_conversions>
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True )
for df in [train, validation]:
df['HasCabin'] = df['Cabin'].notna().astype(int)
 df['FamilySize'] = df['SibSp'] + df['Parch'] + 1
df['IsAlone'] =(df['FamilySize'] > 1 ).astype(int )<drop_column>
with strategy.scope() :
pretrained_model = efn.EfficientNetB7(weights='imagenet', include_top=False, input_shape=[*IMAGE_SIZE, 3])
 pretrained_model.trainable = True
model1 = tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(104, activation='softmax')
 ])
 model1.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 historical = model1.fit(training_dataset,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks = [lr_callback],
validation_data=validation_dataset )
[df.drop(columns=['Cabin', 'SibSp', 'Parch'], inplace = True)for df in [train, validation]]<feature_engineering>
with strategy.scope() :
densenet = tf.keras.applications.DenseNet201(input_shape=[*IMAGE_SIZE, 3], weights='imagenet', include_top=False)
 densenet.trainable = True
model2 = tf.keras.Sequential([
densenet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(104, activation='softmax')
 ])
 model2.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 model2.summary()
historical = model2.fit(training_dataset,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks = [lr_callback],
validation_data=validation_dataset )
train['Title'] = train['Title'].replace('Mlle', 'Miss' ).replace('Ms', 'Miss' ).replace('Mme', 'Mrs' ).replace(['Dr', 'Major', 'Col', 'Rev', 'Lady', 'Jonkheer', 'Don', 'Sir', 'Dona', 'Capt', 'the Countess'], 'Special')
 validation['Title'] = validation['Title'].replace('Mlle', 'Miss' ).replace('Ms', 'Miss' ).replace('Mme', 'Mrs' ).replace(['Dr', 'Major', 'Col', 'Rev', 'Lady', 'Jonkheer', 'Don', 'Sir', 'Dona', 'Capt', 'the Countess'], 'Special')
 [df.drop(columns=['Name'], inplace = True)for df in [train, validation]]
[train, validation] = [pd.get_dummies(data = df, columns = ['Title'])for df in [train, validation]]<split>
cmdataset = get_validation_dataset(ordered=True)
 images_ds = cmdataset.map(lambda image, label: image)
 labels_ds = cmdataset.map(lambda image, label: label ).unbatch()
cm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES)) ).numpy()
m1 = model1.predict(images_ds)
 m2 = model2.predict(images_ds)
 scores = []
for alpha in np.linspace(0,1,100):
cm_probabilities = alpha*m1+(1-alpha)*m2
cm_predictions = np.argmax(cm_probabilities, axis=-1)
 scores.append(f1_score(cm_correct_labels, cm_predictions, labels=range(104), average='macro'))
best_alpha = np.argmax(scores)/100
print('Best alpha: ' + str(best_alpha))
<count_values><EOS> 
test_ds = get_test_dataset(ordered=True)
 print('Computing predictions...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probs1 = model1.predict(test_images_ds)
 probs2 = model2.predict(test_images_ds)
 probabilities = best_alpha*probs1 +(1-best_alpha)*probs2
predictions = np.argmax(probabilities, axis=-1)
 print(predictions)
 print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )