Kaggle competition: Flower Classification with TPUs
<SOS> metric: MacroFScore Kaggle data source: flower-classification-with-tpus<compute_train_metric>
!pip install -q efficientnet
clf = clf_ext
scores = cross_val_score(clf, X, y, cv=5)
 print(scores)
 print("Mean score = %.3f, Std deviation = %.3f"%(np.mean(scores),np.std(scores)) )<compute_test_metric>
print('TensorFlow version', tf.__version__)
 AUTO = tf.data.experimental.AUTOTUNE
score_ext_test = clf_ext.score(X_test,y_test)
 print(score_ext_test )<create_dataframe>
start_time = datetime.now()
print('Time now is', start_time)
 end_training_by_tdelta = timedelta(seconds=8400)
 this_run_file_prefix = start_time.strftime('%Y%m%d_%H%M_')
 print(this_run_file_prefix)
 IMAGE_SIZE = [224, 224]
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
GCS_PATH_SELECT = {
192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',
224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',
331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',
512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'
}
GCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')
 VALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')
 MOREIMAGES_PATH_SELECT = {
192: '/tfrecords-jpeg-192x192',
224: '/tfrecords-jpeg-224x224',
331: '/tfrecords-jpeg-331x331',
512: '/tfrecords-jpeg-512x512'
}
MOREIMAGES_PATH = MOREIMAGES_PATH_SELECT[IMAGE_SIZE[0]]
IMAGENET_FILES = tf.io.gfile.glob(MORE_IMAGES_GCS_DS_PATH + '/imagenet' + MOREIMAGES_PATH + '/*.tfrec')
 INATURELIST_FILES = tf.io.gfile.glob(MORE_IMAGES_GCS_DS_PATH + '/inaturalist' + MOREIMAGES_PATH + '/*.tfrec')
 OPENIMAGE_FILES = tf.io.gfile.glob(MORE_IMAGES_GCS_DS_PATH + '/openimage' + MOREIMAGES_PATH + '/*.tfrec')
 OXFORD_FILES = tf.io.gfile.glob(MORE_IMAGES_GCS_DS_PATH + '/oxford_102' + MOREIMAGES_PATH + '/*.tfrec')
 TENSORFLOW_FILES = tf.io.gfile.glob(MORE_IMAGES_GCS_DS_PATH + '/tf_flowers' + MOREIMAGES_PATH + '/*.tfrec')
 ADDITIONAL_TRAINING_FILENAMES = IMAGENET_FILES + INATURELIST_FILES + OPENIMAGE_FILES + OXFORD_FILES + TENSORFLOW_FILES
print('----')
 TRAINING_FILENAMES = TRAINING_FILENAMES + ADDITIONAL_TRAINING_FILENAMES
CLASSES = ['pink primrose', 'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea', 'wild geranium', 'tiger lily', 'moon orchid', 'bird of paradise', 'monkshood', 'globe thistle',
'snapdragon', "colt's foot", 'king protea', 'spear thistle', 'yellow iris', 'globe-flower', 'purple coneflower', 'peruvian lily', 'balloon flower', 'giant white arum lily',
'fire lily', 'pincushion flower', 'fritillary', 'red ginger', 'grape hyacinth', 'corn poppy', 'prince of wales feathers', 'stemless gentian', 'artichoke', 'sweet william',
'carnation', 'garden phlox', 'love in the mist', 'cosmos', 'alpine sea holly', 'ruby-lipped cattleya', 'cape flower', 'great masterwort', 'siam tulip', 'lenten rose',
'barberton daisy', 'daffodil', 'sword lily', 'poinsettia', 'bolero deep blue', 'wallflower', 'marigold', 'buttercup', 'daisy', 'common dandelion',
'petunia', 'wild pansy', 'primula', 'sunflower', 'lilac hibiscus', 'bishop of llandaff', 'gaura', 'geranium', 'orange dahlia', 'pink-yellow dahlia',
'cautleya spicata', 'japanese anemone', 'black-eyed susan', 'silverbush', 'californian poppy', 'osteospermum', 'spring crocus', 'iris', 'windflower', 'tree poppy',
'gazania', 'azalea', 'water lily', 'rose', 'thorn apple', 'morning glory', 'passion flower', 'lotus', 'toad lily', 'anthurium',
'frangipani', 'clematis', 'hibiscus', 'columbine', 'desert-rose', 'tree mallow', 'magnolia', 'cyclamen ', 'watercress', 'canna lily',
'hippeastrum ', 'bee balm', 'pink quill', 'foxglove', 'bougainvillea', 'camellia', 'mallow', 'mexican petunia', 'bromelia', 'blanket flower',
'trumpet creeper', 'blackberry lily', 'common tulip', 'wild rose']
models = pd.DataFrame({
'Model': ['Support Vector Machines', 'KNN', 'Logistic Regression',
'Random Forest', 'Gradient Boosting', 'Bagging KNN',
'Decision Tree','XGBoost','LightGBM','ExtraTree','Perceptron', 'Naive Bayes'],
'Score': [score_svm, score_knn, score_log, score_rf, score_gb, score_bag,
score_tree,score_xgb,score_lgb,score_ext,score_pctr, score_bay]})
 models.sort_values(by='Score', ascending=False )<find_best_model_class>
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = LR_START
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY = 0.80
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr = LR_START +(epoch *(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS)
 elif epoch <(LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS):
lr = LR_MAX
else:
lr = LR_MIN +(LR_MAX - LR_MIN)* LR_EXP_DECAY **(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)
 return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)
 rng = [i for i in range(20)]
y = [lrfn(x)for x in rng]
print(y )
clf_vote = VotingClassifier(
 estimators=[ 
('knn', clf_knn), 
('svm', clf_svm), 
('extra', clf_ext), 
('xgb', clf_xgb), 
('percep', clf_pctr), 
('logistic', clf_log),
],
weights=[2,2,3,3,1,2],
voting='hard')
 clf_vote.fit(X,y)
 scores = cross_val_score(clf_vote, X, y, cv=5, scoring='accuracy')
 print("Voting: Accuracy: %0.2f(+/- %0.2f)" %(scores.mean() , scores.std())) 
<train_model>
np.set_printoptions(threshold=15, linewidth=80)
 def batch_to_numpy_images_and_labels(data):
images, labels = data
numpy_images = images.numpy()
numpy_labels = labels.numpy()
if numpy_labels.dtype == object:
numpy_labels = [None for _ in enumerate(numpy_images)]
return numpy_images, numpy_labels
def title_from_label_and_target(label, correct_label):
if correct_label is None:
return CLASSES[label], True
correct =(label == correct_label)
 return "{} [{}{}{}]".format(CLASSES[label], 'OK' if correct else 'NO', u"\u2192" if not correct else '',
CLASSES[correct_label] if not correct else ''), correct
def display_one_flower(image, title, subplot, red=False, titlesize=16):
plt.subplot(*subplot)
 plt.axis('off')
 plt.imshow(image)
 if len(title)> 0:
plt.title(title, fontsize=int(titlesize)if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))
return(subplot[0], subplot[1], subplot[2]+1)
 def display_batch_of_images(databatch, predictions=None):

 images, labels = batch_to_numpy_images_and_labels(databatch)
 if labels is None:
labels = [None for _ in enumerate(images)]
rows = int(math.sqrt(len(images)))
 cols = len(images)//rows
FIGSIZE = 13.0
SPACING = 0.1
subplot=(rows,cols,1)
 if rows < cols:
plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))
else:
plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))
for i,(image, label)in enumerate(zip(images[:rows*cols], labels[:rows*cols])) :
title = '' if label is None else CLASSES[label]
correct = True
if predictions is not None:
title, correct = title_from_label_and_target(predictions[i], label)
 dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3
subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)
 plt.tight_layout()
if label is None and predictions is None:
plt.subplots_adjust(wspace=0, hspace=0)
 else:
plt.subplots_adjust(wspace=SPACING, hspace=SPACING)
 plt.show()
def display_confusion_matrix(cmat, score, precision, recall):
plt.figure(figsize=(15,15))
ax = plt.gca()
ax.matshow(cmat, cmap='Reds')
 ax.set_xticks(range(len(CLASSES)))
 ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})
 plt.setp(ax.get_xticklabels() , rotation=45, ha="left", rotation_mode="anchor")
 ax.set_yticks(range(len(CLASSES)))
 ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})
 plt.setp(ax.get_yticklabels() , rotation=45, ha="right", rotation_mode="anchor")
 titlestring = ""
if score is not None:
titlestring += 'f1 = {:.3f} '.format(score)
 if precision is not None:
titlestring += '
precision = {:.3f} '.format(precision)
 if recall is not None:
titlestring += '
recall = {:.3f} '.format(recall)
 if len(titlestring)> 0:
ax.text(101, 1, titlestring, fontdict={'fontsize': 18, 'horizontalalignment':'right', 'verticalalignment':'top', 'color':'
plt.show()
def display_training_curves(training, validation, title, subplot):
if subplot%10==1:
plt.subplots(figsize=(10,10), facecolor='
plt.tight_layout()
ax = plt.subplot(subplot)
 ax.set_facecolor('
ax.plot(training)
 ax.plot(validation)
 ax.set_title('model '+ title)
 ax.set_ylabel(title)
 ax.set_xlabel('epoch')
 ax.legend(['train', 'valid.'] )
train = X
ntrain = train.shape[0]
ntest = test.shape[0]
SEED = 0
NFOLDS = 5
kf = KFold(n_splits=NFOLDS, random_state=SEED)
 class SklearnHelper(object):
def __init__(self, clf, seed=0, params=None):
params['random_state'] = seed
self.clf = clf(**params)
 def train(self, x_train, y_train):
self.clf.fit(x_train, y_train)
 def predict(self, x):
return self.clf.predict(x)
 def fit(self,x,y):
return self.clf.fit(x,y )<train_model>
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels = 3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
'image': tf.io.FixedLenFeature([], tf.string),
'class': tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
'image': tf.io.FixedLenFeature([], tf.string),
'id': tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled = True, ordered = False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO)
 return dataset
def data_augment(image, label):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_saturation(image, 0, 2)
 return image, label
def get_training_dataset() :
dataset = load_dataset(TRAINING_FILENAMES, labeled = True)
 dataset = dataset.map(data_augment, num_parallel_calls = AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(ordered = False):
dataset = load_dataset(VALIDATION_FILENAMES, labeled = True, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered = False):
dataset = load_dataset(TEST_FILENAMES, labeled = False, ordered = ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)
 NUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))
def get_oof(clf, x_train, y_train, x_test):
oof_train = np.zeros(( ntrain,))
oof_test = np.zeros(( ntest,))
oof_test_skf = np.empty(( NFOLDS, ntest))
for i,(train_index, test_index)in enumerate(kf.split(x_train)) :
x_tr = x_train[train_index]
y_tr = y_train[train_index]
x_te = x_train[test_index]
clf.train(x_tr, y_tr)
 oof_train[test_index] = clf.predict(x_te)
 oof_test_skf[i, :] = clf.predict(x_test)
 oof_test[:] = np.median(oof_test_skf,axis=0)
 return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1 )<init_hyperparams>
training_dataset = get_training_dataset()
training_dataset = training_dataset.unbatch().batch(20)
 train_batch = iter(training_dataset)
rf_params = {
'n_estimators': 500,
'warm_start': True,
'max_depth': 6,
'min_samples_leaf': 2,
'max_features' : 'sqrt',
}
et_params = {
'n_estimators':500,
'max_depth': 8,
'min_samples_leaf': 2,
}
ada_params = {
'n_estimators': 500,
'learning_rate' : 0.75
}
gb_params = {
'n_estimators': 500,
'max_depth': 5,
'min_samples_leaf': 2,
}
svc_params = {
'kernel' : 'linear',
'C' : 0.025
}<choose_model_class>
test_dataset = get_test_dataset()
test_dataset = test_dataset.unbatch().batch(20)
 test_batch = iter(test_dataset)
rf = SklearnHelper(clf=RandomForestClassifier, seed=SEED, params=rf_params)
 et = SklearnHelper(clf=ExtraTreesClassifier, seed=SEED, params=et_params)
 ada = SklearnHelper(clf=AdaBoostClassifier, seed=SEED, params=ada_params)
 gb = SklearnHelper(clf=GradientBoostingClassifier, seed=SEED, params=gb_params)
 svc = SklearnHelper(clf=svm.SVC, seed=SEED, params=svc_params )<prepare_x_and_y>
early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=20, restore_best_weights = True )
y_train = y
train = X
foo = test.loc[:,cols]
x_train = train.values
x_test = foo.values<categorify>
def create_VGG16_model() :
pretrained_model = tf.keras.applications.VGG16(weights = 'imagenet', include_top = False, input_shape = [*IMAGE_SIZE, 3])
 pretrained_model.trainable = True
model = tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation = 'softmax')
 ])
 model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['sparse_categorical_accuracy'])
 return model
et_oof_train, et_oof_test = get_oof(et, x_train, y_train, x_test)
 rf_oof_train, rf_oof_test = get_oof(rf,x_train, y_train, x_test)
 ada_oof_train, ada_oof_test = get_oof(ada, x_train, y_train, x_test)
 gb_oof_train, gb_oof_test = get_oof(gb,x_train, y_train, x_test)
 svc_oof_train, svc_oof_test = get_oof(svc,x_train, y_train, x_test)
 print("Training is complete" )<prepare_output>
def create_Xception_model() :
pretrained_model = tf.keras.applications.Xception(include_top = False, input_shape = [*IMAGE_SIZE, 3])
 pretrained_model.trainable = True
model = tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation = 'softmax')
 ])
 model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['sparse_categorical_accuracy'])
 return model
base_predictions_train = pd.DataFrame({'RandomForest': rf_oof_train.ravel() ,
'ExtraTrees': et_oof_train.ravel() ,
'AdaBoost': ada_oof_train.ravel() ,
'SVM' : svc_oof_train.ravel() ,
'GradientBoost': gb_oof_train.ravel()
})
 base_predictions_train.head()<concatenate>
def create_DenseNet_model() :
pretrained_model = tf.keras.applications.DenseNet201(weights = 'imagenet', include_top = False, input_shape = [*IMAGE_SIZE, 3])
 pretrained_model.trainable = True
model = tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation = 'softmax')
 ])
 model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['sparse_categorical_accuracy'])
 return model
x_train = np.concatenate(( et_oof_train, rf_oof_train, ada_oof_train, gb_oof_train, svc_oof_train), axis=1)
 x_test = np.concatenate(( et_oof_test, rf_oof_test, ada_oof_test, gb_oof_test, svc_oof_test), axis=1 )<predict_on_test>
def create_EfficientNet_model() :
pretrained_model = efficientnet.EfficientNetB7(weights = 'noisy-student', include_top = False, input_shape = [*IMAGE_SIZE, 3])
 pretrained_model.trainable = True
model = tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation = 'softmax')
 ])
 model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['sparse_categorical_accuracy'])
 return model
clf_stack = xgb.XGBClassifier(
 n_estimators= 2000,
max_depth= 4,
min_child_weight= 2,
gamma=0.9,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
scale_pos_weight=1)
 clf_stack = clf_stack.fit(x_train, y_train)
 stack_pred = clf_stack.predict(x_test )<compute_train_metric>
def create_InceptionV3_model() :
pretrained_model = tf.keras.applications.InceptionV3(weights = 'imagenet', include_top = False, input_shape = [*IMAGE_SIZE, 3])
 pretrained_model.trainable = True
model = tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation = 'softmax')
 ])
 model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['sparse_categorical_accuracy'])
 return model
scores = cross_val_score(clf_stack, x_train, y_train, cv=5)
 print(scores)
 print("Mean score = %.3f, Std deviation = %.3f"%(np.mean(scores),np.std(scores)) )<predict_on_test>
def create_ResNet152_model() :
pretrained_model = tf.keras.applications.ResNet152V2(weights = 'imagenet', include_top = False, input_shape = [*IMAGE_SIZE, 3])
 pretrained_model.trainable = True
model = tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation = 'softmax')
 ])
 model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['sparse_categorical_accuracy'])
 return model
clf = clf_vote
df2 = test.loc[:,cols].fillna(method='pad')
 surv_pred = clf.predict(df2 )<save_to_csv>
def create_MobileNetV2_model() :
pretrained_model = tf.keras.applications.MobileNetV2(weights = 'imagenet', include_top = False, input_shape = [*IMAGE_SIZE, 3])
 pretrained_model.trainable = True
model = tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation = 'softmax')
 ])
 model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['sparse_categorical_accuracy'])
 return model
submit = pd.DataFrame({'PassengerId' : test.loc[:,'PassengerId'],
'Survived': stack_pred.T})
 submit.to_csv(".. /working/submit.csv", index=False)
<set_options>
def create_InceptionResNetV2_model() :
pretrained_model = tf.keras.applications.InceptionResNetV2(weights = 'imagenet', include_top = False, input_shape = [*IMAGE_SIZE, 3])
 pretrained_model.trainable = True
model = tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation = 'softmax')
 ])
 model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['sparse_categorical_accuracy'])
 return model
%reload_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
from fastai import *
from fastai.tabular import *
from fastai.metrics import *
import numpy as np
import pandas as pd<define_variables>
%%time
EPOCHS = 30
print('LR_EXP_DECAY:', LR_EXP_DECAY, '.LR_MAX:', LR_MAX)
 history = model.fit(get_training_dataset() , steps_per_epoch = STEPS_PER_EPOCH, epochs = EPOCHS, validation_data = get_validation_dataset() , callbacks = [lr_callback, early_stop] )
path = Path(".. /input/titanic" )<load_from_csv>
cmdataset = get_validation_dataset(ordered = True)
 images_ds = cmdataset.map(lambda image, label: image)
 labels_ds = cmdataset.map(lambda image, label: label ).unbatch()
cm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES)) ).numpy()
df = pd.read_csv(path/'train.csv')
 test_df = pd.read_csv(path/'test.csv')
 df.head()<feature_engineering>
%%time
test_ds = get_test_dataset(ordered = True)
 test_images_ds = test_ds.map(lambda image, idnum: image)
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U' )
class ZeroFare(TabularProc):
def apply_train(self, df:DataFrame):
df['Fare'] = df['Fare'].fillna(0)
 return df
class InspectName(TabularProc):
"Extracts valuable information from names, such as family name and title."
def apply_train(self, df:DataFrame):
df['FamilyName'] = df['Name'].apply(lambda s: s.split(",")[0])
 df['Title'] = df['Name'].apply(lambda s: s.split(", ")[1].split(".")[0] ).replace({'Mlle':'Miss', 'Mme':'Mrs', 'Ms':'Miss'})
 df['FamilySize'] = df['SibSp'] + df['Parch'] + 1
df['Family'] = df['FamilyName'] + df['FamilySize'].apply(lambda i: "_" + str(i))
return df
class InspectTicket(TabularProc):
def apply_train(self, df:DataFrame):
df['TicketPrefix'] = df['Ticket'].apply(self.find_ticket_prefix)
 df['TicketClass'] = df['Pclass'].apply(lambda s: int(s))
df['TicketSequence'] = df['Ticket'].apply(self.find_ticket_sequence)
 return df
def find_ticket_prefix(self, s: str)-> str:
res = re.findall(r'[^\d]+', s)
 if len(res)> 0:
return res[0].strip()
else:
return None
def find_ticket_sequence(self, s: str)-> int:
res = re.findall(r'[\d]+$', s)
 if len(res)> 0:
return int(res[0])
 else:
return None
class InspectCabin(TabularProc):
def apply_train(self, df: DataFrame):
df['Deck'] = df['Cabin'].apply(self.extract_deck)
 df['CabinNumber'] = df['Cabin'].apply(self.extract_cabin_number)
 return df
def extract_deck(self, cabin: str)-> str:
if type(cabin)is float and math.isnan(cabin):
return None
res = re.findall(r'^(\w)', cabin)
 if len(res)> 0:
return res[0]
else:
return None
def extract_cabin_number(self, cabin: str)-> int:
if type(cabin)is float and math.isnan(cabin):
return None
res = re.findall(r'(\d+)', cabin)
 if len(res)> 0:
return int(res[0])
 else:
return None
class Motherhood(TabularProc):
def apply_train(self, df: DataFrame):
df['Mother'] = df[['Sex', 'Parch']].apply(self.determine_motherhood, axis=1)
 return df
def determine_motherhood(self, row)-> bool:
return row['Sex'] == "female" and row['Parch'] > 0<define_variables>
val_probabilities = model.predict(images_ds)
 np.save(model_name + '_val_probabilities', val_probabilities)
 cm_predictions = np.argmax(val_probabilities, axis = -1)
 print('Correct labels: ', cm_correct_labels.shape, cm_correct_labels)
 print('Predicted labels: ', cm_predictions.shape, cm_predictions )
data.show_batch()<init_hyperparams>
def getFitPrecisionRecall(correct_labels, predictions):
score = f1_score(correct_labels, predictions, labels = range(len(CLASSES)) , average = 'macro')
 precision = precision_score(correct_labels, predictions, labels = range(len(CLASSES)) , average = 'macro')
 recall = recall_score(correct_labels, predictions, labels = range(len(CLASSES)) , average = 'macro')
 return score, precision, recall
class Ranger(Optimizer):
def __init__(self, params, lr=1e-3, alpha=0.5, k=6, betas= (.9,0.999), eps=1e-8, weight_decay=0):
if not 0.0 <= alpha <= 1.0:
raise ValueError(f'Invalid slow update rate: {alpha}')
 if not 1 <= k:
raise ValueError(f'Invalid lookahead steps: {k}')
 if not lr > 0:
raise ValueError(f'Invalid Learning Rate: {lr}')
 if not eps > 0:
raise ValueError(f'Invalid eps: {eps}')
 defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)
 super().__init__(params,defaults)
 for group in self.param_groups:
group["step_counter"] = 0
self.alpha = alpha
self.k = k
self.radam_buffer = [[None,None,None] for ind in range(10)]
self.slow_weights = [[p.clone().detach() for p in group['params']]
for group in self.param_groups]
for w in it.chain(*self.slow_weights):
w.requires_grad = False
def __setstate__(self, state):
print("set state called")
 super(Ranger, self ).__setstate__(state)
 def step(self, closure=None):
loss = None
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data.float()
if grad.is_sparse:
raise RuntimeError('RAdam does not support sparse gradients')
 p_data_fp32 = p.data.float()
state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p_data_fp32)
 state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)
 else:
state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)
 state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
beta1, beta2 = group['betas']
exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 state['step'] += 1
buffered = self.radam_buffer[int(state['step'] % 10)]
if state['step'] == buffered[0]:
N_sma, step_size = buffered[1], buffered[2]
else:
buffered[0] = state['step']
beta2_t = beta2 ** state['step']
N_sma_max = 2 /(1 - beta2)- 1
N_sma = N_sma_max - 2 * state['step'] * beta2_t /(1 - beta2_t)
 buffered[1] = N_sma
if N_sma > 5:
step_size = group['lr'] * math.sqrt(( 1 - beta2_t)*(N_sma - 4)/(N_sma_max - 4)*(N_sma - 2)/ N_sma * N_sma_max /(N_sma_max - 2)) /(1 - beta1 ** state['step'])
 else:
step_size = group['lr'] /(1 - beta1 ** state['step'])
 buffered[2] = step_size
if group['weight_decay'] != 0:
p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)
 if N_sma > 5:
denom = exp_avg_sq.sqrt().add_(group['eps'])
 p_data_fp32.addcdiv_(-step_size, exp_avg, denom)
 else:
p_data_fp32.add_(-step_size, exp_avg)
 p.data.copy_(p_data_fp32)
 for group,slow_weights in zip(self.param_groups,self.slow_weights):
group['step_counter'] += 1
if group['step_counter'] % self.k != 0:
continue
for p,q in zip(group['params'],slow_weights):
if p.grad is None:
continue
q.data.add_(self.alpha,p.data - q.data)
 p.data.copy_(q.data)
 return loss 
<choose_model_class>
cmat = confusion_matrix(cm_correct_labels, cm_predictions, labels = range(len(CLASSES)))
 score, precision, recall = getFitPrecisionRecall(cm_correct_labels, cm_predictions)
 cmat =(cmat.T / cmat.sum(axis = -1)).T
display_confusion_matrix(cmat, score, precision, recall)
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))
learn = tabular_learner(data,
layers=[50,30],
metrics=accuracy,
opt_func=Ranger,
emb_drop=0.1,
model_dir="/kaggle/working/models")
 learn.lr_find()
learn.recorder.plot(suggestion=True )<train_model>
def create_submission_file(filename, probabilities):
predictions = np.argmax(probabilities, axis = -1)
 print('Generating submission file...', filename)
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt(filename, np.rec.fromarrays([test_ids, predictions]), fmt = ['%s', '%d'], delimiter = ',', header = 'id,label', comments = '')
learn.fit_one_cycle(3, 1e-02 )<save_model>
%%time
test_probabilities_no_val = model.predict(test_images_ds)
 create_submission_file('submission_no_val.csv', test_probabilities_no_val )
learn.save('stage-1' )<train_model>
%%time
print('LR_EXP_DECAY:', LR_EXP_DECAY, '.LR_MAX:', LR_MAX)
 model.fit(get_validation_dataset() , steps_per_epoch = STEPS_PER_EPOCH, epochs = 25, callbacks = [lr_callback] )
learn.fit_one_cycle(10, 1e-02 )<save_model>
%%time
test_probabilities = model.predict(test_images_ds)
 create_submission_file('submission.csv', test_probabilities )
<predict_on_test><EOS> 
pklfile = open(model_name + '_test_probs_with_val.pkl', 'ab')
 pickle.dump(dict(zip(test_ids, test_probabilities)) , pklfile)
 pklfile.close()