Kaggle competition: Petals to the Metal - Flower Classification on TPU
<SOS> metric: macrofscore Kaggle data source: petals-to-the-metal-flower-classification-on-tpu<prepare_x_and_y>
print("TF version " + tf.__version__)
 AUTO = tf.data.experimental.AUTOTUNE
y = df_train['SalePrice']<concatenate>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
df_all = pd.concat([df_train, df_test], sort=False )<categorify>
IMAGE_SIZE = [512, 512]
EPOCHS = 40
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
SEED = 752
SKIP_VALIDATION = False
TTA_NUM = 5
random.seed(SEED)
 np.random.seed(SEED)
 tf.random.set_seed(SEED )
Cat_toNone =('PoolQC','Fence','MiscFeature','Alley','FireplaceQu','GarageType','GarageFinish',
'GarageQual','GarageCond','BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1',
'BsmtFinType2','MasVnrType','MSSubClass')
 for c in Cat_toNone:
df_all[c] = df_all[c].fillna('None' )<categorify>
np.set_printoptions(threshold=15, linewidth=80)
 def batch_to_numpy_images_and_labels(data):
images, labels = data
numpy_images = images.numpy()
numpy_labels = labels.numpy()
if numpy_labels.dtype == object:
numpy_labels = [None for _ in enumerate(numpy_images)]
return numpy_images, numpy_labels
def title_from_label_and_target(label, correct_label):
if correct_label is None:
return CLASSES[label], True
correct =(label == correct_label)
 return "{} [{}{}{}]".format(CLASSES[label], 'OK' if correct else 'NO', u"\u2192" if not correct else '',
CLASSES[correct_label] if not correct else ''), correct
def display_one_flower(image, title, subplot, red=False, titlesize=16):
plt.subplot(*subplot)
 plt.axis('off')
 plt.imshow(image)
 if len(title)> 0:
plt.title(title, fontsize=int(titlesize)if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))
return(subplot[0], subplot[1], subplot[2]+1)
 def display_batch_of_images(databatch, predictions=None):

 images, labels = batch_to_numpy_images_and_labels(databatch)
 if labels is None:
labels = [None for _ in enumerate(images)]
rows = int(math.sqrt(len(images)))
 cols = len(images)//rows
FIGSIZE = 13.0
SPACING = 0.1
subplot=(rows,cols,1)
 if rows < cols:
plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))
else:
plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))
for i,(image, label)in enumerate(zip(images[:rows*cols], labels[:rows*cols])) :
title = '' if label is None else CLASSES[label]
correct = True
if predictions is not None:
title, correct = title_from_label_and_target(predictions[i], label)
 dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3
subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)
 plt.tight_layout()
if label is None and predictions is None:
plt.subplots_adjust(wspace=0, hspace=0)
 else:
plt.subplots_adjust(wspace=SPACING, hspace=SPACING)
 plt.show()
def display_confusion_matrix(cmat, score, precision, recall):
plt.figure(figsize=(15,15))
ax = plt.gca()
ax.matshow(cmat, cmap='Reds')
 ax.set_xticks(range(len(CLASSES)))
 ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})
 plt.setp(ax.get_xticklabels() , rotation=45, ha="left", rotation_mode="anchor")
 ax.set_yticks(range(len(CLASSES)))
 ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})
 plt.setp(ax.get_yticklabels() , rotation=45, ha="right", rotation_mode="anchor")
 titlestring = ""
if score is not None:
titlestring += 'f1 = {:.3f} '.format(score)
 if precision is not None:
titlestring += '
precision = {:.3f} '.format(precision)
 if recall is not None:
titlestring += '
recall = {:.3f} '.format(recall)
 if len(titlestring)> 0:
ax.text(101, 1, titlestring, fontdict={'fontsize': 18, 'horizontalalignment':'right', 'verticalalignment':'top', 'color':'
plt.show()
def display_training_curves(training, validation, title, subplot):
if subplot%10==1:
plt.subplots(figsize=(10,10), facecolor='
plt.tight_layout()
ax = plt.subplot(subplot)
 ax.set_facecolor('
ax.plot(training)
 ax.plot(validation)
 ax.set_title('model '+ title)
 ax.set_ylabel(title)
 ax.set_xlabel('epoch')
 ax.legend(['train', 'valid.'] )
Cat_toMode =('MSZoning','Electrical','KitchenQual','Exterior1st','Exterior2nd',
'Utilities','SaleType','Functional')
 for c in Cat_toMode:
df_all[c] = df_all[c].fillna(df_all[c].mode() [0] )<data_type_conversions>
def random_blockout(img, sl=0.1, sh=0.2, rl=0.4):
p=random.random()
if p>=0.25:
w, h, c = IMAGE_SIZE[0], IMAGE_SIZE[1], 3
origin_area = tf.cast(h*w, tf.float32)
 e_size_l = tf.cast(tf.round(tf.sqrt(origin_area * sl * rl)) , tf.int32)
 e_size_h = tf.cast(tf.round(tf.sqrt(origin_area * sh / rl)) , tf.int32)
 e_height_h = tf.minimum(e_size_h, h)
 e_width_h = tf.minimum(e_size_h, w)
 erase_height = tf.random.uniform(shape=[], minval=e_size_l, maxval=e_height_h, dtype=tf.int32)
 erase_width = tf.random.uniform(shape=[], minval=e_size_l, maxval=e_width_h, dtype=tf.int32)
 erase_area = tf.zeros(shape=[erase_height, erase_width, c])
 erase_area = tf.cast(erase_area, tf.uint8)
 pad_h = h - erase_height
pad_top = tf.random.uniform(shape=[], minval=0, maxval=pad_h, dtype=tf.int32)
 pad_bottom = pad_h - pad_top
pad_w = w - erase_width
pad_left = tf.random.uniform(shape=[], minval=0, maxval=pad_w, dtype=tf.int32)
 pad_right = pad_w - pad_left
erase_mask = tf.pad([erase_area], [[0,0],[pad_top, pad_bottom], [pad_left, pad_right], [0,0]], constant_values=1)
 erase_mask = tf.squeeze(erase_mask, axis=0)
 erased_img = tf.multiply(tf.cast(img,tf.float32), tf.cast(erase_mask, tf.float32))
return tf.cast(erased_img, img.dtype)
 else:
return tf.cast(img, img.dtype )
Num_toZero =('GarageArea','GarageCars','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF',
'BsmtFullBath','BsmtHalfBath','GarageYrBlt','MasVnrArea')
 for c in Num_toZero:
df_all[c] = df_all[c].fillna(0 )<feature_engineering>
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def data_augment(image, label):
image = tf.image.random_flip_left_right(image, seed=SEED)
 image = random_blockout(image)
 return image, label
def get_training_dataset() :
dataset = load_dataset(TRAINING_FILENAMES, labeled=True)
 dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(ordered=False):
dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)
 NUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))
df_all["LotFrontage"] = df_all.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))<data_type_conversions>
if SKIP_VALIDATION:
TRAINING_FILENAMES = TRAINING_FILENAMES + VALIDATION_FILENAMES
df_all['MSSubClass'] = df_all['MSSubClass'].astype(str )<feature_engineering>
print("Training data shapes:")
 for image, label in get_training_dataset().take(3):
print(image.numpy().shape, label.numpy().shape)
 print("Training data label examples:", label.numpy())
 print("Validation data shapes:")
 for image, label in get_validation_dataset().take(3):
print(image.numpy().shape, label.numpy().shape)
 print("Validation data label examples:", label.numpy())
 print("Test data shapes:")
 for image, idnum in get_test_dataset().take(3):
print(image.numpy().shape, idnum.numpy().shape)
 print("Test data IDs:", idnum.numpy().astype('U'))
skewed_feats = df_all[numeric].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.75]
skewed_feats = skewed_feats.index
df_all[skewed_feats] = np.log1p(df_all[skewed_feats])
<categorify>
training_dataset = get_training_dataset()
training_dataset = training_dataset.unbatch().batch(20)
 train_batch = iter(training_dataset )
df_all = df_all.replace({"Alley" : {"None" : 0, "Grvl" : 1, "Pave" : 2},
"BsmtCond" : {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"BsmtExposure" : {"None" : 0, "Mn" : 1, "Av": 2, "Gd" : 3},
"BsmtFinType1" : {"None" : 0, "Unf" : 1, "LwQ": 2, "Rec" : 3, "BLQ" : 4,
"ALQ" : 5, "GLQ" : 6},
"BsmtFinType2" : {"None" : 0, "Unf" : 1, "LwQ": 2, "Rec" : 3, "BLQ" : 4,
"ALQ" : 5, "GLQ" : 6},
"BsmtQual" : {"None" : 0, "Po" : 1, "Fa" : 2, "TA": 3, "Gd" : 4, "Ex" : 5},
"ExterCond" : {"Po" : 1, "Fa" : 2, "TA": 3, "Gd": 4, "Ex" : 5},
"ExterQual" : {"Po" : 1, "Fa" : 2, "TA": 3, "Gd": 4, "Ex" : 5},
"FireplaceQu" : {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"Functional" : {"Sal" : 1, "Sev" : 2, "Maj2" : 3, "Maj1" : 4, "Mod": 5,
"Min2" : 6, "Min1" : 7, "Typ" : 8},
"GarageCond" : {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"GarageQual" : {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"HeatingQC" : {"Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"KitchenQual" : {"Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"LandSlope" : {"Sev" : 1, "Mod" : 2, "Gtl" : 3},
"LotShape" : {"IR3" : 1, "IR2" : 2, "IR1" : 3, "Reg" : 4},
"PavedDrive" : {"N" : 0, "P" : 1, "Y" : 2},
"PoolQC" : {"None" : 0, "Fa" : 1, "TA" : 2, "Gd" : 3, "Ex" : 4},
"Street" : {"Grvl" : 1, "Pave" : 2},
"Utilities" : {"ELO" : 1, "NoSeWa" : 2, "NoSewr" : 3, "AllPub" : 4}}
)<feature_engineering>
display_batch_of_images(next(train_batch))
df_all['Bathrooms'] = df_all['FullBath'] +(df_all['HalfBath']*0.5)+ df_all['BsmtFullBath'] +(df_all['BsmtHalfBath']*0.5)
 df_all['FireplaceScore'] = df_all['Fireplaces'] * df_all['FireplaceQu']
df_all['GarageScore'] = df_all['GarageCars'] * df_all['GarageQual'] * df_all['GarageArea']
df_all['HouseAge'] = df_all['YrSold'] - df_all['YearBuilt']
df_all['TotalLivingSF'] = df_all['GrLivArea'] + df_all['TotalBsmtSF'] - df_all['LowQualFinSF']<drop_column>
test_dataset = get_test_dataset()
test_dataset = test_dataset.unbatch().batch(20)
 test_batch = iter(test_dataset )
df_all = df_all.drop(['FullBath', 'HalfBath', 'BsmtFullBath', 'BsmtHalfBath',
'GarageCars','GarageArea','GarageQual','GarageCond',
'FireplaceQu','Fireplaces',
'PoolArea',
'Utilities', 'Street',
'GarageYrBlt'
], axis = 1 )<categorify>
display_batch_of_images(next(test_batch))
df_all = pd.get_dummies(df_all, drop_first=True )<drop_column>
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)
 rng = [i for i in range(25 if EPOCHS<25 else EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))
df_all = df_all.drop(['SalePrice'], axis = 1)
 train = df_all[:len_train]
test = df_all[len_train:]<split>
!pip install -q efficientnet
X_train, X_test, y_train, y_test = train_test_split(train, y, test_size = 0.3, random_state = 0 )<choose_model_class>
with strategy.scope() :
enet = efn.EfficientNetB7(input_shape=[*IMAGE_SIZE, 3], weights='noisy-student', include_top=False)
 enet.trainable = True
model1 = tf.keras.Sequential([
enet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model1.compile(
 optimizer=tf.keras.optimizers.Adam() ,
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 model1.summary()
regressor = LinearRegression()
regressor.fit(X_train, y_train)
 train_lm = regressor.predict(X_train)
 test_lm = regressor.predict(X_test)
 ridge = Ridge(alpha = 10)
 ridge.fit(X_train, y_train)
 ridge_train_lm = ridge.predict(X_train)
 ridge_test_lm = ridge.predict(X_test)
 lasso = Lasso(max_iter=500,alpha = 0.001)
 lasso.fit(X_train, y_train)
 lasso_train_lm = lasso.predict(X_train)
 lasso_test_lm = lasso.predict(X_test)
 en = ElasticNet(max_iter=500,alpha = 0.001)
 en.fit(X_train, y_train)
 en_train_lm = en.predict(X_train)
 en_test_lm = en.predict(X_test)
 dtr = DecisionTreeRegressor(max_depth=5)
 dtr.fit(X_train, y_train)
 train_dtr = dtr.predict(X_train)
 test_dtr = dtr.predict(X_test)
 rf = RandomForestRegressor(random_state = 0)
 rf.fit(X_train, y_train)
 train_rf = rf.predict(X_train)
 test_rf = rf.predict(X_test )<compute_train_metric>
if not SKIP_VALIDATION:
history1 = model1.fit(get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
validation_data=get_validation_dataset() ,
callbacks = [lr_callback])
 else:
history1 = model1.fit(get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks = [lr_callback] )
scorer = metrics.make_scorer(metrics.mean_squared_error, greater_is_better = False)
 kfolds = KFold(n_splits=10, shuffle=True, random_state=42 )<compute_train_metric>
with strategy.scope() :
densenet = tf.keras.applications.DenseNet201(input_shape=[*IMAGE_SIZE, 3], weights='imagenet', include_top=False)
 densenet.trainable = True
model2 = tf.keras.Sequential([
densenet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model2.compile(
 optimizer=tf.keras.optimizers.Adam() ,
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 model2.summary()
print('Linear Regression')
 print("RMSE on Training set :", rmse_cv(regressor,X_train,y_train ).mean())
 print("RMSE on Test set :", rmse_cv(regressor,X_test,y_test ).mean())
 print('RMSLE score on train data: ', rmsle(y_train, regressor.predict(X_train)))
 print('RMSLE score on test data: ', rmsle(y_test, regressor.predict(X_test)))
 print('-'*25)
 print('Linear Regression Ridge')
 print("RMSE on Training set :", rmse_cv(ridge,X_train,y_train ).mean())
 print("RMSE on Test set :", rmse_cv(ridge,X_test,y_test ).mean())
 print('RMSLE score on train data: ', rmsle(y_train, ridge.predict(X_train)))
 print('RMSLE score on test data: ', rmsle(y_test, ridge.predict(X_test)))
 print('-'*25)
 print('Linear Regression Lasso')
 print("RMSE on Training set :", rmse_cv(lasso,X_train,y_train ).mean())
 print("RMSE on Test set :", rmse_cv(lasso,X_test,y_test ).mean())
 print('RMSLE score on train data: ', rmsle(y_train, lasso.predict(X_train)))
 print('RMSLE score on test data: ', rmsle(y_test, lasso.predict(X_test)))
 print('-'*25)
 print('Linear Regression ElasticNet')
 print("RMSE on Training set :", rmse_cv(en,X_train,y_train ).mean())
 print("RMSE on Test set :", rmse_cv(en,X_test,y_test ).mean())
 print('RMSLE score on train data: ', rmsle(y_train, en.predict(X_train)))
 print('RMSLE score on test data: ', rmsle(y_test, en.predict(X_test)))
 print('-'*25)
 print('Decision Tree Regression')
 print("RMSE on Training set :", rmse_cv(dtr,X_train,y_train ).mean())
 print("RMSE on Test set :", rmse_cv(dtr,X_test,y_test ).mean())
 print('RMSLE score on train data: ', rmsle(y_train, dtr.predict(X_train)))
 print('RMSLE score on test data: ', rmsle(y_test, dtr.predict(X_test)))
 print('-'*25)
 print('Random Forest Regression')
 print("RMSE on Training set :", rmse_cv(rf,X_train,y_train ).mean())
 print("RMSE on Test set :", rmse_cv(rf,X_test,y_test ).mean())
 print('RMSLE score on train data: ', rmsle(y_train, rf.predict(X_train)))
 print('RMSLE score on test data: ', rmsle(y_test, rf.predict(X_test)) )<choose_model_class>
if not SKIP_VALIDATION:
history2 = model2.fit(get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
validation_data=get_validation_dataset() ,
callbacks = [lr_callback])
 else:
history2 = model2.fit(get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks = [lr_callback] )
ridge = Ridge(alpha = cv_ridge[cv_ridge == min(cv_ridge)].index[0])
 ridge.fit(X_train, y_train)
 ridge_train_lm = ridge.predict(X_train)
 ridge_test_lm = ridge.predict(X_test)
 lasso = Lasso(max_iter=500,alpha = cv_lasso[cv_lasso == min(cv_lasso)].index[0])
 lasso.fit(X_train, y_train)
 lasso_train_lm = lasso.predict(X_train)
 lasso_test_lm = lasso.predict(X_test)
 en = ElasticNet(max_iter=500,alpha = cv_elasticnet[cv_elasticnet == min(cv_elasticnet)].index[0])
 en.fit(X_train, y_train)
 en_train_lm = en.predict(X_train)
 en_test_lm = en.predict(X_test )<compute_train_metric>
if not SKIP_VALIDATION:
cmdataset = get_validation_dataset(ordered=True)
 images_ds = cmdataset.map(lambda image, label: image)
 labels_ds = cmdataset.map(lambda image, label: label ).unbatch()
cm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES)) ).numpy()
m1 = model1.predict(images_ds)
 m2 = model2.predict(images_ds)
 scores = []
for alpha in np.linspace(0,1,100):
cm_probabilities = alpha*m1+(1-alpha)*m2
cm_predictions = np.argmax(cm_probabilities, axis=-1)
 scores.append(f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro'))
best_alpha = np.argmax(scores)/100
else:
best_alpha = 0.51
print('Best alpha: ' + str(best_alpha))
print('Linear Regression Ridge')
 print("RMSE on Training set :", rmse_cv(ridge,X_train,y_train ).mean())
 print("RMSE on Test set :", rmse_cv(ridge,X_test,y_test ).mean())
 print('RMSLE score on train data: ', rmsle(y_train, ridge.predict(X_train)))
 print('RMSLE score on test data: ', rmsle(y_test, ridge.predict(X_test)))
 print('-'*25)
 print('Linear Regression Lasso')
 print("RMSE on Training set :", rmse_cv(lasso,X_train,y_train ).mean())
 print("RMSE on Test set :", rmse_cv(lasso,X_test,y_test ).mean())
 print('RMSLE score on train data: ', rmsle(y_train, lasso.predict(X_train)))
 print('RMSLE score on test data: ', rmsle(y_test, lasso.predict(X_test)))
 print('-'*25)
 print('Linear Regression ElasticNet')
 print("RMSE on Training set :", rmse_cv(en,X_train,y_train ).mean())
 print("RMSE on Test set :", rmse_cv(en,X_test,y_test ).mean())
 print('RMSLE score on train data: ', rmsle(y_train, en.predict(X_train)))
 print('RMSLE score on test data: ', rmsle(y_test, en.predict(X_test)) )<define_search_model>
if not SKIP_VALIDATION:
cmat = confusion_matrix(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)))
 score = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 precision = precision_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 recall = recall_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 display_confusion_matrix(cmat, score, precision, recall)
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))
alphas_ridge = [2.5, 5, 7.5, 10, 12.5, 15, 17.5, 20, 22.5, 25, 27.5, 30]
alphas_lasso = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008, 0.0009, 0.001, 0.0011]
alphas_en = [0.0006, 0.0007, 0.0008, 0.0009, 0.001, 0.0011, 0.0012, 0.0013, 0.0014, 0.0015, 0.0016]
l1ratio_en = [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 0.99, 1]
ridge = make_pipeline(RidgeCV(alphas=alphas_ridge, cv=kfolds))
lasso = make_pipeline(LassoCV(max_iter=500, alphas=alphas_lasso, random_state=42, cv=kfolds))
elasticnet = make_pipeline(ElasticNetCV(max_iter=500,alphas=alphas_en, cv=kfolds, l1_ratio=l1ratio_en))<train_model>
def predict_tta(model, n_iter):
probs = []
for i in range(n_iter):
test_ds = get_test_dataset(ordered=True)
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probs.append(model.predict(test_images_ds,verbose=0))
return probs
<compute_train_metric><EOS> 
test_ds = get_test_dataset(ordered=True)
 print('Calculating predictions...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probs1 = np.mean(predict_tta(model1, TTA_NUM), axis=0)
 probs2 = np.mean(predict_tta(model2, TTA_NUM), axis=0)
 probabilities = best_alpha*probs1 +(1-best_alpha)*probs2
predictions = np.argmax(probabilities, axis=-1)
 print('Generating submission file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )