Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<set_options>
train = pd.read_csv('.. /input/siim-isic-melanoma-classification/train.csv')
 print('Examples WITH Melanoma')
 imgs = train.loc[train.target==1].sample(10 ).image_name.values
plt.figure(figsize=(20,8))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()
print('Examples WITHOUT Melanoma')
 imgs = train.loc[train.target==0].sample(10 ).image_name.values
plt.figure(figsize=(20,8))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()
def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
warnings.simplefilter(action='ignore', category=FutureWarning)
 %matplotlib inline
color = sns.color_palette()
sns.set_style('darkgrid' )<load_from_csv>
!pip install -q efficientnet >> /dev/null
train_path = ".. /input/eda-and-preprocessing-of-the-datasets/preprocessed_train.csv"
test_path = ".. /input/eda-and-preprocessing-of-the-datasets/preprocessed_test.csv"
train = pd.read_csv(train_path)
 test = pd.read_csv(test_path )<prepare_x_and_y>
import pandas as pd, numpy as np
from kaggle_datasets import KaggleDatasets
import tensorflow as tf, re, math
import tensorflow.keras.backend as K
import efficientnet.tfkeras as efn
from sklearn.model_selection import KFold
from sklearn.metrics import roc_auc_score
import matplotlib.pyplot as plt
n_train = train.shape[0]
n_test = test.shape[0]
train_y = train['SalePrice']
train_ids = train['Id']
test_ids = test['Id']
train = train.drop('SalePrice', axis=1)
 train = train.drop('Id', axis=1)
 test = test.drop('Id', axis=1 )<load_pretrained>
DEVICE = "TPU"
SEED = 42
FOLDS = 5
IMG_SIZES = [512]*FOLDS
INC2019 = [1]*FOLDS
INC2018 = [1,1,1,1,1]
M1 = [1]*5
M2 = [1]*5
M3 = [1]*5
M4 = [1]*5
MS = [1]*5
BATCH_SIZES = [32]*FOLDS
EPOCHS = [0]*FOLDS
EFF_NETS = [6,6,6,6,6]
WGTS = [1/FOLDS]*FOLDS
TTA = 11
def reset_weights(model):
session = K.get_session()
for layer in model.layers:
if hasattr(layer, 'kernel_initializer'):
layer.kernel.initializer.run(session=session )<choose_model_class>
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )
def get_callbacks(bm_path):
checkpointer = keras.callbacks.ModelCheckpoint(
 filepath=bm_path,
monitor='val_loss',
save_best_only=True,
save_weights_only=False,
mode='auto',
period=1 
)
 lr_patience = 10
lr_scheduler = keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss',
factor=0.95,
patience=lr_patience,
verbose=0,
mode='auto',
min_delta=0.0001,
cooldown=25,
min_lr=0 
)
 return lr_scheduler, checkpointer<choose_model_class>
GCS_PATH = [None]*FOLDS; GCS_PATH2 = [None]*FOLDS; GCS_PATH3 = [None]*FOLDS; GCS_PATH4 = [None]*FOLDS
for i,k in enumerate(IMG_SIZES[:FOLDS]):
GCS_PATH[i] = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(k,k))
GCS_PATH2[i] = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(k,k))
GCS_PATH3[i] = KaggleDatasets().get_gcs_path('malignant-v2-%ix%i'%(k,k))
for i,k in enumerate(IMG_SIZES[:FOLDS]):
GCS_PATH4[i] = KaggleDatasets().get_gcs_path('unknowndatas')
 files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/train*.tfrec')))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec')) )
def get_model(input_shape, num_neurons):
model = keras.Sequential([
keras.layers.Dense(num_neurons, activation='linear', input_shape=input_shape),
keras.layers.Dropout(0.25),
keras.layers.LeakyReLU(alpha=0.1),
keras.layers.Dense(1)
 ])
 return model<prepare_x_and_y>
ROT_ = 180.0
SHR_ = 2.0
HZOOM_ = 8.0
WZOOM_ = 8.0
HSHIFT_ = 8.0
WSHIFT_ = 8.0
X = train.values
Y = train_y.values
BATCH_SIZE = 2
NUM_FOLDS = 5
EPOCHS = 250
HOLDOUT_DICT = {'fold{}'.format(fc): {} for fc in range(NUM_FOLDS)}
if os.path.exists('out'):
shutil.rmtree('out')
 os.makedirs('out')
 BASE_MODELS = {'m0' : 16, 'm1' : 32, 'm2' : 64, 'm3' : 128}<feature_engineering>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, DIM=256):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )
best_models = {mod_name : [] for mod_name in BASE_MODELS.keys() }
r2_stacked = []
mse_stacked = []
mae_stacked = []
rmsle_stacked = []
kfold = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=42)
 fold_counter = 0
for train_idxs, test_idxs in kfold.split(X, Y):
HOLDOUT_DICT['fold{}'.format(fold_counter)]['gt'] = Y[test_idxs]
for model_name, model_config in BASE_MODELS.items() :
model = get_model(( train.shape[1],), num_neurons=model_config)
 reset_weights(model)
 optimizer = 'adam'
model.compile(loss='mse', optimizer=optimizer, metrics=['mse','mae'])
 bm_path = "out/fold{}{}_config{}.h5".format(fold_counter, model_name, model_config)
 lr_scheduler, checkpointer = get_callbacks(bm_path)
 history = model.fit(
 x=X[train_idxs],
y=Y[train_idxs],
validation_data=(X[test_idxs], Y[test_idxs]),
batch_size=BATCH_SIZE,
epochs=EPOCHS,
callbacks=[
checkpointer,
],
verbose=0,
shuffle=True 
)
 model.load_weights(bm_path)
 predict_val = model.predict(X[test_idxs] ).flatten()
HOLDOUT_DICT['fold{}'.format(fold_counter)]['{}'.format(model_name)] = predict_val.copy()
r2 = r2_score(predict_val, Y[test_idxs])
 mse = mean_squared_error(predict_val, Y[test_idxs])
 mae = mean_absolute_error(predict_val, Y[test_idxs])
 rmsle = math.sqrt(mse)
 best_models[model_name].append(bm_path)
 mse_stacked.append(mse)
 mae_stacked.append(mae)
 rmsle_stacked.append(rmsle)
 r2_stacked.append(r2)
 plot_history(history)
 print("{}/{} folds model {}: MSE: {} || MAE: {} || RMSLE: {} || R2: {}".format(fold_counter + 1, NUM_FOLDS, model_name, mse, mae, rmsle, r2))
K.clear_session()
fold_counter += 1
print("AVG_RMSLE: {} || AVG_R2: {}".format(np.mean(rmsle_stacked), np.mean(r2_stacked)) )<import_modules>
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img, augment=True, dim=256):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img,DIM=dim)
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 img = tf.reshape(img, [dim,dim, 3])
 return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )
from sklearn.linear_model import Lasso<import_modules>
def get_dataset(files, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True, batch_size=16, dim=256):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, imgname_or_label:(prepare_image(img, augment=augment, dim=dim),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(batch_size * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds
from sklearn.linear_model import Lasso<compute_train_metric>
def binary_focal_loss(gamma=2., alpha=.25):

 def binary_focal_loss_fixed(y_true, y_pred):

 pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
epsilon = K.epsilon()
pt_1 = K.clip(pt_1, epsilon, 1.- epsilon)
 pt_0 = K.clip(pt_0, epsilon, 1.- epsilon)
 return -K.sum(alpha * K.pow(1.- pt_1, gamma)* K.log(pt_1)) \
-K.sum(( 1 - alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0))
return binary_focal_loss_fixed
def categorical_focal_loss(gamma=2., alpha=.25):

 def categorical_focal_loss_fixed(y_true, y_pred):

 y_pred /= K.sum(y_pred, axis=-1, keepdims=True)
 epsilon = K.epsilon()
y_pred = K.clip(y_pred, epsilon, 1.- epsilon)
 cross_entropy = -y_true * K.log(y_pred)
 loss = alpha * K.pow(1 - y_pred, gamma)* cross_entropy
return K.sum(loss, axis=1)
 return categorical_focal_loss_fixed
meta_model = Lasso(alpha=0.0005, random_state=1)
 meta_model.fit(X=holdouts_df[list(BASE_MODELS.keys())], y=holdouts_df['gt'])
 rmsle = math.sqrt(mean_squared_error(meta_model.predict(holdouts_df[list(BASE_MODELS.keys())]), holdouts_df['gt']))
print(rmsle )<choose_model_class>
EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3,
efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6]
def build_model(dim=128, ef=0):
inp = tf.keras.layers.Input(shape=(dim,dim,3))
base = EFNS[ef](input_shape=(dim,dim,3),weights='noisy-student',include_top=False)
 x = base(inp)
 x = tf.keras.layers.GlobalAveragePooling2D()(x)
 x = tf.keras.layers.Dense(1,activation='sigmoid' )(x)
 model = tf.keras.Model(inputs=inp,outputs=x)
 opt = tf.keras.optimizers.Adam(learning_rate=0.001)
 loss = binary_focal_loss()
model.compile(optimizer=opt,loss=loss,metrics=['AUC'])
 return model
def ensemble_models(models, model_input):
yModels = [model(model_input)for model in models]
yAvg = average(yModels)
 modelEns = Model(inputs=model_input, outputs=yAvg, name='ensembleModel')
 return modelEns<predict_on_test>
def get_lr_callback(batch_size=8):
lr_start = 0.000005
lr_max = 0.00000125 * REPLICAS * batch_size
lr_min = 0.000001
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.8
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback
def metamodel_predict(ensemble_models_dict, meta_model, data):
res = {}
for model_name, model in ensemble_models_dict.items() :
res[model_name] = model.predict(data ).flatten()
res_df = pd.DataFrame(res)
 meta_res = meta_model.predict(res_df.values)
 res_df['meta'] = meta_res
return res_df, meta_res<compute_test_metric>
VERBOSE = 0
DISPLAY_PLOT = True
skf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)
 oof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = []
preds = np.zeros(( count_data_items(files_test),1))
for fold,(idxT,idxV)in enumerate(skf.split(np.arange(15))):
if DEVICE=='TPU':
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 print('
print(' 
(IMG_SIZES[fold],EFF_NETS[fold],BATCH_SIZES[fold]*REPLICAS))
files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxT])
 if INC2019[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1])
 print('
if INC2018[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2])
 print('
for k in range(M1[fold]):
files_train += tf.io.gfile.glob([GCS_PATH3[fold] + '/train%.2i*.tfrec'%x for x in idxT])
 print('
for k in range(M2[fold]):
files_train += tf.io.gfile.glob([GCS_PATH3[fold] + '/train%.2i*.tfrec'%x for x in idxT+15])
 print('
for k in range(M3[fold]):
files_train += tf.io.gfile.glob([GCS_PATH3[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1+30])
 print('
for k in range(M4[fold]):
files_train += tf.io.gfile.glob([GCS_PATH3[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+30])
 print('
for k in range(MS[fold]):
files_train += tf.io.gfile.glob([GCS_PATH4[fold] + '/train%.2i*.tfrec'%x for x in idxT])
 print('
np.random.shuffle(files_train); print('
files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxV])
 if INC2019[fold]:
files_valid += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxV*2+1])
 print('
if INC2018[fold]:
files_valid += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxV*2])
 print('
for k in range(M1[fold]):
files_valid += tf.io.gfile.glob([GCS_PATH3[fold] + '/train%.2i*.tfrec'%x for x in idxV])
 print('
for k in range(M2[fold]):
files_valid += tf.io.gfile.glob([GCS_PATH3[fold] + '/train%.2i*.tfrec'%x for x in idxV+15])
 print('
for k in range(M3[fold]):
files_valid += tf.io.gfile.glob([GCS_PATH3[fold] + '/train%.2i*.tfrec'%x for x in idxV*2+1+30])
 print('
for k in range(M4[fold]):
files_valid += tf.io.gfile.glob([GCS_PATH3[fold] + '/train%.2i*.tfrec'%x for x in idxV*2+30])
 print('
for k in range(MS[fold]):
print('
files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/test*.tfrec')))
 K.clear_session()
with strategy.scope() :
model = build_model(dim=IMG_SIZES[fold],ef=EFF_NETS[fold])
 model.load_weights('.. /input/melanomaefnet6/fold-%i.h5'%fold)
 sv = tf.keras.callbacks.ModelCheckpoint(
 'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min', save_freq='epoch')
 print('Training...')
 history = model.fit(
 get_dataset(files_train, augment=True, shuffle=True, repeat=True,
dim=IMG_SIZES[fold], batch_size = BATCH_SIZES[fold]),
epochs=EPOCHS[fold], callbacks = [sv,get_lr_callback(BATCH_SIZES[fold])],
steps_per_epoch=count_data_items(files_train)/BATCH_SIZES[fold]//REPLICAS,
validation_data=get_dataset(files_valid,augment=False,shuffle=False,
repeat=False,dim=IMG_SIZES[fold]),
verbose=VERBOSE 
)
 print('Loading best model...')
 print('Predicting OOF with TTA...')
 ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_valid = count_data_items(files_valid); STEPS = TTA * ct_valid/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,]
print(pred.shape)
 oof_pred.append(np.mean(pred.reshape(( ct_valid,TTA),order='F'),axis=1))
ds_valid = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=True, return_image_names=True)
 oof_tar.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]))
oof_folds.append(np.ones_like(oof_tar[-1],dtype='int8')*fold)
 ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 oof_names.append(np.array([img_name.numpy().decode("utf-8")for img, img_name in iter(ds.unbatch())]))
print('Predicting Test with TTA...')
 ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,]
print(pred.shape)
 preds[:,0] += np.mean(pred.reshape(( ct_test,TTA),order='F'),axis=1)* WGTS[fold]
auc = roc_auc_score(oof_tar[-1],oof_pred[-1])
res_df, meta_res = metamodel_predict(ensembles, meta_model, X)
 for col in res_df.columns:
pred = res_df[col]
print("Model {}: RMSLE: {} || R2: {}".format(col, math.sqrt(mean_squared_error(pred, Y)) , r2_score(pred, Y)) )<save_to_csv>
oof = np.concatenate(oof_pred); true = np.concatenate(oof_tar);
names = np.concatenate(oof_names); folds = np.concatenate(oof_folds)
 auc = roc_auc_score(true,oof)
 print('Overall OOF AUC with TTA = %.3f'%auc)
 df_oof = pd.DataFrame(dict(
 image_name = names, target=true, pred = oof, fold=folds))
df_oof.to_csv('oof.csv',index=False)
 df_oof.head()
res_submission_df, meta_res_submission = metamodel_predict(ensembles, meta_model, test)
 sub = pd.DataFrame()
sub['Id'] = test_ids
sub['SalePrice'] = np.expm1(meta_res_submission)
 sub.to_csv(os.path.join('submission_sm.csv'),index=False)
 sub.head()<load_from_csv>
ds = get_dataset(files_test, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )
<train_model><EOS> 
submission = pd.DataFrame(dict(image_name=image_names, target=preds[:,0]))
submission = submission.sort_values('image_name')
 submission.to_csv('submission.csv', index=False)
 submission.head()