Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<split>
warnings.filterwarnings('ignore')
 %matplotlib inline
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.15,
stratify=y,
random_state=42)
 cv = StratifiedKFold(5, shuffle=True, random_state=42 )<choose_model_class>
df_train.columns[df_train.isnull().any() ]
df_train_missing=df_train[df_train.columns[df_train.isnull().any() ]]
def num_missing(x):
return sum(x.isnull())
 df_train_missing.apply(num_missing, axis=0)
 df_train_missing_table=round(df_train_missing.apply(num_missing, axis=0)/1460,4)*100
df_train_missing_table =pd.DataFrame({'MissingPercentage':df_train_missing_table})
 df_train_missing_table
xg = xgb.XGBClassifier(
 n_estimators=750,
min_child_weight=0.81,
learning_rate=0.025,
max_depth=2,
subsample=0.80,
colsample_bytree=0.42,
gamma=0.10,
random_state=42,
n_jobs=-1,
)<define_variables>
var1=['Alley','MasVnrType','MasVnrArea','BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','FireplaceQu','GarageType','GarageYrBlt','GarageFinish','GarageQual','GarageCond','PoolQC','Fence','MiscFeature']
df_train[var1]=df_train[var1].fillna('NA')
 df_train['LotFrontage']=df_train['LotFrontage'].fillna(df_train['LotFrontage'].mean())
 df_train['Electrical']=df_train['Electrical'].fillna(mode(df_train['Electrical']))
estimators = [xg]<compute_train_metric>
df_test.columns[df_test.isnull().any() ]
df_test_missing=df_test[df_test.columns[df_test.isnull().any() ]]
df_test_missing.apply(num_missing, axis=0)
 var1=['BsmtQual','Alley','Utilities','Exterior1st','Exterior2nd','MasVnrType','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','FireplaceQu','GarageType','GarageYrBlt','GarageFinish','GarageQual','GarageCond','PoolQC','Fence','MiscFeature']
df_test[var1]=df_test[var1].fillna('NA')
 df_test['LotFrontage']=df_test['LotFrontage'].fillna(df_test['LotFrontage'].mean())
 df_test['MasVnrArea']=df_test['MasVnrArea'].fillna(df_test['MasVnrArea'].mean())
 df_test['BsmtFinSF1']=df_test['BsmtFinSF1'].fillna(df_test['BsmtFinSF1'].mean())
 df_test['BsmtFinSF2']=df_test['BsmtFinSF2'].fillna(df_test['BsmtFinSF2'].mean())
 df_test['BsmtUnfSF']=df_test['BsmtUnfSF'].fillna(df_test['BsmtUnfSF'].mean())
 df_test['TotalBsmtSF']=df_test['TotalBsmtSF'].fillna(df_test['TotalBsmtSF'].mean())
 df_test['GarageCars']=df_test['GarageCars'].fillna(df_test['GarageCars'].mean())
 df_test['GarageArea']=df_test['GarageArea'].fillna(df_test['GarageArea'].mean())
 df_test['GarageArea']=df_test['GarageArea'].fillna(df_test['GarageArea'].mean())
 df_test['MSZoning']=df_test['MSZoning'].fillna(mode('MSZoning'))
df_test['BsmtHalfBath']=df_test['BsmtHalfBath'].fillna(0)
 df_test['KitchenQual']=df_test['KitchenQual'].fillna('TA')
 df_test['Functional']=df_test['Functional'].fillna(mode('Functional'))
df_test['BsmtFullBath']=df_test['BsmtFullBath'].fillna(0)
 df_test['SaleType']=df_test['SaleType'].fillna(mode('SaleType'))
def model_check(X_train, y_train, estimators, cv):
model_table = pd.DataFrame()
row_index = 0
for est in estimators:
MLA_name = est.__class__.__name__
model_table.loc[row_index, 'Model Name'] = MLA_name
cv_results = cross_validate(est,
X_train,
y_train,
cv=cv,
scoring='roc_auc',
return_train_score=True,
n_jobs=-1)
 model_table.loc[row_index,
'Train roc Mean'] = cv_results['train_score'].mean()
model_table.loc[row_index,
'Test roc Mean'] = cv_results['test_score'].mean()
model_table.loc[row_index, 'Test Std'] = cv_results['test_score'].std()
model_table.loc[row_index, 'Time'] = cv_results['fit_time'].mean()
row_index += 1
model_table.sort_values(by=['Test roc Mean'],
ascending=False,
inplace=True)
 return model_table<compute_train_metric>
df_train=df_train[np.abs(df_train.SalePrice-df_train.SalePrice.mean())<=(6*df_train.SalePrice.std())]
xg.fit(X_train, y_train)
 validation = xg.predict_proba(X_test)[:, 1]
roc_auc_score(y_test, validation )<prepare_x_and_y>
df_test['TT_SF']=df_test['1stFlrSF']+df_test['2ndFlrSF']+df_test['TotalBsmtSF']+df_test['GarageArea']+df_test['GrLivArea']
df_test['TT_bathrooms']=df_test['FullBath']+df_test['HalfBath']+df_test['BsmtFullBath']+df_test['HalfBath']
df_test['TT_rooms']=df_test['TotRmsAbvGrd']+df_test['BsmtFullBath']+df_test['BsmtHalfBath']
df_train['TT_SF']=df_train['1stFlrSF']+df_train['2ndFlrSF']+df_train['TotalBsmtSF']+df_train['GarageArea']+df_train['GrLivArea']
df_train['TT_bathrooms']=df_train['FullBath']+df_train['HalfBath']+df_train['BsmtFullBath']+df_train['HalfBath']
df_train['TT_rooms']=df_train['TotRmsAbvGrd']+df_train['BsmtFullBath']+df_train['BsmtHalfBath']
adv_train = train.copy()
adv_train.drop('target', axis=1, inplace=True)
 adv_test = test.copy()
adv_train['dataset_label'] = 0
adv_test['dataset_label'] = 1
adv_master = pd.concat([adv_train, adv_test], axis=0)
 adv_X = adv_master.drop('dataset_label', axis=1)
 adv_y = adv_master['dataset_label']<split>
adv_X_train, adv_X_test, adv_y_train, adv_y_test = train_test_split(adv_X,
adv_y,
test_size=0.4,
stratify=adv_y,
random_state=42 )<train_model>
df_train['SalePrice'] = np.log(df_train['SalePrice'] )
xg_adv = xgb.XGBClassifier(
 random_state=42,
n_jobs=-1, 
)
 xg_adv.fit(adv_X_train, adv_y_train)
 validation = xg_adv.predict_proba(adv_X_test)[:,1]<split>
numeric_vars = df_train.dtypes[df_train.dtypes != "object"].index
skewed_vars = df_train[numeric_vars].skew().sort_values(ascending=False)
 skewness = pd.DataFrame({'Skew':skewed_vars})
 skewness = skewness[abs(skewness['Skew'])> 0.7]
skewness
skewed_vars = skewness.index
df_train[skewed_vars] = np.log(df_train[skewed_vars]+0.001)
 df_test[skewed_vars] = np.log(df_test[skewed_vars]+0.001 )
adv_X.drop(['sex_unknown', 'height', 'width'], axis=1, inplace=True)
 adv_X_train, adv_X_test, adv_y_train, adv_y_test = train_test_split(adv_X,
adv_y,
test_size=0.4,
stratify=adv_y,
random_state=42)
 xg_adv.fit(adv_X_train, adv_y_train)
 validation = xg_adv.predict_proba(adv_X_test)[:,1]<drop_column>
df_train=pd.get_dummies(df_train)
 df_test=pd.get_dummies(df_test)
 final_train, final_test = df_train.align(df_test,join='inner',axis=1)
X_train.drop(['sex_unknown', 'width','height'], axis=1, inplace=True)
 test.drop(['width','height'], axis=1, inplace=True )<choose_model_class>
GBR_model.fit(final_train,df_train.SalePrice)
 predictions= GBR_model.predict(final_test )
xg= xgb.XGBClassifier(
 n_estimators=750,
learning_rate=0.015,
min_child_weight= 59,
max_delta_step= 6,
max_depth= 6,
subsample= 0.751,
colsample_bytree= 0.8595,
gamma= 0,
reg_lambda= 37,
random_state=42,
n_jobs=-1,
)<save_to_csv>
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.neural_network import MLPRegressor
from sklearn.linear_model import Ridge
from sklearn.linear_model import LinearRegression
from mlxtend.regressor import StackingRegressor
from sklearn.linear_model import Lasso
from sklearn.metrics import mean_squared_error
from math import sqrt
xg.fit(X_train, y_train)
 predictions = xg.predict_proba(test)[:, 1]
meta_df = pd.DataFrame(columns=['image_name', 'target'])
 meta_df['image_name'] = sample['image_name']
meta_df['target'] = predictions
meta_df.to_csv('external_tabular_predicts.csv', header=True, index=False )<save_to_csv>
lr = LinearRegression()
rd = Ridge()
rf = RandomForestRegressor(
 n_estimators = 12,
max_depth = 3,
n_jobs = -1)
 gb = GradientBoostingRegressor()
lasso=Lasso(alpha =0.0005, random_state=1)
effnet = pd.read_csv('.. /input/melanoma-effnet-ensemble/effnet_ensemble.csv')
 meta = pd.read_csv('./external_tabular_predicts.csv')
 sample['target'] =(
 effnet['target'] * 0.9 +
meta['target'] * 0.1 
)
 sample.to_csv('external_meta_ensembled.csv', header=True, index=False )<import_modules>
model = StackingRegressor(
 regressors=[rf, gb, rd, lasso],
meta_regressor=lr)
 model.fit(final_train, df_train.SalePrice);
!pip install -q efficientnet 
<set_options>
y_pred = model.predict(final_train)
 print(sqrt(mean_squared_error(df_train.SalePrice, y_pred)) )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<load_from_csv>
y_pred = model.predict(final_test )
AUTO = tf.data.experimental.AUTOTUNE
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-384x384')
 EPOCHS = 40
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
AUG_BATCH = BATCH_SIZE
IMAGE_SIZE = [384, 384]
SEED = 123
LR = 0.0003
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')
 SUB = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv' )<normalization>
lasso.fit(final_train, df_train.SalePrice)
 score = cross_val_score(lasso,final_train, df_train.SalePrice, scoring= 'neg_mean_absolute_error')
 print('Mean Absolute Error %2f' %(-1 * score.mean()))
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))
def transform(image, label):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
if 0.5 > tf.random.uniform([1], minval = 0, maxval = 1):
rot = 15.* tf.random.normal([1],dtype='float32')
 else:
rot = 180.* tf.random.normal([1],dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image['inp1'],tf.transpose(idx3))
return {'inp1': tf.reshape(d,[DIM,DIM,3]), 'inp2': image['inp2']}, label
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
"age_approx": tf.io.FixedLenFeature([], tf.int64),
"sex": tf.io.FixedLenFeature([], tf.int64),
"anatom_site_general_challenge": tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['target'], tf.int32)
 data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, label, data
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
"age_approx": tf.io.FixedLenFeature([], tf.int64),
"sex": tf.io.FixedLenFeature([], tf.int64),
"anatom_site_general_challenge": tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 image_name = example['image_name']
data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, image_name, data
def load_dataset(filenames, labeled = True, ordered = False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO)
 return dataset
def setup_input1(image, label, data):
anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'inp1': image, 'inp2': tabular}, label
def setup_input2(image, image_name, data):
anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'inp1': image, 'inp2': tabular}, image_name
def setup_input3(image, image_name, target, data):
anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'inp1': image, 'inp2': tabular}, image_name, target
def data_augment(data, label):
data['inp1'] = tf.image.random_flip_left_right(data['inp1'])
 data['inp1'] = tf.image.random_flip_up_down(data['inp1'])
 data['inp1'] = tf.image.random_hue(data['inp1'], 0.01)
 data['inp1'] = tf.image.random_saturation(data['inp1'], 0.7, 1.3)
 data['inp1'] = tf.image.random_contrast(data['inp1'], 0.8, 1.2)
 data['inp1'] = tf.image.random_brightness(data['inp1'], 0.1)
 return data, label
def get_training_dataset(filenames, labeled = True, ordered = False):
dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(setup_input1, num_parallel_calls = AUTO)
 dataset = dataset.map(data_augment, num_parallel_calls = AUTO)
 dataset = dataset.map(transform, num_parallel_calls = AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(filenames, labeled = True, ordered = True):
dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(setup_input1, num_parallel_calls = AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(filenames, labeled = False, ordered = True):
dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(setup_input2, num_parallel_calls = AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 def read_tfrecord_full(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
"age_approx": tf.io.FixedLenFeature([], tf.int64),
"sex": tf.io.FixedLenFeature([], tf.int64),
"anatom_site_general_challenge": tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 image_name = example['image_name']
target = tf.cast(example['target'], tf.int32)
 data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, image_name, target, data
def load_dataset_full(filenames):
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.map(read_tfrecord_full, num_parallel_calls = AUTO)
 return dataset
def get_data_full(filenames):
dataset = load_dataset_full(filenames)
 dataset = dataset.map(setup_input3, num_parallel_calls = AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
NUM_TRAINING_IMAGES = int(count_data_items(TRAINING_FILENAMES)* 0.8)
 NUM_VALIDATION_IMAGES = int(count_data_items(TRAINING_FILENAMES)* 0.2)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))<compute_train_metric>
submission = pd.DataFrame({'Id': df_test.Id, 'SalePrice': np.exp(predictions)})
 submission.to_csv('submissionGB.csv', index=False)
 submission.head()
def binary_focal_loss(gamma=2., alpha=.25):

 def binary_focal_loss_fixed(y_true, y_pred):

 pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
epsilon = K.epsilon()
pt_1 = K.clip(pt_1, epsilon, 1.- epsilon)
 pt_0 = K.clip(pt_0, epsilon, 1.- epsilon)
 return -K.sum(alpha * K.pow(1.- pt_1, gamma)* K.log(pt_1)) \
-K.sum(( 1 - alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0))
return binary_focal_loss_fixed
def get_model() :
with strategy.scope() :
inp1 = tf.keras.layers.Input(shape =(*IMAGE_SIZE, 3), name = 'inp1')
 inp2 = tf.keras.layers.Input(shape =(9), name = 'inp2')
 efnetb3 = efn.EfficientNetB3(weights = 'imagenet', include_top = False)
 x = efnetb3(inp1)
 x = tf.keras.layers.GlobalAveragePooling2D()(x)
 x1 = tf.keras.layers.Dense(64 )(inp2)
 x1 = tf.keras.layers.BatchNormalization()(x1)
 x1 = tf.keras.layers.Activation('relu' )(x1)
 concat = tf.keras.layers.concatenate([x, x1])
 concat = tf.keras.layers.Dense(512, activation = 'relu' )(concat)
 concat = tf.keras.layers.BatchNormalization()(concat)
 concat = tf.keras.layers.Dropout(0.2 )(concat)
 concat = tf.keras.layers.Dense(256, activation = 'relu' )(concat)
 concat = tf.keras.layers.BatchNormalization()(concat)
 concat = tf.keras.layers.Dropout(0.2 )(concat)
 output = tf.keras.layers.Dense(1, activation = 'sigmoid' )(concat)
 model = tf.keras.models.Model(inputs = [inp1, inp2], outputs = [output])
 opt = tf.keras.optimizers.Adam(learning_rate = LR)
 model.compile(
 optimizer = opt,
loss = [binary_focal_loss(gamma = 2.0, alpha = 0.80)],
metrics = [tf.keras.metrics.BinaryAccuracy() , tf.keras.metrics.AUC() ] 
)
 return model
def train_and_predict(SUB, folds = 5):
models = []
oof_image_name = []
oof_target = []
oof_prediction = []
seed_everything(SEED)
 kfold = KFold(folds, shuffle = True, random_state = SEED)
 for fold,(trn_ind, val_ind)in enumerate(kfold.split(TRAINING_FILENAMES)) :
print('
')
 print('-'*50)
 print(f'Training fold {fold + 1}')
 train_dataset = get_training_dataset([TRAINING_FILENAMES[x] for x in trn_ind], labeled = True, ordered = False)
 val_dataset = get_validation_dataset([TRAINING_FILENAMES[x] for x in val_ind], labeled = True, ordered = True)
 K.clear_session()
model = get_model()
early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_auc', mode = 'max', patience = 5,
verbose = 1, min_delta = 0.0001, restore_best_weights = True)
 cb_lr_schedule = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_auc', factor = 0.5, patience = 2, verbose = 1, min_delta = 0.0001, mode = 'max')
 history = model.fit(train_dataset,
steps_per_epoch = STEPS_PER_EPOCH,
epochs = EPOCHS,
callbacks = [early_stopping, cb_lr_schedule],
validation_data = val_dataset,
verbose = 2)
 models.append(model)
 number_of_files = count_data_items([TRAINING_FILENAMES[x] for x in val_ind])
 dataset = get_data_full([TRAINING_FILENAMES[x] for x in val_ind])
 image_name = dataset.map(lambda image, image_name, target: image_name ).unbatch()
image_name = next(iter(image_name.batch(number_of_files)) ).numpy().astype('U')
 target = dataset.map(lambda image, image_name, target: target ).unbatch()
target = next(iter(target.batch(number_of_files)) ).numpy()
image = dataset.map(lambda image, image_name, target: image)
 probabilities = model.predict(image)
 oof_image_name.extend(list(image_name))
oof_target.extend(list(target))
oof_prediction.extend(list(np.concatenate(probabilities)))
 print('
')
 print('-'*50)
 oof_df = pd.DataFrame({'image_name': oof_image_name, 'target': oof_target, 'predictions': oof_prediction})
 oof_df.to_csv('EfficientNetB3_384.csv', index = False)
 test_ds = get_test_dataset(TEST_FILENAMES, labeled = False, ordered = True)
 test_images_ds = test_ds.map(lambda image, image_name: image)
 print('Computing predictions...')
 probabilities = np.average([np.concatenate(models[i].predict(test_images_ds)) for i in range(folds)], axis = 0)
 print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, image_name: image_name ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 pred_df = pd.DataFrame({'image_name': test_ids, 'target': probabilities})
 SUB.drop('target', inplace = True, axis = 1)
 SUB = SUB.merge(pred_df, on = 'image_name')
 SUB.to_csv('sub_EfficientNetB3_384.csv', index = False)
 return oof_target, oof_prediction
oof_target, oof_prediction = train_and_predict(SUB )<compute_test_metric>
submission = pd.DataFrame({'Id': df_test.Id, 'SalePrice': np.exp(y_pred)})
 submission.to_csv('submissionEM.csv',index=False)
 submission.head()