Kaggle competition: House Prices - Advanced Regression Techniques
model = tta_segmentation(model, h_flip=True, v_flip=True, h_shift=(-10, 10), v_shift=(-10, 10), merge='mean' )<define_variables>
import numpy as np
import pandas as pd
import copy
from functools import partial
from pandas import DataFrame
import xgboost as xgb
import lightgbm as lgb
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Activation, BatchNormalization, Dropout
from tensorflow.keras.optimizers import Adam
from sklearn.metrics import mean_squared_log_error
import optuna
test_df = []
for i in range(0, test.shape[0], 300):
batch_idx = list(range(i, min(test.shape[0], i + 300)))
 batch_set = test[batch_idx[0]: batch_idx[-1]+1]
test_generator = DataGenerator(
 directory=test_images_dest_path,
dataframe=batch_set,
target_df=submission,
batch_size=1,
target_size=(HEIGHT, WIDTH),
n_channels=CHANNELS,
n_classes=N_CLASSES,
preprocessing=preprocessing,
seed=seed,
mode='predict',
shuffle=False)
 preds = model.predict_generator(test_generator)
 for index, b in enumerate(batch_idx):
filename = test['image'].iloc[b]
image_df = submission[submission['image'] == filename].copy()
pred_masks = preds[index, ].round().astype(int)
 pred_rles = build_rles(pred_masks, reshape=(350, 525))
image_df['EncodedPixels'] = pred_rles
pred_masks_post = preds[index, ].astype('float32')
 for class_index in range(N_CLASSES):
pred_mask = pred_masks_post[...,class_index]
pred_mask = post_process(pred_mask, threshold=best_tresholds[class_index], min_size=best_masks[class_index])
 pred_masks_post[...,class_index] = pred_mask
pred_rles_post = build_rles(pred_masks_post, reshape=(350, 525))
image_df['EncodedPixels_post'] = pred_rles_post
test_df.append(image_df)
 sub_df = pd.concat(test_df )<save_to_csv>
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 train.shape, test.shape
submission_df = sub_df[['Image_Label' ,'EncodedPixels']]
submission_df.to_csv('submission.csv', index=False)
 display(submission_df.head() )<save_to_csv>
X_train = np.array(pd.read_csv('.. /input/houseprice-ppdata/train_x.csv'))
X_train = X_train[:, 1:]
y_train = np.array(pd.read_csv('.. /input/houseprice-ppdata/train_y.csv'))
y_train = y_train[:, 1]
XX_TEST = np.array(pd.read_csv('.. /input/houseprice-ppdata/test_x.csv'))
XX_TEST = XX_TEST[:, 1:]
std1 = X_train.std(axis=0)
 X_train = X_train[:, std1 != 0]
XX_TEST = XX_TEST[:, std1 != 0]
mean1 = X_train.mean(axis=0)
 std1 = X_train.std(axis=0)
 print(X_train.shape, XX_TEST.shape )
submission_df_post = sub_df[['Image_Label' ,'EncodedPixels_post']]
submission_df_post.columns = ['Image_Label' ,'EncodedPixels']
submission_df_post.to_csv('submission_post.csv', index=False)
 display(submission_df_post.head() )<load_pretrained>
X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, train_size= 0.90, random_state=2020)
 print(X_train.shape, y_train.shape)
 print(X_test.shape, y_test.shape )
if os.path.exists(train_images_dest_path):
shutil.rmtree(train_images_dest_path)
 if os.path.exists(validation_images_dest_path):
shutil.rmtree(validation_images_dest_path)
 if os.path.exists(test_images_dest_path):
shutil.rmtree(test_images_dest_path )<set_options>
def objective_xgb(X_train, X_test, y_train, y_test, trial):
params = {
'objective': 'reg:squarederror',
'max_depth': trial.suggest_int('max_depth', 1, 9),
'n_estimators': trial.suggest_int('n_estimators', 4000, 7000),
'learning_rate': trial.suggest_loguniform('learning_rate', 1e-3, 1.0),
'gamma': trial.suggest_uniform('gamma', 0.0, 1.0),
'min_child_weight': trial.suggest_uniform('min_child_weight', 0.0, 1.0),
'subsample': trial.suggest_uniform('subsample', 0.0, 1.0),
'colsample_bytree': trial.suggest_uniform('colsample_bytree', 0.0, 1.0)
 }
model = xgb.XGBRegressor(**params)
 model.fit(X_train, y_train)
 pred = model.predict(X_test)
 mse = mean_squared_error(y_test, pred)
 return mse
seed(10)
 set_random_seed(10)
 %matplotlib inline<define_variables>
mod_xgb_opt = xgb.XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.7245610853936947,
gamma=0.005388388541926817, gpu_id=-1, importance_type='gain',
interaction_constraints='', learning_rate=0.004021246542657597,
max_delta_step=0, max_depth=3, min_child_weight=0.5256214626878104,
monotone_constraints='() ', n_estimators=5880,
n_jobs=0, num_parallel_tree=1, random_state=0, reg_alpha=0,
reg_lambda=1, scale_pos_weight=1, subsample=0.793174165882878,
tree_method='exact', validate_parameters=1, verbosity=None)
 mod_xgb_opt.fit(X_train, y_train)
test_imgs_folder = '.. /input/understanding_cloud_organization/test_images/'
train_imgs_folder = '.. /input/understanding_cloud_organization/train_images/'
num_cores = multiprocessing.cpu_count()<load_from_csv>
y_train_pred_xgb_opt = mod_xgb_opt.predict(X_train)
 y_test_pred_xgb_opt = mod_xgb_opt.predict(X_test)
 print('RMSLE train : %.6f, test : %.6f' %(np.sqrt(mean_squared_log_error(np.exp(y_train)-1, np.exp(y_train_pred_xgb_opt)-1)) , np.sqrt(mean_squared_log_error(np.exp(y_test)-1, np.exp(y_test_pred_xgb_opt)-1))))
train_df = pd.read_csv('.. /input/understanding_cloud_organization/train.csv')
 train_df.head()<feature_engineering>
def objective_lgbm(X_train, X_test, y_train, y_test, trial):
params = {
'objective':'regression',
'metric':'mse',
'num_leaves': trial.suggest_int('num_leaves', 2, 64),
'max_bin': trial.suggest_int('max_bin', 128, 512),
'n_estimators': trial.suggest_int('n_estimators', 4000, 7000),
'learning_rate': trial.suggest_loguniform('learning_rate', 1e-3, 1.0),
'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.0, 1.0),
'bagging_freq': trial.suggest_int('bagging_freq', 0, 10),
'feature_fraction': trial.suggest_uniform('feature_fraction', 0.0, 1.0),
'min_sum_hessian_in_leaf': trial.suggest_loguniform('min_sum_hessian_in_leaf', 1e-3, 10.0)
 }
model = lgb.LGBMRegressor(**params)
 model.fit(X_train, y_train)
 pred = model.predict(X_test)
 mse = mean_squared_error(y_test, pred)
 return mse
train_df = train_df[~train_df['EncodedPixels'].isnull() ]
train_df['Image'] = train_df['Image_Label'].map(lambda x: x.split('_')[0])
 train_df['Class'] = train_df['Image_Label'].map(lambda x: x.split('_')[1])
 classes = train_df['Class'].unique()
train_df = train_df.groupby('Image')['Class'].agg(set ).reset_index()
for class_name in classes:
train_df[class_name] = train_df['Class'].map(lambda x: 1 if class_name in x else 0)
 train_df.head()<define_variables>
mod_lgb_opt = lgb.LGBMRegressor(bagging_fraction=0.9724444503168079, bagging_freq=2,
feature_fraction=0.572706177440646,
learning_rate=0.007117847154213432, max_bin=199,
min_sum_hessian_in_leaf=4.837478916656753, n_estimators=6157,
num_leaves=5)
 mod_lgb_opt.fit(X_train, y_train)
img_2_ohe_vector = {img:vec for img, vec in zip(train_df['Image'], train_df.iloc[:, 2:].values)}<split>
y_train_pred_lgbm_opt = mod_lgb_opt.predict(X_train)
 y_test_pred_lgbm_opt = mod_lgb_opt.predict(X_test)
 print('RMSLE train : %.6f, test : %.6f' %(np.sqrt(mean_squared_log_error(np.exp(y_train)-1, np.exp(y_train_pred_lgbm_opt)-1)) , np.sqrt(mean_squared_log_error(np.exp(y_test)-1, np.exp(y_test_pred_lgbm_opt)-1))))
train_imgs, val_imgs = train_test_split(train_df['Image'].values,
test_size=0.2,
stratify=train_df['Class'].map(lambda x: str(sorted(list(x)))) ,
random_state=10 )<define_variables>
X_train_nn = copy.deepcopy(X_train)
 X_test_nn = copy.deepcopy(X_test)
 y_train_nn = copy.deepcopy(y_train)
 y_test_nn = copy.deepcopy(y_test)
 X_train_nn =(X_train_nn - mean1)/ std1
X_test_nn =(X_test_nn - mean1)/ std1
mean2 = y_train_nn.mean(axis=0)
 std2 = y_train_nn.std(axis=0)
 y_train_nn =(y_train_nn - mean2)/ std2
y_test_nn =(y_test_nn - mean2)/ std2
print(np.any(std1 == 0))
print(np.any(std2 == 0))
class DataGenenerator(Sequence):
def __init__(self, images_list=None, folder_imgs=train_imgs_folder,
batch_size=32, shuffle=True, augmentation=None,
resized_height=224, resized_width=224, num_channels=3):
self.batch_size = batch_size
self.shuffle = shuffle
self.augmentation = augmentation
if images_list is None:
self.images_list = os.listdir(folder_imgs)
 else:
self.images_list = deepcopy(images_list)
 self.folder_imgs = folder_imgs
self.len = len(self.images_list)// self.batch_size
self.resized_height = resized_height
self.resized_width = resized_width
self.num_channels = num_channels
self.num_classes = 4
self.is_test = not 'train' in folder_imgs
if not shuffle and not self.is_test:
self.labels = [img_2_ohe_vector[img] for img in self.images_list[:self.len*self.batch_size]]
def __len__(self):
return self.len
def on_epoch_start(self):
if self.shuffle:
random.shuffle(self.images_list)
 def __getitem__(self, idx):
current_batch = self.images_list[idx * self.batch_size:(idx + 1)* self.batch_size]
X = np.empty(( self.batch_size, self.resized_height, self.resized_width, self.num_channels))
y = np.empty(( self.batch_size, self.num_classes))
for i, image_name in enumerate(current_batch):
path = os.path.join(self.folder_imgs, image_name)
 img = cv2.resize(cv2.imread(path),(self.resized_height, self.resized_width)).astype(np.float32)
 if not self.augmentation is None:
augmented = self.augmentation(image=img)
 img = augmented['image']
X[i, :, :, :] = img/255.0
if not self.is_test:
y[i, :] = img_2_ohe_vector[image_name]
return X, y
def get_labels(self):
if self.shuffle:
images_current = self.images_list[:self.len*self.batch_size]
labels = [img_2_ohe_vector[img] for img in images_current]
else:
labels = self.labels
return np.array(labels )<concatenate>
np.random.seed(seed=0)
 os.environ['PYTHONHASHSEED'] = '0'
tf.random.set_seed(0 )
albumentations_train = Compose([
VerticalFlip() , HorizontalFlip() , Rotate(limit=30), GridDistortion()
], p=1 )<choose_model_class>
model_nn = Sequential()
model_nn.add(Dense(256, activation='relu', input_shape=(X_train_nn.shape[1],)))
 model_nn.add(Dense(512))
model_nn.add(Activation('relu'))
model_nn.add(Dense(512, activation='relu'))
model_nn.add(Dense(512))
model_nn.add(Activation('relu'))
model_nn.add(Dense(512, activation='relu'))
model_nn.add(Dense(1))
optimizer = Adam(lr=1e-3)
 model_nn.compile(loss="mean_squared_error", optimizer=optimizer)
 model_nn.summary()
data_generator_train = DataGenenerator(train_imgs, augmentation=albumentations_train)
 data_generator_train_eval = DataGenenerator(train_imgs, shuffle=False)
 data_generator_val = DataGenenerator(val_imgs, shuffle=False )<load_pretrained>
model_nn.fit(X_train_nn, y_train_nn, epochs=100, batch_size=16 )
class PrAucCallback(Callback):
def __init__(self, data_generator, num_workers=num_cores,
early_stopping_patience=5,
plateau_patience=3, reduction_rate=0.5,
stage='train', checkpoints_path='checkpoints/'):
super(Callback, self ).__init__()
self.data_generator = data_generator
self.num_workers = num_workers
self.class_names = ['Fish', 'Flower', 'Sugar', 'Gravel']
self.history = [[] for _ in range(len(self.class_names)+ 1)]
self.early_stopping_patience = early_stopping_patience
self.plateau_patience = plateau_patience
self.reduction_rate = reduction_rate
self.stage = stage
self.best_pr_auc = -float('inf')
 if not os.path.exists(checkpoints_path):
os.makedirs(checkpoints_path)
 self.checkpoints_path = checkpoints_path
def compute_pr_auc(self, y_true, y_pred):
pr_auc_mean = 0
print(f"
{'
")
 for class_i in range(len(self.class_names)) :
precision, recall, _ = precision_recall_curve(y_true[:, class_i], y_pred[:, class_i])
 pr_auc = auc(recall, precision)
 pr_auc_mean += pr_auc/len(self.class_names)
 print(f"PR AUC {self.class_names[class_i]}, {self.stage}: {pr_auc:.3f}
")
 self.history[class_i].append(pr_auc)
 print(f"
{'
PR AUC mean, {self.stage}: {pr_auc_mean:.3f}
{'
")
 self.history[-1].append(pr_auc_mean)
 return pr_auc_mean
def is_patience_lost(self, patience):
if len(self.history[-1])> patience:
best_performance = max(self.history[-1][-(patience + 1):-1])
 return best_performance == self.history[-1][-(patience + 1)] and best_performance >= self.history[-1][-1]
def early_stopping_check(self, pr_auc_mean):
if self.is_patience_lost(self.early_stopping_patience):
self.model.stop_training = True
def model_checkpoint(self, pr_auc_mean, epoch):
if pr_auc_mean > self.best_pr_auc:
for checkpoint in glob.glob(os.path.join(self.checkpoints_path, 'classifier_densenet169_epoch_*')) :
os.remove(checkpoint)
 self.best_pr_auc = pr_auc_mean
self.model.save(os.path.join(self.checkpoints_path, f'classifier_densenet169_epoch_{epoch}_val_pr_auc_{pr_auc_mean}.h5'))
print(f"
{'
Saved new checkpoint
{'
")
 def reduce_lr_on_plateau(self):
if self.is_patience_lost(self.plateau_patience):
new_lr = float(keras.backend.get_value(self.model.optimizer.lr)) * self.reduction_rate
keras.backend.set_value(self.model.optimizer.lr, new_lr)
 print(f"
{'
Reduced learning rate to {new_lr}.
 {'
")
 def on_epoch_end(self, epoch, logs={}):
y_pred = self.model.predict_generator(self.data_generator, workers=self.num_workers)
 y_true = self.data_generator.get_labels()
pr_auc_mean = self.compute_pr_auc(y_true, y_pred)
 if self.stage == 'val':
self.early_stopping_check(pr_auc_mean)
 self.model_checkpoint(pr_auc_mean, epoch)
 self.reduce_lr_on_plateau()
def get_pr_auc_history(self):
return self.history<groupby>
y_train_pred_nn = np.squeeze(model_nn.predict(X_train_nn)* std2 + mean2)
 y_test_pred_nn = np.squeeze(model_nn.predict(X_test_nn)* std2 + mean2)
 print(np.any(np.isnan(y_test_pred_nn)== True))
print('RMSLE train : %.6f, test : %.6f' %(np.sqrt(mean_squared_log_error(np.exp(y_train)-1, np.exp(y_train_pred_nn)-1)) , np.sqrt(mean_squared_log_error(np.exp(y_test)-1, np.exp(y_test_pred_nn)-1))))
train_metric_callback = PrAucCallback(data_generator_train_eval)
 val_callback = PrAucCallback(data_generator_val, stage='val' )<choose_model_class>
regLasso = Lasso(alpha=0.0007)
 regLasso.fit(X_train, y_train )
def get_model() :
K.clear_session()
base_model = DenseNet169(weights='imagenet', include_top=False, pooling='avg', input_shape=(224, 224, 3))
x = base_model.output
y_pred = Dense(4, activation='sigmoid' )(x)
 return Model(inputs=base_model.input, outputs=y_pred)
 model = get_model()<train_model>
y_train_pred_lasso = regLasso.predict(X_train)
 y_test_pred_lasso = regLasso.predict(X_test)
 print('RMSLE train : %.6f, test : %.6f' %(np.sqrt(mean_squared_log_error(np.exp(y_train)-1, np.exp(y_train_pred_lasso)-1)) , np.sqrt(mean_squared_log_error(np.exp(y_test)-1, np.exp(y_test_pred_lasso)-1))))
for base_layer in model.layers[:-1]:
base_layer.trainable = False
model.compile(optimizer=Adam(lr=1e-4), loss='binary_crossentropy')
 history_0 = model.fit_generator(generator=data_generator_train,
validation_data=data_generator_val,
epochs=1,
callbacks=[train_metric_callback, val_callback],
workers=num_cores,
verbose=1
)<train_model>
def blended_predictions(X, X_nn):
return(2.27101366e-02 * mod_xgb_opt.predict(X)) +(5.76237370e-01 * mod_lgb_opt.predict(X)) \
+(1.90522937e-19 * np.squeeze(model_nn.predict(X_nn)* std2 + mean2)) +(4.02316541e-01 * regLasso.predict(X))
def blended_predictions2(X, X_nn):
return(0.678351173 * mod_lgb_opt.predict(X)) +(0.322875938 * regLasso.predict(X))
def blended_predictions_tune111(X, X_nn, r1, r2, r3):
return(r1 * mod_xgb_opt.predict(X)) +(r2 * mod_lgb_opt.predict(X)) +(r3 * np.squeeze(model_nn.predict(X_nn)* std2 + mean2))
for base_layer in model.layers[:-1]:
base_layer.trainable = True
model.compile(optimizer=Adam(lr=1e-5), loss='binary_crossentropy')
 history_1 = model.fit_generator(generator=data_generator_train,
validation_data=data_generator_val,
epochs=2,
callbacks=[train_metric_callback, val_callback],
workers=num_cores,
verbose=1,
initial_epoch=1
)<load_pretrained>
model = load_model('.. /input/clouds-classifier-files/classifier_densenet169_epoch_21_val_pr_auc_0.8365921057512743.h5' )<define_variables>
y_train_pred_blended = blended_predictions(X_train, X_train_nn)
 y_test_pred_blended = blended_predictions(X_test, X_test_nn)
 print('RMSLE train : %.6f, test : %.6f' %(np.sqrt(mean_squared_log_error(np.exp(y_train)-1, np.exp(y_train_pred_blended)-1)) , np.sqrt(mean_squared_log_error(np.exp(y_test)-1, np.exp(y_test_pred_blended)-1))))
Image(".. /input/clouds-classifier-files/loss_hist_densenet169.png" )<define_variables>
XX_TEST_nn = copy.deepcopy(XX_TEST)
 XX_TEST_nn =(XX_TEST_nn - mean1)/ std1
Image(".. /input/clouds-classifier-files/pr_auc_hist_densenet169.png" )<find_best_params>
TEST_pred = blended_predictions(XX_TEST, XX_TEST_nn)
 print(TEST_pred )
class_names = ['Fish', 'Flower', 'Sugar', 'Gravel']
def get_threshold_for_recall(y_true, y_pred, class_i, recall_threshold=0.95, precision_threshold=0.94, plot=False):
precision, recall, thresholds = precision_recall_curve(y_true[:, class_i], y_pred[:, class_i])
 i = len(thresholds)- 1
best_recall_threshold = None
while best_recall_threshold is None:
next_threshold = thresholds[i]
next_recall = recall[i]
if next_recall >= recall_threshold:
best_recall_threshold = next_threshold
i -= 1
best_precision_threshold = [thres for prec, thres in zip(precision, thresholds)if prec >= precision_threshold][0]
if plot:
plt.figure(figsize=(10, 7))
plt.step(recall, precision, color='r', alpha=0.3, where='post')
 plt.fill_between(recall, precision, alpha=0.3, color='r')
 plt.axhline(y=precision[i + 1])
 recall_for_prec_thres = [rec for rec, thres in zip(recall, thresholds)
 if thres == best_precision_threshold][0]
plt.axvline(x=recall_for_prec_thres, color='g')
 plt.xlabel('Recall')
 plt.ylabel('Precision')
 plt.ylim([0.0, 1.05])
 plt.xlim([0.0, 1.0])
 plt.legend(['PR curve',
f'Precision {precision[i + 1]:.2f} corresponding to selected recall threshold',
f'Recall {recall_for_prec_thres:.2f} corresponding to selected precision threshold'])
 plt.title(f'Precision-Recall curve for Class {class_names[class_i]}')
 return best_recall_threshold, best_precision_threshold
y_pred = model.predict_generator(data_generator_val, workers=num_cores)
 y_true = data_generator_val.get_labels()
recall_thresholds = dict()
precision_thresholds = dict()
for i, class_name in tqdm(enumerate(class_names)) :
recall_thresholds[class_name], precision_thresholds[class_name] = get_threshold_for_recall(y_true, y_pred, i, plot=True )<predict_on_test>
TEST_pred_raw = np.exp(TEST_pred)-1
print(TEST_pred_raw )
data_generator_test = DataGenenerator(folder_imgs=test_imgs_folder, shuffle=False)
 y_pred_test = model.predict_generator(data_generator_test, workers=num_cores )<define_variables>
TEST_pred_raw2 = np.exp(TEST_pred_2)-1
print(TEST_pred_raw2 )
image_labels_empty = set()
for i,(img, predictions)in enumerate(zip(os.listdir(test_imgs_folder), y_pred_test)) :
for class_i, class_name in enumerate(class_names):
if predictions[class_i] < recall_thresholds[class_name]:
image_labels_empty.add(f'{img}_{class_name}' )<load_from_csv>
submission= pd.DataFrame()
submission['id'] = test['Id']
submission['SalePrice'] = TEST_pred_raw
submission2= pd.DataFrame()
submission2['id'] = test['Id']
submission2['SalePrice'] = TEST_pred_raw2
submission = pd.read_csv('.. /input/efficient-net-b4-unet-clouds/submission.csv')
 submission.head()<filter>
submission.to_csv('submission.csv', index=False )
predictions_nonempty = set(submission.loc[~submission['EncodedPixels'].isnull() , 'Image_Label'].values )<concatenate>
submission.to_csv('submission.csv', index=False )
print(f'{len(image_labels_empty.intersection(predictions_nonempty)) } masks would be removed' )<save_to_csv>
submission2.to_csv('submission2.csv', index=False )