Kaggle competition: House Prices - Advanced Regression Techniques
def create_DenseNet_model() :
pretrained_model = tf.keras.applications.DenseNet201(weights = 'imagenet', include_top = False, input_shape = [*IMAGE_SIZE, 3])
 pretrained_model.trainable = True
model = tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation = 'softmax')
 ])
 model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['sparse_categorical_accuracy'])
 return model<choose_model_class>
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numeric = []
for i in all_features.columns:
if all_features[i].dtype in numeric_dtypes:
numeric.append(i)
 skew_features = all_features[numeric].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skew = skew_features[skew_features > 0.5]
skew_index = high_skew.index
print("There are {} numerical features with Skew > 0.5 :".format(high_skew.shape[0]))
skewness = pd.DataFrame({'Skew' :high_skew})
 skew_features.head(5)
 for i in skew_index:
all_features[i] = boxcox1p(all_features[i], boxcox_normmax(all_features[i] + 1))
print("temporal_features:",temporal_features)
 def create_temporal_features(features):
df = features[temporal_features].copy()
for feature in temporal_features:
if feature == 'YrSold' or feature == 'MoSold':
pass
else:
df[feature + "_diff"] = df['YrSold'] - df[feature]
df[feature + "_diff"] = df[feature + "_diff"].fillna(df[feature + "_diff"].max())
 df['YrBltAndRemod'] = df['YearBuilt']+df['YearRemodAdd']
for feature in temporal_features:
df[feature] = df[feature].astype(str)
 df[feature] = df[feature].fillna("Missing")
 df = df.drop(overfit_reducer(df), axis=1)
 return df
def create_categorical_features(features):
df = features[categorical_features].copy()
df['Functional'] = df['Functional'].fillna('Typ')
 df['Utilities'] = df['Utilities'].fillna('AllPub')
 df['Electrical'] = df['Electrical'].fillna("SBrkr")
 df['KitchenQual'] = df['KitchenQual'].fillna("TA")
 df['Exterior1st'] = df['Exterior1st'].fillna(df['Exterior1st'].mode() [0])
 df['Exterior2nd'] = df['Exterior2nd'].fillna(df['Exterior2nd'].mode() [0])
 df['SaleType'] = df['SaleType'].fillna(df['SaleType'].mode() [0])
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
df[col] = df[col].fillna('None')
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
features[col] = features[col].fillna('None')
 features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
for feature in categorical_features:
df[feature] = df[feature].fillna("Missing")
 df = pd.get_dummies(df)
 df = df.drop(overfit_reducer(df), axis=1)
 return df
def create_numeric_features(features):
df = features[numeric_features + ["Neighborhood"]].copy()
for col in('GarageArea', 'GarageCars'):
df[col] = df[col].fillna(0)
 for feature in numeric_features:
df[feature] = df.groupby("Neighborhood")[feature].transform(lambda x: x.fillna(x.median()))
df = df.drop("Neighborhood", axis=1)
 df['TotalHouseSF'] = df['TotalBsmtSF'] + df['1stFlrSF'] + df['2ndFlrSF']
df['TotalArea'] = df['TotalBsmtSF'] + df['1stFlrSF'] + df['2ndFlrSF'] + df['GrLivArea'] + df['GarageArea']
df['Total_Bathrooms'] =(df['FullBath'] +(0.5 * df['HalfBath'])+
df['BsmtFullBath'] +(0.5 * df['BsmtHalfBath']))
df['TotalBath'] = df['FullBath'] + 0.5 * df['HalfBath']
df['TotalLot'] = df['LotFrontage'] + df['LotArea']
df['TotalPorch'] = df['OpenPorchSF'] + df['EnclosedPorch'] + df['ScreenPorch']
df['Total_sqr_footage'] =(df['BsmtFinSF1'] + df['BsmtFinSF2'] +
df['1stFlrSF'] + df['2ndFlrSF'])
 df['TotalBsmtFin'] = df['BsmtFinSF1'] + df['BsmtFinSF2']
df['Total_porch_sf'] =(df['OpenPorchSF'] + df['3SsnPorch'] +
df['EnclosedPorch'] + df['ScreenPorch'] +
df['WoodDeckSF'])
 df['haspool'] = df['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 df['has2ndfloor'] = df['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 df['hasgarage'] = df['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 df['hasbsmt'] = df['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 df['hasfireplace'] = df['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 df = df.drop(overfit_reducer(df), axis=1)
 df = fixing_skewness(df)
 return df
all_features['BsmtFinType1_Unf'] = 1*(all_features['BsmtFinType1'] == 'Unf')
 all_features['HasWoodDeck'] =(all_features['WoodDeckSF'] == 0)* 1
all_features['HasOpenPorch'] =(all_features['OpenPorchSF'] == 0)* 1
all_features['HasEnclosedPorch'] =(all_features['EnclosedPorch'] == 0)* 1
all_features['Has3SsnPorch'] =(all_features['3SsnPorch'] == 0)* 1
all_features['HasScreenPorch'] =(all_features['ScreenPorch'] == 0)* 1
all_features['Total_Home_Quality'] = all_features['OverallQual'] + all_features['OverallCond']
all_features = all_features.drop(['Utilities', 'Street', 'PoolQC',], axis=1)
 all_features['TotalSF'] = all_features['TotalBsmtSF'] + all_features['1stFlrSF'] + all_features['2ndFlrSF']
all_features['Total_sqr_footage'] =(all_features['BsmtFinSF1'] + all_features['BsmtFinSF2'] +
all_features['1stFlrSF'] + all_features['2ndFlrSF'])
 all_features['Total_Bathrooms'] =(all_features['FullBath'] +(0.5 * all_features['HalfBath'])+
all_features['BsmtFullBath'] +(0.5 * all_features['BsmtHalfBath']))
all_features['Total_porch_sf'] =(all_features['OpenPorchSF'] + all_features['3SsnPorch'] +
all_features['EnclosedPorch'] + all_features['ScreenPorch'] +
all_features['WoodDeckSF'])
 all_features['TotalBsmtSF'] = all_features['TotalBsmtSF'].apply(lambda x: np.exp(6)if x <= 0.0 else x)
 all_features['2ndFlrSF'] = all_features['2ndFlrSF'].apply(lambda x: np.exp(6.5)if x <= 0.0 else x)
 all_features['GarageArea'] = all_features['GarageArea'].apply(lambda x: np.exp(6)if x <= 0.0 else x)
 all_features['GarageCars'] = all_features['GarageCars'].apply(lambda x: 0 if x <= 0.0 else x)
 all_features['LotFrontage'] = all_features['LotFrontage'].apply(lambda x: np.exp(4.2)if x <= 0.0 else x)
 all_features['MasVnrArea'] = all_features['MasVnrArea'].apply(lambda x: np.exp(4)if x <= 0.0 else x)
 all_features['BsmtFinSF1'] = all_features['BsmtFinSF1'].apply(lambda x: np.exp(6.5)if x <= 0.0 else x)
 all_features['haspool'] = all_features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 all_features['has2ndfloor'] = all_features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 all_features['hasgarage'] = all_features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 all_features['hasbsmt'] = all_features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 all_features['hasfireplace'] = all_features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 temporal_df = create_temporal_features(all_features[temporal_features])
 all_features = all_features.drop(['YearBuilt', 'YearRemodAdd', 'GarageYrBlt', 'MoSold', 'YrSold'], axis=1)
 all_features = pd.concat([temporal_df,all_features],axis=1)
 all_features['YearRemodAdd'] = all_features['YearRemodAdd'].astype(int)
 all_features = all_features.drop(overfit_reducer(all_features), axis=1)
 def logs(res, ls):
m = res.shape[1]
for l in ls:
res = res.assign(newcol=pd.Series(np.log(1.01+res[l])).values)
 res.columns.values[m] = l + '_log'
m += 1
return res
log_features = ['LotFrontage','LotArea','MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF',
'TotalBsmtSF','1stFlrSF','2ndFlrSF','LowQualFinSF','GrLivArea',
'BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','BedroomAbvGr','KitchenAbvGr',
'TotRmsAbvGrd','Fireplaces','GarageCars','GarageArea','WoodDeckSF','OpenPorchSF',
'EnclosedPorch','3SsnPorch','ScreenPorch','MiscVal','YearRemodAdd','TotalSF']
all_features = logs(all_features, log_features)
 def squares(res, ls):
m = res.shape[1]
for l in ls:
res = res.assign(newcol=pd.Series(res[l]*res[l] ).values)
 res.columns.values[m] = l + '_sq'
m += 1
return res
squared_features = [ 'LotFrontage_log',
'TotalBsmtSF_log', '1stFlrSF_log', '2ndFlrSF_log', 'GrLivArea_log',
'GarageCars_log', 'GarageArea_log']
all_features = squares(all_features, squared_features)
 all_features = pd.get_dummies(all_features ).reset_index(drop=True)
 all_features = all_features.loc[:,~all_features.columns.duplicated() ]
def create_EfficientNet_model() :
pretrained_model = efficientnet.EfficientNetB7(weights = 'noisy-student', include_top = False, input_shape = [*IMAGE_SIZE, 3])
 pretrained_model.trainable = True
model = tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation = 'softmax')
 ])
 model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['sparse_categorical_accuracy'])
 return model<choose_model_class>
X = all_features.iloc[:len(train_labels), :]
X_test = all_features.iloc[len(train_labels):, :]
kf = KFold(n_splits=12, random_state=42, shuffle=True)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, train_labels, scoring="neg_mean_squared_error", cv=kf))
return(rmse)
def create_InceptionV3_model() :
pretrained_model = tf.keras.applications.InceptionV3(weights = 'imagenet', include_top = False, input_shape = [*IMAGE_SIZE, 3])
 pretrained_model.trainable = True
model = tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation = 'softmax')
 ])
 model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['sparse_categorical_accuracy'])
 return model<choose_model_class>
lightgbm = LGBMRegressor(objective='regression',
num_leaves=6,
learning_rate=0.01,
n_estimators=7000,
max_bin=200,
bagging_fraction=0.8,
bagging_freq=4,
bagging_seed=8,
feature_fraction=0.2,
feature_fraction_seed=8,
min_sum_hessian_in_leaf = 11,
verbose=-1,
random_state=42)
 xgboost = XGBRegressor(learning_rate=0.01,
n_estimators=6000,
max_depth=4,
min_child_weight=0,
gamma=0.6,
subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear',
nthread=-1,
scale_pos_weight=1,
seed=27,
reg_alpha=0.00006,
random_state=42)
 catboost = CatBoostRegressor(learning_rate= 0.01,
iterations= 6000,
eval_metric= "RMSE",
random_seed= 42,
logging_level= "Silent",
loss_function= "RMSE",
od_type= "Iter",
od_wait= 100,
one_hot_max_size= 20,
l2_leaf_reg= 1.4264966279202298,
depth= 5,
rsm= 0.6253447219486281,
random_strength= 0.009926214908500652,
bagging_temperature= 232)
 ridge_alphas = [1e-15, 1e-10, 1e-8, 9e-4, 7e-4, 5e-4, 3e-4, 1e-4, 1e-3, 5e-2, 1e-2, 0.1, 0.3, 1, 3, 5, 10, 15, 18, 20, 30, 50, 75, 100]
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=ridge_alphas, cv=kf))
lasso_alphas = [ 5e-5, 1e-4, 5e-4, 1e-3 ]
lasso = make_pipeline(RobustScaler() , LassoCV(max_iter=1e7, alphas=lasso_alphas, cv=kf, random_state=42))
elasticnet_alphas = [5e-5, 1e-4, 5e-4, 1e-3]
elasticnet_l1ratios = [0.8, 0.85, 0.9, 0.95, 1 ]
elasticnet = make_pipeline(RobustScaler() , ElasticNetCV(max_iter=1e7, alphas=elasticnet_alphas, l1_ratio=elasticnet_l1ratios, cv=kf))
kernelridge_alphas = [ ]
kernelridge = make_pipeline(RobustScaler() , KernelRidge(alpha=9.2304694336833,
kernel="polynomial", degree=3, coef0=7.405346770365897))
svr = make_pipeline(RobustScaler() , SVR(C= 20, epsilon= 0.008, gamma=0.0003))
gbr = GradientBoostingRegressor(n_estimators=6000,
learning_rate=0.01,
max_depth=4,
max_features='sqrt',
min_samples_leaf=15,
min_samples_split=10,
loss='huber',
random_state=42)
 rf = RandomForestRegressor(n_estimators=1200,
max_depth=15,
min_samples_split=5,
min_samples_leaf=5,
max_features=None,
oob_score=True,
random_state=42)
 stack_gen = StackingCVRegressor(regressors=(xgboost, lightgbm, catboost, svr, ridge, lasso, elasticnet, kernelridge, gbr, rf),
meta_regressor=xgboost,
use_features_in_secondary=True )
def create_ResNet152_model() :
pretrained_model = tf.keras.applications.ResNet152V2(weights = 'imagenet', include_top = False, input_shape = [*IMAGE_SIZE, 3])
 pretrained_model.trainable = True
model = tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation = 'softmax')
 ])
 model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['sparse_categorical_accuracy'])
 return model<choose_model_class>
scores = {}
score = cv_rmse(lightgbm)
 print("lightgbm: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['lgb'] =(score.mean() , score.std())
 score = cv_rmse(xgboost)
 print("xgboost: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['xgb'] =(score.mean() , score.std())
 score = cv_rmse(catboost)
 print("Catboost: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['Cat'] =(score.mean() , score.std())
 score = cv_rmse(svr)
 print("SVR: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['svr'] =(score.mean() , score.std())
 score = cv_rmse(ridge)
 print("ridge: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['ridge'] =(score.mean() , score.std())
 score = cv_rmse(lasso)
 print("lasso: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['lasso'] =(score.mean() , score.std())
 score = cv_rmse(elasticnet)
 print("elasticnet: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['elasticnet'] =(score.mean() , score.std())
 score = cv_rmse(kernelridge)
 print("kernelridge: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['kernelridge'] =(score.mean() , score.std())
 score = cv_rmse(gbr)
 print("gbr: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['gbr'] =(score.mean() , score.std())
 score = cv_rmse(rf)
 print("rf: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['rf'] =(score.mean() , score.std())
def create_MobileNetV2_model() :
pretrained_model = tf.keras.applications.MobileNetV2(weights = 'imagenet', include_top = False, input_shape = [*IMAGE_SIZE, 3])
 pretrained_model.trainable = True
model = tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation = 'softmax')
 ])
 model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['sparse_categorical_accuracy'])
 return model<choose_model_class>
print('stack_gen')
 stack_gen_model = stack_gen.fit(np.array(X), np.array(train_labels))
print('lightgbm')
 lgb_model_full_data = lightgbm.fit(X, train_labels)
 print('xgboost')
 xgb_model_full_data = xgboost.fit(X, train_labels)
 print('catoost')
 catboost_model_full_data = catboost.fit(X, train_labels)
 print('Svr')
 svr_model_full_data = svr.fit(X, train_labels)
 print('Ridge')
 ridge_model_full_data = ridge.fit(X, train_labels)
 print("Lasso")
 lasso_model_full_data = lasso.fit(X, train_labels)
 print("Elasticnet")
 elasticnet_model_full_data = elasticnet.fit(X, train_labels)
 print("kernelridge")
 kernelridge_model_full_data = kernelridge.fit(X, train_labels)
 print('RandomForest')
 rf_model_full_data = rf.fit(X, train_labels)
 print('GradientBoosting')
 gbr_model_full_data = gbr.fit(X, train_labels )
<define_variables><EOS> 
def blended_predictions(X):
return(( 0.1 * lgb_model_full_data.predict(X)) 
(0.1 * ridge_model_full_data.predict(X)) + \ 
(0.2 * svr_model_full_data.predict(X)) + \ 
(0.1 * gbr_model_full_data.predict(X)) + \ 
(0.1 * xgb_model_full_data.predict(X)) + \ 
(0.05 * rf_model_full_data.predict(X)) + \ 
(0.05 * lasso_model_full_data.predict(X)) + \ 
(0.05 * elasticnet_model_full_data.predict(X)) + \ 
(0.05 * kernelridge_model_full_data.predict(X)) + \ 
(0.35 * stack_gen_model.predict(np.array(X)))
 +(0.05 * catboost_model_full_data.predict(X)))
 blended_score = rmsle(train_labels, blended_predictions(X))
scores['blended'] =(blended_score, 0)
 print('RMSLE score on train data:')
 print(blended_score)
 sns.set_style("white")
 fig = plt.figure(figsize=(24, 12))
ax = sns.pointplot(x=list(scores.keys()), y=[score for score, _ in scores.values() ], markers=['o'], linestyles=['-'])
 for i, score in enumerate(scores.values()):
ax.text(i, score[0] + 0.002, '{:.6f}'.format(score[0]), horizontalalignment='left', size='large', color='black', weight='semibold')
 plt.ylabel('Score(RMSE)', size=20, labelpad=12.5)
 plt.xlabel('Model', size=20, labelpad=12.5)
 plt.tick_params(axis='x', labelsize=13.5)
 plt.tick_params(axis='y', labelsize=12.5)
 plt.title('Scores of Models', size=20)
 plt.show()
print("Done")