Kaggle competition: House Prices - Advanced Regression Techniques
seasons = {12 : 0, 1 : 0, 2 : 0,
3 : 1, 4 : 1, 5 : 1,
6 : 2, 7 : 2, 8 : 2,
9 : 3, 10 : 3, 11 : 3}
feat["SeasonSold"] = feat["MoSold"].map(seasons)
 feat["YrActualAge"] = feat["YrSold"] - feat["YearBuilt"]
feat['TotalSF1'] = feat['TotalBsmtSF'] + feat['1stFlrSF'] + feat['2ndFlrSF']
feat['TotalSF2'] = feat['BsmtFinSF1'] + feat['BsmtFinSF2'] + feat['1stFlrSF'] + feat['2ndFlrSF']
feat["AllSF"] = feat["GrLivArea"] + feat["TotalBsmtSF"]
feat["AllFlrsSF"] = feat["1stFlrSF"] + feat["2ndFlrSF"]
feat["AllPorchSF"] = feat["OpenPorchSF"] + feat["EnclosedPorch"] + feat["3SsnPorch"] + feat["ScreenPorch"]
feat['TotalBath'] = 2 *(feat['FullBath'] +(0.5 * feat['HalfBath'])+ feat['BsmtFullBath'] +(0.5 * feat['BsmtHalfBath']))
feat["TotalBath"] = feat["TotalBath"].astype(int)
 feat['TotalPorch'] = feat['OpenPorchSF'] + feat['3SsnPorch'] + feat['EnclosedPorch'] + feat['ScreenPorch'] + feat['WoodDeckSF']
feat["OverallScore"] = feat["OverallQual"] * feat["OverallCond"]
feat["GarageScore"] = feat["GarageQual"] * feat["GarageCond"]
feat["ExterScore"] = feat["ExterQual"] * feat["ExterCond"]
feat["KitchenScore"] = feat["KitchenAbvGr"] * feat["KitchenQual"]
feat["FireplaceScore"] = feat["Fireplaces"] * feat["FireplaceQu"]
feat["GarageScore"] = feat["GarageArea"] * feat["GarageQual"]
feat["PoolScore"] = feat["PoolArea"] * feat["PoolQC"]
feat['hasPool'] = feat['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 feat['has2ndFloor'] = feat['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 feat['hasGarage'] = feat['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 feat['hasBsmt'] = feat['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 feat['hasFireplace'] = feat['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )<prepare_x_and_y>
%matplotlib inline
warnings.filterwarnings('ignore' )
<compute_train_metric>
house_train= pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 house_test= pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 print("Dataset shape:",'house_train', house_train.shape, 'house_test', house_test.shape )
<prepare_x_and_y>
house_train['SalePrice'] = np.log1p(house_train['SalePrice'] )
X = dtrain[dtrain.GrLivArea < 4000].drop(["SalePrice"], axis=1)
 y = np.log1p(dtrain[dtrain.GrLivArea < 4000].SalePrice)
 X_test = dtest
metric = 'neg_mean_squared_error'<feature_engineering>
outs = detect_outliers(house_train['GrLivArea'], house_train['SalePrice'],top=5)
 outs
sk = pd.DataFrame(X.iloc[:, :60].skew() , columns=["skewness"])
 sk = sk[sk.skewness >.75]
for feature_ in sk.index:
X[feature_] = boxcox1p(X[feature_], 0.15)
 X_test[feature_] = boxcox1p(X_test[feature_], 0.15 )<define_search_model>
quantitative = [feature for feature in house_train.columns if house_train.dtypes[feature] != 'object']
quantitative.remove('SalePrice')
 quantitative.remove('Id')
 qualitative = [feature for feature in house_train.columns if house_train.dtypes[feature] == 'object']
ridge = make_pipeline(RobustScaler() , Ridge(alpha=15, random_state=1))
lasso = make_pipeline(RobustScaler() , Lasso(alpha=0.0005, random_state=1))
elastic = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005,
l1_ratio=1, random_state=1))
kernel = KernelRidge(alpha=1.0)
 gbm = GradientBoostingRegressor(n_estimators=2500, learning_rate=0.04,
max_depth=2, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state=1)
 lgbm = lgb.LGBMRegressor(objective='regression', num_leaves=5,
learning_rate=0.03, n_estimators=8000,
max_bin=55, bagging_fraction=0.8,
bagging_freq=5, feature_fraction=0.23,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf=6, min_sum_hessian_in_leaf=11)
 xgbreg = xgb.XGBRegressor(objective="reg:squarederror",
colsample_bytree=0.46, gamma=0.047,
learning_rate=0.04, max_depth=2,
min_child_weight=0.5, n_estimators=2000,
reg_alpha=0.46, reg_lambda=0.86,
subsample=0.52, random_state=1, n_jobs=-1 )<compute_train_metric>
house_train.reset_index(drop=True, inplace=True)
 y_train = house_train['SalePrice']
X_train = house_train.drop(['SalePrice'], axis=1)
 X_test = house_test
print("Dataset shape:",'X_train', X_train.shape, 'y_train', y_train.shape, 'X_test', X_test.shape )
classifiers = [ridge, lasso, elastic, kernel, gbm, lgbm, xgbreg]
clf_names = ["ridge ", "lasso ", "elastic", "kernel ", "gbm ", "lgbm ", "xgbreg "]
predictions_exp = []
for clf_name, clf in zip(clf_names, classifiers):
print(f"{clf_name} {np.sqrt(-cross_val_score(clf, X, y, scoring=metric ).mean()):.5f}")
 clf.fit(X, y)
 preds = clf.predict(X_test)
 predictions_exp.append(np.expm1(preds))<define_variables>
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics_train = []; numerics_test = []
for i in house_train.columns:
if house_train[i].dtype in numeric_dtypes:
numerics_train.append(i)
 house_train.update(house_train[numerics_train].fillna(0))
for i in house_test.columns:
if house_test[i].dtype in numeric_dtypes:
numerics_test.append(i)
 house_test.update(house_test[numerics_test].fillna(0))
weighted =(1 * predictions_exp[0] + \
1 * predictions_exp[1] + \
1 * predictions_exp[2] + \
1 * predictions_exp[3] + \
1 * predictions_exp[4] + \
1 * predictions_exp[5] + \
1 * predictions_exp[6])/ 7<save_to_csv>
skew_train = house_train[quantitative].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skew_train = skew_train[skew_train > 0.5]
for i in high_skew_train.index:
house_train[i] = boxcox1p(house_train[i], boxcox_normmax(house_train[i] + 1))
skew_test = house_test[quantitative].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skew_test = skew_test[skew_train > 0.5]
for i in high_skew_test.index:
house_test[i] = boxcox1p(house_test[i], boxcox_normmax(house_test[i] + 1))
prediction_final = pd.DataFrame(weighted, columns=["SalePrice"])
 submission = pd.DataFrame({'Id': df_test.Id.values, 'SalePrice': prediction_final.SalePrice.values})
 submission.to_csv(f"submission.csv", index=False )<set_options>
all_data = pd.concat([X_train, house_test], axis=0, sort=False)
 all_data.drop(['Id'], axis=1, inplace=True)
 all_data.shape
%matplotlib inline 
<load_from_csv>
all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str )
data_df = pd.read_csv('.. /input/train.csv')
 data_df.head()<load_from_csv>
na_total = all_data.isnull().sum().sort_values(ascending=False)
 na_ratio =(all_data.isnull().sum() / all_data.shape[0] ).sort_values(ascending=False)
 missing_data = pd.concat([na_total, na_ratio], axis=1, keys=['Total', 'Ratio'])
 missing_data.head(10 )
test_data_df = pd.read_csv('.. /input/test.csv' )<create_dataframe>
all_data.drop(['PoolQC', 'Utilities', 'Street', 'MiscFeature', ], axis=1, inplace=True )
train_df = data_df.copy()
test_df = test_data_df.copy()<count_missing_values>
for col in('Alley','Fence','FireplaceQu','GarageQual','GarageFinish','GarageCond','GarageType','BsmtExposure',
'BsmtCond','BsmtQual','BsmtFinType2','BsmtFinType1'):
all_data[col] = all_data[col].fillna(0 )
data_df['LotFrontage'].isnull().sum()<filter>
print(all_data[all_data['GarageCars'].isnull() ][['GarageArea', 'GarageCars', 'GarageType', 'GarageYrBlt', 'GarageQual']])
 all_data['GarageArea'].fillna(0, inplace=True)
 all_data['GarageCars'].fillna(0, inplace=True )
data_df[data_df['TotalBsmtSF'] > 6000]<define_variables>
print(all_data[all_data['TotalBsmtSF'].isnull() ][
['TotalBsmtSF', 'BsmtFinSF2', 'BsmtUnfSF', 'BsmtFinSF1', 'BsmtFullBath','BsmtHalfBath']])
 for col in('TotalBsmtSF', 'BsmtFinSF2', 'BsmtUnfSF', 'BsmtFinSF1', 'BsmtFullBath','BsmtHalfBath'):
all_data[col] = all_data[col].fillna(0)
outliers = set(data_df[data_df['TotalBsmtSF'] > 6000].index.values)
 print('Running list of outliers: ', outliers )<filter>
all_data['MasVnrType'].fillna('None', inplace=True)
 all_data['HasMasVnr'] = all_data['MasVnrType'].apply(lambda x: 0 if x == 'None' else 1 )
data_df[data_df['1stFlrSF'] > 4000]<filter>
X_train['MSZoning'] = X_train.groupby(['MSSubClass'])['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
X_test['MSZoning'] = X_test.groupby(['MSSubClass'])['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
for col in('Functional','Exterior1st','Electrical','KitchenQual','SaleType','Exterior2nd'):
X_train[col] = X_train[col].fillna(X_train[col].mode() [0])
 X_test[col] = X_test[col].fillna(X_test[col].mode() [0] )
data_df[data_df['GrLivArea'] > 4000].loc[:, ['GrLivArea', 'SalePrice']]<compute_test_metric>
X_train['LotFrontage'] = X_train.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
X_train['GarageYrBlt'] =(X_train['YearBuilt'] + X_train['YearRemodAdd'])/2
X_train['MasVnrArea'] = X_train.groupby(['MasVnrType'])['MasVnrArea'].transform(lambda x: x.fillna(x.median()))
X_test['LotFrontage'] = X_test.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
X_test['GarageYrBlt'] =(X_test['YearBuilt'] + X_test['YearRemodAdd'])/2
X_test['MasVnrArea'] = X_test.groupby(['MasVnrType'])['MasVnrArea'].transform(lambda x: x.fillna(x.median()))
zscore = stats.zscore(data_df['GrLivArea'])
 thresh = 4
print(np.where(zscore > thresh), zscore[np.where(zscore > thresh)] )<define_variables>
X_train['YrBltAndRemod']=X_train['YearBuilt']+X_train['YearRemodAdd']
X_train['TotalSF']=X_train['TotalBsmtSF'] + X_train['1stFlrSF'] + X_train['2ndFlrSF']
X_train['TotalSqrFootage'] =(X_train['BsmtFinSF1'] + X_train['BsmtFinSF2'] +
X_train['1stFlrSF'] + X_train['2ndFlrSF'])
 X_train['TotalBathrooms'] =(X_train['FullBath'] +(0.5 * X_train['HalfBath'])+
X_train['BsmtFullBath'] +(0.5 * X_train['BsmtHalfBath']))
X_train['TotalPorchSF'] =(X_train['OpenPorchSF'] + X_train['3SsnPorch'] +
X_train['EnclosedPorch'] + X_train['ScreenPorch'] +
X_train['WoodDeckSF'])
 X_test['YrBltAndRemod']=X_test['YearBuilt']+X_test['YearRemodAdd']
X_test['TotalSF']=X_test['TotalBsmtSF'] + X_test['1stFlrSF'] + X_test['2ndFlrSF']
X_test['TotalSqrFootage'] =(X_test['BsmtFinSF1'] + X_test['BsmtFinSF2'] +
X_test['1stFlrSF'] + X_test['2ndFlrSF'])
 X_test['TotalBathrooms'] =(X_test['FullBath'] +(0.5 * X_test['HalfBath'])+
X_test['BsmtFullBath'] +(0.5 * X_test['BsmtHalfBath']))
X_test['TotalPorchSF'] =(X_test['OpenPorchSF'] + X_test['3SsnPorch'] +
X_test['EnclosedPorch'] + X_test['ScreenPorch'] +
X_test['WoodDeckSF'] )
outliers.update([outlier for outlier in list(np.where(data_df['GrLivArea'] > 4000)[0])if outlier not in outliers])
 print('Running list of outliers: ', outliers )<filter>
X_train['has2ndfloor'] = X_train['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 X_train['hasgarage'] = X_train['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 X_train['hasbsmt'] = X_train['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 X_train['hasfireplace'] = X_train['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 X_test['has2ndfloor'] = X_test['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 X_test['hasgarage'] = X_test['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 X_test['hasbsmt'] = X_test['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 X_test['hasfireplace'] = X_test['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )
data_df[data_df['PoolArea'] != 0].loc[:, ['PoolArea', 'PoolQC']]<data_type_conversions>
ohe = ce.OneHotEncoder(handle_unknown='ignore', use_cat_names=True)
 encoded_data=ohe.fit_transform(pd.concat([X_train,X_test], axis=0, sort=False)).reset_index(drop=True)
 X_train = encoded_data.iloc[:len(y_train), :]
X_test = encoded_data.iloc[len(y_train):, :]
print("Dataset shape:",'X_train', X_train.shape, 'y_train', y_train.shape, 'X_test', X_test.shape )
train_df['PoolQC'].fillna('None', inplace=True )<count_missing_values>
X_test.isnull().sum().sort_values(ascending=False )
test_df['PoolQC'].isnull().sum()<filter>
overfit = []
len_X_train =len(X_train)
 for i in X_train.columns:
counts = X_train[i].value_counts()
zeros = counts.iloc[0]
if zeros / len_X_train * 100 > 99.94 :
overfit.append(i)
 overfit = list(overfit)
 overfit.append('MSZoning_C(all)')
 X_train = np.array(X_train.drop(overfit, axis=1 ).copy())
 y_train = np.array(y_train)
 X_test = np.array(X_test.drop(overfit, axis=1 ).copy())
 print("Dataset shape:",'X_train', X_train.shape, 'y_train', y_train.shape, 'X_test', X_test.shape )
test_df[test_df['PoolArea'] != 0].loc[:, ['PoolArea', 'PoolQC']]<filter>
from sklearn.model_selection import cross_val_score, KFold
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import RobustScaler
from sklearn.pipeline import make_pipeline
from sklearn.linear_model import ElasticNet, Lasso, Ridge, ElasticNetCV, LassoCV, RidgeCV
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor
from sklearn.svm import SVR
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from mlxtend.regressor import StackingCVRegressor
test_df[(test_df['PoolArea'] != 0)&(test_df['PoolQC'].isnull())].loc[:, ['PoolQC']].fillna('Fa', inplace=True)
 test_df['PoolQC'].fillna('None', inplace=True )<drop_column>
kfolds = KFold(n_splits=10, shuffle=True, random_state=42)
 def rmsle(y, y_pred):
return np.sqrt(mse(y, y_pred))
def cv_rmse(model, X_train=X_train):
rmse = np.sqrt(-cross_val_score(model, X_train, y_train, scoring="neg_mean_squared_error", cv=kfolds))
return(rmse )
train_df.drop(columns=['MiscFeature'], inplace=True )<drop_column>
alphas_alt = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas2 = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
test_df.drop(columns=['MiscFeature'], inplace=True )<data_type_conversions>
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=alphas_alt, cv=kfolds))
lasso = make_pipeline(
 RobustScaler() ,
LassoCV(max_iter=1e7, alphas=alphas2, random_state=42, cv=kfolds))
elasticnet = make_pipeline(
 RobustScaler() ,
ElasticNetCV(max_iter=1e7, alphas=e_alphas, cv=kfolds, l1_ratio=e_l1ratio))
svr = make_pipeline(RobustScaler() , SVR(
 C=20,
epsilon=0.009,
gamma=0.0003, 
))
gbr = GradientBoostingRegressor(n_estimators=3000,
learning_rate=0.05,
max_depth=4,
max_features='sqrt',
min_samples_leaf=15,
min_samples_split=10,
loss='huber',
random_state=42)
 lightgbm = LGBMRegressor(
 objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=5000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
verbose=-1, 
)
 xgboost = XGBRegressor(learning_rate=0.01,
n_estimators=3460,
max_depth=5,
min_child_weight=0,
gamma=0,
subsample=0.7,
colsample_bytree=0.7,
objective='reg:squarederror',
nthread=-1,
scale_pos_weight=1,
seed=27,
reg_alpha=0.00006 )
train_df['Alley'].fillna('None', inplace=True)
 test_df['Alley'].fillna('None', inplace=True )<drop_column>
stack_gen = StackingCVRegressor(regressors=(ridge, lasso, elasticnet, gbr, xgboost, lightgbm),
meta_regressor=xgboost,
use_features_in_secondary=True )
train_df.drop(columns=['Fence'], inplace=True)
 test_df.drop(columns=['Fence'], inplace=True )<count_missing_values>
print('TEST score on CV')
 score = cv_rmse(ridge)
 print("Ridge score: {:.4f}({:.4f})".format(score.mean() , score.std()))
score = cv_rmse(lasso)
 print("Lasso score: {:.4f}({:.4f})".format(score.mean() , score.std()))
score = cv_rmse(elasticnet)
 print("ElasticNet score: {:.4f}({:.4f})".format(score.mean() , score.std()))
score = cv_rmse(svr)
 print("SVR score: {:.4f}({:.4f})".format(score.mean() , score.std()))
score = cv_rmse(lightgbm)
 print("Lightgbm score: {:.4f}({:.4f})".format(score.mean() , score.std()))
score = cv_rmse(gbr)
 print("GradientBoosting score: {:.4f}({:.4f})".format(score.mean() , score.std()))
score = cv_rmse(xgboost)
 print("Xgboost score: {:.4f}({:.4f})".format(score.mean() , score.std()))
print('Number of missing values in FireplaceQu = ', data_df['FireplaceQu'].isnull().sum())
 print('Number of missing values in Fireplaces = ', data_df['Fireplaces'].isnull().sum() )<filter>
print('Training Model')
 stack_gen_model = stack_gen.fit(X_train, y_train)
 print('Model Trained' )
len(data_df[(data_df['Fireplaces']!=0)&(data_df['FireplaceQu'].isnull())] )<filter>
print('Predict submission')
 result = np.floor(np.expm1(stack_gen_model.predict(X_test)) )
<data_type_conversions><EOS> 
submission=pd.DataFrame()
submission['Id'] = house_test['Id']
submission['SalePrice']= result
submission.head()
submission.to_csv("houseprice_submission.csv", index=False )