Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<data_type_conversions>
warnings.filterwarnings('ignore' )
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
all_data[col] = all_data[col].fillna('None' )<data_type_conversions>
def rmse_cv(model, X, y):

 rmse = np.sqrt(-cross_val_score(model, X_train, y_train, scoring="neg_mean_squared_error", cv=kfolds))
return(rmse)
 def rmsle(y, y_pred):

 return(np.sqrt(metrics.mean_squared_error(y, y_pred)))
 def CVscore(model, X, y):

 result = cross_val_score(model, X, y, cv=kfold)
 return result.mean()
def blend_models_predict(X, models):

 blend = []
for model in models:
blend.append(1/len(models)*model.predict(X))
return sum(blend )
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
all_data[col] = all_data[col].fillna(0 )<data_type_conversions>
df_train = pd.read_csv('.. /input/train.csv',index_col='Id')
 df_test = pd.read_csv('.. /input/test.csv',index_col='Id' )
for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
all_data[col] = all_data[col].fillna(0 )<data_type_conversions>
df_train.drop(df_train[(df_train['GrLivArea']>4000)&(df_train['SalePrice']<300000)].index, inplace=True )
for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
all_data[col] = all_data[col].fillna('None' )<data_type_conversions>
len_train = len(df_train )
all_data["MasVnrType"] = all_data["MasVnrType"].fillna("None")
 all_data["MasVnrArea"] = all_data["MasVnrArea"].fillna(0 )<data_type_conversions>
y = df_train['SalePrice']
all_data['MSZoning'] = all_data['MSZoning'].fillna(all_data['MSZoning'].mode() [0] )<drop_column>
df_all = pd.concat([df_train, df_test], sort=False )
all_data = all_data.drop(['Utilities'], axis=1 )<data_type_conversions>
Cat_toNone =('PoolQC','Fence','MiscFeature','Alley','FireplaceQu','GarageType','GarageFinish',
'GarageQual','GarageCond','BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1',
'BsmtFinType2','MasVnrType','MSSubClass')
 for c in Cat_toNone:
df_all[c] = df_all[c].fillna('None' )
all_data["Functional"] = all_data["Functional"].fillna("Typ" )<data_type_conversions>
Cat_toMode =('MSZoning','Electrical','KitchenQual','Exterior1st','Exterior2nd',
'Utilities','SaleType','Functional')
 for c in Cat_toMode:
df_all[c] = df_all[c].fillna(df_all[c].mode() [0] )
all_data['Electrical'] = all_data['Electrical'].fillna(all_data['Electrical'].mode() [0] )<data_type_conversions>
Num_toZero =('GarageArea','GarageCars','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF',
'BsmtFullBath','BsmtHalfBath','GarageYrBlt','MasVnrArea')
 for c in Num_toZero:
df_all[c] = df_all[c].fillna(0 )
all_data['KitchenQual'] = all_data['KitchenQual'].fillna(all_data['KitchenQual'].mode() [0] )<categorify>
df_all["LotFrontage"] = df_all.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))
all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0] )<data_type_conversions>
df_all['MSSubClass'] = df_all['MSSubClass'].astype(str )
all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0] )<data_type_conversions>
skewed_feats = df_all[numeric].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.75]
skewed_feats = skewed_feats.index
df_all[skewed_feats] = np.log1p(df_all[skewed_feats])
all_data['MSSubClass'] = all_data['MSSubClass'].fillna("None" )<count_missing_values>
df_all = df_all.replace({"Alley" : {"None" : 0, "Grvl" : 1, "Pave" : 2},
"BsmtCond" : {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"BsmtExposure" : {"None" : 0, "Mn" : 1, "Av": 2, "Gd" : 3},
"BsmtFinType1" : {"None" : 0, "Unf" : 1, "LwQ": 2, "Rec" : 3, "BLQ" : 4,
"ALQ" : 5, "GLQ" : 6},
"BsmtFinType2" : {"None" : 0, "Unf" : 1, "LwQ": 2, "Rec" : 3, "BLQ" : 4,
"ALQ" : 5, "GLQ" : 6},
"BsmtQual" : {"None" : 0, "Po" : 1, "Fa" : 2, "TA": 3, "Gd" : 4, "Ex" : 5},
"ExterCond" : {"Po" : 1, "Fa" : 2, "TA": 3, "Gd": 4, "Ex" : 5},
"ExterQual" : {"Po" : 1, "Fa" : 2, "TA": 3, "Gd": 4, "Ex" : 5},
"FireplaceQu" : {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"Functional" : {"Sal" : 1, "Sev" : 2, "Maj2" : 3, "Maj1" : 4, "Mod": 5,
"Min2" : 6, "Min1" : 7, "Typ" : 8},
"GarageCond" : {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"GarageQual" : {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"HeatingQC" : {"Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"KitchenQual" : {"Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"LandSlope" : {"Sev" : 1, "Mod" : 2, "Gtl" : 3},
"LotShape" : {"IR3" : 1, "IR2" : 2, "IR1" : 3, "Reg" : 4},
"PavedDrive" : {"N" : 0, "P" : 1, "Y" : 2},
"PoolQC" : {"None" : 0, "Fa" : 1, "TA" : 2, "Gd" : 3, "Ex" : 4},
"Street" : {"Grvl" : 1, "Pave" : 2},
"Utilities" : {"ELO" : 1, "NoSeWa" : 2, "NoSewr" : 3, "AllPub" : 4}}
)
all_data.isnull().sum().max()<data_type_conversions>
df_all['Bathrooms'] = df_all['FullBath'] +(df_all['HalfBath']*0.5)+ df_all['BsmtFullBath'] +(df_all['BsmtHalfBath']*0.5)
 df_all['FireplaceScore'] = df_all['Fireplaces'] * df_all['FireplaceQu']
df_all['GarageScore'] = df_all['GarageCars'] * df_all['GarageQual'] * df_all['GarageArea']
df_all['HouseAge'] = df_all['YrSold'] - df_all['YearBuilt']
df_all['TotalLivingSF'] = df_all['GrLivArea'] + df_all['TotalBsmtSF'] - df_all['LowQualFinSF']
all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 all_data['OverallCond'] = all_data['OverallCond'].astype(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str )<categorify>
df_all = df_all.drop(['FullBath', 'HalfBath', 'BsmtFullBath', 'BsmtHalfBath',
'GarageCars','GarageArea','GarageQual','GarageCond',
'FireplaceQu','Fireplaces',
'PoolArea',
'Utilities', 'Street',
'GarageYrBlt'
], axis = 1 )
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[c].values))
all_data[c] = lbl.transform(list(all_data[c].values))
print('Shape all_data: {}'.format(all_data.shape))<feature_engineering>
df_all = pd.get_dummies(df_all, drop_first=True )
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']<feature_engineering>
df_all = df_all.drop(['SalePrice'], axis = 1)
 train = df_all[:len_train]
test = df_all[len_train:]
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
all_data[feat] = boxcox1p(all_data[feat], lam)
<categorify>
X_train, X_test, y_train, y_test = train_test_split(train, y, test_size = 0.3, random_state = 0 )
all_data = pd.get_dummies(all_data)
 print(all_data.shape )<split>
regressor = LinearRegression()
regressor.fit(X_train, y_train)
 train_lm = regressor.predict(X_train)
 test_lm = regressor.predict(X_test)
 ridge = Ridge(alpha = 10)
 ridge.fit(X_train, y_train)
 ridge_train_lm = ridge.predict(X_train)
 ridge_test_lm = ridge.predict(X_test)
 lasso = Lasso(max_iter=500,alpha = 0.001)
 lasso.fit(X_train, y_train)
 lasso_train_lm = lasso.predict(X_train)
 lasso_test_lm = lasso.predict(X_test)
 en = ElasticNet(max_iter=500,alpha = 0.001)
 en.fit(X_train, y_train)
 en_train_lm = en.predict(X_train)
 en_test_lm = en.predict(X_test)
 dtr = DecisionTreeRegressor(max_depth=5)
 dtr.fit(X_train, y_train)
 train_dtr = dtr.predict(X_train)
 test_dtr = dtr.predict(X_test)
 rf = RandomForestRegressor(random_state = 0)
 rf.fit(X_train, y_train)
 train_rf = rf.predict(X_train)
 test_rf = rf.predict(X_test )
train = all_data[:ntrain]
test = all_data[ntrain:]<import_modules>
scorer = metrics.make_scorer(metrics.mean_squared_error, greater_is_better = False)
 kfolds = KFold(n_splits=10, shuffle=True, random_state=42 )
from sklearn.ensemble import RandomForestRegressor,GradientBoostingRegressor
import xgboost as xgb
import lightgbm as lgb
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error<compute_train_metric>
print('Linear Regression')
 print("RMSE on Training set :", rmse_cv(regressor,X_train,y_train ).mean())
 print("RMSE on Test set :", rmse_cv(regressor,X_test,y_test ).mean())
 print('RMSLE score on train data: ', rmsle(y_train, regressor.predict(X_train)))
 print('RMSLE score on test data: ', rmsle(y_test, regressor.predict(X_test)))
 print('-'*25)
 print('Linear Regression Ridge')
 print("RMSE on Training set :", rmse_cv(ridge,X_train,y_train ).mean())
 print("RMSE on Test set :", rmse_cv(ridge,X_test,y_test ).mean())
 print('RMSLE score on train data: ', rmsle(y_train, ridge.predict(X_train)))
 print('RMSLE score on test data: ', rmsle(y_test, ridge.predict(X_test)))
 print('-'*25)
 print('Linear Regression Lasso')
 print("RMSE on Training set :", rmse_cv(lasso,X_train,y_train ).mean())
 print("RMSE on Test set :", rmse_cv(lasso,X_test,y_test ).mean())
 print('RMSLE score on train data: ', rmsle(y_train, lasso.predict(X_train)))
 print('RMSLE score on test data: ', rmsle(y_test, lasso.predict(X_test)))
 print('-'*25)
 print('Linear Regression ElasticNet')
 print("RMSE on Training set :", rmse_cv(en,X_train,y_train ).mean())
 print("RMSE on Test set :", rmse_cv(en,X_test,y_test ).mean())
 print('RMSLE score on train data: ', rmsle(y_train, en.predict(X_train)))
 print('RMSLE score on test data: ', rmsle(y_test, en.predict(X_test)))
 print('-'*25)
 print('Decision Tree Regression')
 print("RMSE on Training set :", rmse_cv(dtr,X_train,y_train ).mean())
 print("RMSE on Test set :", rmse_cv(dtr,X_test,y_test ).mean())
 print('RMSLE score on train data: ', rmsle(y_train, dtr.predict(X_train)))
 print('RMSLE score on test data: ', rmsle(y_test, dtr.predict(X_test)))
 print('-'*25)
 print('Random Forest Regression')
 print("RMSE on Training set :", rmse_cv(rf,X_train,y_train ).mean())
 print("RMSE on Test set :", rmse_cv(rf,X_test,y_test ).mean())
 print('RMSLE score on train data: ', rmsle(y_train, rf.predict(X_train)))
 print('RMSLE score on test data: ', rmsle(y_test, rf.predict(X_test)) )
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )<choose_model_class>
ridge = Ridge(alpha = cv_ridge[cv_ridge == min(cv_ridge)].index[0])
 ridge.fit(X_train, y_train)
 ridge_train_lm = ridge.predict(X_train)
 ridge_test_lm = ridge.predict(X_test)
 lasso = Lasso(max_iter=500,alpha = cv_lasso[cv_lasso == min(cv_lasso)].index[0])
 lasso.fit(X_train, y_train)
 lasso_train_lm = lasso.predict(X_train)
 lasso_test_lm = lasso.predict(X_test)
 en = ElasticNet(max_iter=500,alpha = cv_elasticnet[cv_elasticnet == min(cv_elasticnet)].index[0])
 en.fit(X_train, y_train)
 en_train_lm = en.predict(X_train)
 en_test_lm = en.predict(X_test )
rfc=RandomForestRegressor(n_estimators=1000 )<choose_model_class>
print('Linear Regression Ridge')
 print("RMSE on Training set :", rmse_cv(ridge,X_train,y_train ).mean())
 print("RMSE on Test set :", rmse_cv(ridge,X_test,y_test ).mean())
 print('RMSLE score on train data: ', rmsle(y_train, ridge.predict(X_train)))
 print('RMSLE score on test data: ', rmsle(y_test, ridge.predict(X_test)))
 print('-'*25)
 print('Linear Regression Lasso')
 print("RMSE on Training set :", rmse_cv(lasso,X_train,y_train ).mean())
 print("RMSE on Test set :", rmse_cv(lasso,X_test,y_test ).mean())
 print('RMSLE score on train data: ', rmsle(y_train, lasso.predict(X_train)))
 print('RMSLE score on test data: ', rmsle(y_test, lasso.predict(X_test)))
 print('-'*25)
 print('Linear Regression ElasticNet')
 print("RMSE on Training set :", rmse_cv(en,X_train,y_train ).mean())
 print("RMSE on Test set :", rmse_cv(en,X_test,y_test ).mean())
 print('RMSLE score on train data: ', rmsle(y_train, en.predict(X_train)))
 print('RMSLE score on test data: ', rmsle(y_test, en.predict(X_test)) )
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5 )<choose_model_class>
alphas_ridge = [2.5, 5, 7.5, 10, 12.5, 15, 17.5, 20, 22.5, 25, 27.5, 30]
alphas_lasso = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008, 0.0009, 0.001, 0.0011]
alphas_en = [0.0006, 0.0007, 0.0008, 0.0009, 0.001, 0.0011, 0.0012, 0.0013, 0.0014, 0.0015, 0.0016]
l1ratio_en = [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 0.99, 1]
ridge = make_pipeline(RidgeCV(alphas=alphas_ridge, cv=kfolds))
lasso = make_pipeline(LassoCV(max_iter=500, alphas=alphas_lasso, random_state=42, cv=kfolds))
elasticnet = make_pipeline(ElasticNetCV(max_iter=500,alphas=alphas_en, cv=kfolds, l1_ratio=l1ratio_en))
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1 )<choose_model_class>
ridge_full = ridge.fit(X_train,y_train)
 lasso_full = lasso.fit(X_train,y_train)
 en_full = elasticnet.fit(X_train,y_train )
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )<compute_test_metric>
print('Linear Regression Ridge')
 print("RMSE on Training set :", rmse_cv(ridge_full,X_train,y_train ).mean())
 print("RMSE on Test set :", rmse_cv(ridge_full,X_test,y_test ).mean())
 print('RMSLE score on train data: ', rmsle(y_train, ridge_full.predict(X_train)))
 print('RMSLE score on test data: ', rmsle(y_test, ridge_full.predict(X_test)))
 print('-'*25)
 print('Linear Regression Lasso')
 print("RMSE on Training set :", rmse_cv(lasso_full,X_train,y_train ).mean())
 print("RMSE on Test set :", rmse_cv(lasso_full,X_test,y_test ).mean())
 print('RMSLE score on train data: ', rmsle(y_train, lasso_full.predict(X_train)))
 print('RMSLE score on test data: ', rmsle(y_test, lasso_full.predict(X_test)))
 print('-'*25)
 print('Linear Regression ElasticNet')
 print("RMSE on Training set: ", rmse_cv(en_full,X_train,y_train ).mean())
 print("RMSE on Test set: ", rmse_cv(en_full,X_test,y_test ).mean())
 print('RMSLE score on train data: ', rmsle(y_train, en_full.predict(X_train)))
 print('RMSLE score on test data: ', rmsle(y_test, en_full.predict(X_test)) )
score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
print('Ensembled Model')
 print('RMSLE score on train data: ', rmsle(y_train, blend_models_predict(X_train,[ridge_full,lasso_full,en_full])))
 print('RMSLE score on test data: ', rmsle(y_test, blend_models_predict(X_test,[ridge_full,lasso_full,en_full])) )
score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
blended_train = pd.DataFrame({'Id': X_train.index.values, 'SalePrice': np.expm1(blend_models_predict(X_train,[ridge_full,lasso_full,en_full])) })
 blended_test = pd.DataFrame({'Id': X_test.index.values, 'SalePrice': np.expm1(blend_models_predict(X_test,[ridge_full,lasso_full,en_full])) } )
score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<predict_on_test>
q1_tr = blended_train['SalePrice'].quantile(0.01)
 q2_tr = blended_train['SalePrice'].quantile(0.99)
 q1_te = blended_test['SalePrice'].quantile(0.01)
 q2_te = blended_test['SalePrice'].quantile(0.99)
 blended_train['SalePrice'] = blended_train['SalePrice'].apply(lambda x: x if x > q1_tr else x*0.8)
 blended_train['SalePrice'] = blended_train['SalePrice'].apply(lambda x: x if x < q2_tr else x*1.1)
 blended_test['SalePrice'] = blended_test['SalePrice'].apply(lambda x: x if x > q1_te else x*0.8)
 blended_test['SalePrice'] = blended_test['SalePrice'].apply(lambda x: x if x < q2_te else x*1.1 )
rfc.fit(train,y_train)
 rfc_train_pred = rfc.predict(train.values)
 rfc_pred = np.expm1(rfc.predict(test.values))<predict_on_test>
print('Ensembled Model')
 print('RMSLE score on train data: ', rmsle(y_train, np.log1p(blended_train['SalePrice'])))
 print('RMSLE score on test data: ', rmsle(y_test, np.log1p(blended_test['SalePrice'])) )
model_xgb.fit(train, y_train)
 xgb_train_pred = model_xgb.predict(train)
 xgb_pred = np.expm1(model_xgb.predict(test))<predict_on_test>
ridge_full = ridge.fit(train, y)
 lasso_full = lasso.fit(train, y)
 en_full = elasticnet.fit(train, y )
model_lgb.fit(train, y_train)
 lgb_train_pred = model_lgb.predict(train)
 lgb_pred = np.expm1(model_lgb.predict(test.values))<predict_on_test>
submission = pd.DataFrame({'Id': df_test.index.values, 'SalePrice': np.expm1(blend_models_predict(test,[ridge_full,lasso_full,en_full])) } )
GBoost.fit(train,y_train)
 GBoost_train_pred = GBoost.predict(train)
 GB_pred = np.expm1(GBoost.predict(test.values))<save_to_csv>
q1 = submission['SalePrice'].quantile(0.01)
 q2 = submission['SalePrice'].quantile(0.99)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x > q1 else x*0.8)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x < q2 else x*1.1 )
<import_modules><EOS> 
submission.to_csv("submission.csv", index=False)
 print('Save submission', )