Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<init_hyperparams>
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
plt.style.use('ggplot')
 warnings.filterwarnings('ignore')
 plt.rc('font', size=18)
 plt.rc('axes', titlesize=22)
 plt.rc('axes', labelsize=18)
 plt.rc('xtick', labelsize=12)
 plt.rc('ytick', labelsize=12)
 plt.rc('legend', fontsize=12)
 plt.rcParams['font.sans-serif'] = ['Verdana']
pd.options.mode.chained_assignment = None
pd.options.display.max_seq_items = 500
pd.options.display.max_rows = 500
pd.set_option('display.float_format', lambda x: '%.5f' % x)
 BASE_PATH = "/kaggle/input/house-prices-advanced-regression-techniques/"
params={'learning_rate':0.01,
'boosting_type':'gbdt',
'objective':'regression',
'metric':'rmse',
'sub_feature':0.5,
'num_leaves':180,
'feature_fraction': 0.5,
'bagging_fraction': 0.5,
'min_data':50,
'max_depth':-1,
'reg_alpha': 0.3,
'reg_lambda': 0.1,
'min_child_weight': 10,
'verbose': 1,
'nthread':5,
'max_bin':512,
'subsample_for_bin':200,
'min_split_gain':0.0001,
'min_child_samples':5
}<train_model>
df = pd.read_csv(f"{BASE_PATH}train.csv")
 df_test = pd.read_csv(f"{BASE_PATH}test.csv")
 feat = pd.concat([df, df_test] ).reset_index(drop=True ).copy()
lgbm = lightgbm.train(params,
train_data,
25000,
valid_sets=valid_data,
early_stopping_rounds= 80,
verbose_eval= 10
)<predict_on_test>
feat.Alley = feat.Alley.fillna("None")
 bsmt_cols = ['BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'BsmtQual']
for col in bsmt_cols:
feat[col] = feat[col].fillna("None")
 for col in ['BsmtFullBath', 'BsmtHalfBath', 'BsmtUnfSF', 'BsmtFinSF1', 'BsmtFinSF2', 'TotalBsmtSF']:
feat[col] = feat[col].fillna(0 ).astype(int)
 feat.Electrical = feat.Electrical.fillna("SBrkr")
 feat.Exterior1st = feat.Exterior1st.fillna("VinylSd")
 feat.Exterior2nd = feat.Exterior2nd.fillna("None")
 feat.Fence = feat.Fence.fillna("None")
 feat.FireplaceQu = feat.FireplaceQu.fillna("None")
 feat.Functional = feat.Functional.fillna("Typ")
 grg_cols = ['GarageCond', 'GarageFinish', 'GarageQual', 'GarageType']
for col in grg_cols:
feat[col] = feat[col].fillna("None")
 for col in ['GarageArea', 'GarageCars', 'GarageYrBlt']:
feat[col] = feat[col].fillna(0 ).astype(int)
 to_fix = feat[feat.GarageYrBlt == 2207].index
feat.loc[to_fix, "GarageYrBlt"] = int(feat.GarageYrBlt.mean())
 feat.KitchenQual = feat.KitchenQual.fillna("TA")
 feat.LotFrontage = feat.LotFrontage.fillna(0 ).astype(int)
 feat.MSZoning = feat.MSZoning.fillna("RL")
 feat.MSSubClass = feat.MSSubClass.astype(str)
 feat.MasVnrType = feat.MasVnrType.fillna("None")
 feat.MasVnrArea = feat.MasVnrArea.fillna(0 ).astype(int)
 feat.MiscFeature = feat.MiscFeature.fillna("None")
 feat.PoolQC = feat.PoolQC.fillna("None")
 feat.SaleType = feat.SaleType.fillna("WD")
 feat.Utilities = feat.Utilities.fillna("AllPub" )
pred_lgbm=np.expm1(lgbm.predict(test_sc))
pred_lgbm<import_modules>
feat = feat.replace({ "Alley": {"None" : 0, "Grvl" : 1, "Pave" : 2},
"BsmtCond": {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3,
"Gd" : 4, "Ex" : 5},
"BsmtExposure": {"None" : 0, "No" : 2, "Mn" : 2, "Av": 3,
"Gd" : 4},
"BsmtFinType1": {"None" : 0, "Unf" : 1, "LwQ": 2, "Rec" : 3,
"BLQ" : 4, "ALQ" : 5, "GLQ" : 6},
"BsmtFinType2": {"None" : 0, "Unf" : 1, "LwQ": 2, "Rec" : 3,
"BLQ" : 4,
"ALQ" : 5, "GLQ" : 6},
"BsmtQual": {"None" : 0, "Po" : 1, "Fa" : 2, "TA": 3,
"Gd" : 4, "Ex" : 5},
"CentralAir": {"None" : 0, "N" : 1, "Y" : 2},
"ExterCond": {"None" : 0, "Po" : 1, "Fa" : 2, "TA": 3,
"Gd": 4, "Ex" : 5},
"ExterQual": {"None" : 0, "Po" : 1, "Fa" : 2, "TA": 3,
"Gd": 4, "Ex" : 5},
"Fence": {"None" : 0, "MnWw" : 1, "GdWo" : 2, "MnPrv": 3,
"GdPrv" : 4},
"FireplaceQu": {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3,
"Gd" : 4, "Ex" : 5},
"Functional": {"None" : 0, "Sal" : 1, "Sev" : 2, "Maj2" : 3,
"Maj1" : 4, "Mod": 5, "Min2" : 6, "Min1" : 7,
"Typ" : 8},
"GarageCond": {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3,
"Gd" : 4, "Ex" : 5},
"GarageQual": {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3,
"Gd" : 4, "Ex" : 5},
"GarageFinish": {"None" : 0, "Unf" : 1, "RFn" : 2, "Fin" : 3},
"HeatingQC": {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3,
"Gd" : 4, "Ex" : 5},
"KitchenQual": {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3,
"Gd" : 4, "Ex" : 5},
"LandContour": {"None" : 0, "Low" : 1, "HLS" : 2, "Bnk" : 3,
"Lvl" : 4},
"LandSlope": {"None" : 0, "Sev" : 1, "Mod" : 2, "Gtl" : 3},
"LotShape": {"None" : 0, "IR3" : 1, "IR2" : 2, "IR1" : 3,
"Reg" : 4},
"PavedDrive": {"None" : 0, "N" : 0, "P" : 1, "Y" : 2},
"PoolQC": {"None" : 0, "Fa" : 1, "TA" : 2, "Gd" : 3,
"Ex" : 4},
"Street": {"None" : 0, "Grvl" : 1, "Pave" : 2},
"Utilities": {"None" : 0, "ELO" : 1, "NoSeWa" : 2, "NoSewr" : 3,
"AllPub" : 4}} 
)
 feat.BsmtCond = feat.BsmtCond.astype(int )
from catboost import CatBoostRegressor<choose_model_class>
def onehot_encode(data):
df_numeric = data.select_dtypes(exclude=['object'])
 df_obj = data.select_dtypes(include=['object'] ).copy()
cols = []
for c in df_obj:
dummies = pd.get_dummies(df_obj[c])
 dummies.columns = [c + "_" + str(x)for x in dummies.columns]
cols.append(dummies)
 df_obj = pd.concat(cols, axis=1)
 data = pd.concat([df_numeric, df_obj], axis=1)
 data.reset_index(inplace=True, drop=True)
 return data
feat = onehot_encode(feat )
cb_model = CatBoostRegressor(iterations=500,
learning_rate=0.1,
depth=5,
l2_leaf_reg=20,
bootstrap_type='Bernoulli',
subsample=0.6,
eval_metric='RMSE',
random_seed = 42,
od_type='Iter',
metric_period = 50,
od_wait=45 )<train_model>
seasons = {12 : 0, 1 : 0, 2 : 0,
3 : 1, 4 : 1, 5 : 1,
6 : 2, 7 : 2, 8 : 2,
9 : 3, 10 : 3, 11 : 3}
feat["SeasonSold"] = feat["MoSold"].map(seasons)
 feat["YrActualAge"] = feat["YrSold"] - feat["YearBuilt"]
feat['TotalSF1'] = feat['TotalBsmtSF'] + feat['1stFlrSF'] + feat['2ndFlrSF']
feat['TotalSF2'] = feat['BsmtFinSF1'] + feat['BsmtFinSF2'] + feat['1stFlrSF'] + feat['2ndFlrSF']
feat["AllSF"] = feat["GrLivArea"] + feat["TotalBsmtSF"]
feat["AllFlrsSF"] = feat["1stFlrSF"] + feat["2ndFlrSF"]
feat["AllPorchSF"] = feat["OpenPorchSF"] + feat["EnclosedPorch"] + feat["3SsnPorch"] + feat["ScreenPorch"]
feat['TotalBath'] = 2 *(feat['FullBath'] +(0.5 * feat['HalfBath'])+ feat['BsmtFullBath'] +(0.5 * feat['BsmtHalfBath']))
feat["TotalBath"] = feat["TotalBath"].astype(int)
 feat['TotalPorch'] = feat['OpenPorchSF'] + feat['3SsnPorch'] + feat['EnclosedPorch'] + feat['ScreenPorch'] + feat['WoodDeckSF']
feat["OverallScore"] = feat["OverallQual"] * feat["OverallCond"]
feat["GarageScore"] = feat["GarageQual"] * feat["GarageCond"]
feat["ExterScore"] = feat["ExterQual"] * feat["ExterCond"]
feat["KitchenScore"] = feat["KitchenAbvGr"] * feat["KitchenQual"]
feat["FireplaceScore"] = feat["Fireplaces"] * feat["FireplaceQu"]
feat["GarageScore"] = feat["GarageArea"] * feat["GarageQual"]
feat["PoolScore"] = feat["PoolArea"] * feat["PoolQC"]
feat['hasPool'] = feat['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 feat['has2ndFloor'] = feat['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 feat['hasGarage'] = feat['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 feat['hasBsmt'] = feat['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 feat['hasFireplace'] = feat['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )
cb_model.fit(X_train, y_train,
eval_set=(X_val, y_val),
use_best_model=True,
verbose=True )<predict_on_test>
pred_cb=np.expm1(cb_model.predict(test_sc))
pred_cb<import_modules>
from sklearn.ensemble import RandomForestRegressor<import_modules>
X = dtrain[dtrain.GrLivArea < 4000].drop(["SalePrice"], axis=1)
 y = np.log1p(dtrain[dtrain.GrLivArea < 4000].SalePrice)
 X_test = dtest
metric = 'neg_mean_squared_error'
from sklearn.ensemble import RandomForestRegressor<choose_model_class>
sk = pd.DataFrame(X.iloc[:, :60].skew() , columns=["skewness"])
 sk = sk[sk.skewness >.75]
for feature_ in sk.index:
X[feature_] = boxcox1p(X[feature_], 0.15)
 X_test[feature_] = boxcox1p(X_test[feature_], 0.15 )
rf_model = RandomForestRegressor(n_estimators=100 )<train_model>
ridge = make_pipeline(RobustScaler() , Ridge(alpha=15, random_state=1))
lasso = make_pipeline(RobustScaler() , Lasso(alpha=0.0005, random_state=1))
elastic = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005,
l1_ratio=1, random_state=1))
kernel = KernelRidge(alpha=1.0)
 gbm = GradientBoostingRegressor(n_estimators=2500, learning_rate=0.04,
max_depth=2, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state=1)
 lgbm = lgb.LGBMRegressor(objective='regression', num_leaves=5,
learning_rate=0.03, n_estimators=8000,
max_bin=55, bagging_fraction=0.8,
bagging_freq=5, feature_fraction=0.23,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf=6, min_sum_hessian_in_leaf=11)
 xgbreg = xgb.XGBRegressor(objective="reg:squarederror",
colsample_bytree=0.46, gamma=0.047,
learning_rate=0.04, max_depth=2,
min_child_weight=0.5, n_estimators=2000,
reg_alpha=0.46, reg_lambda=0.86,
subsample=0.52, random_state=1, n_jobs=-1 )
rf_model.fit(X_train, y_train )<predict_on_test>
classifiers = [ridge, lasso, elastic, kernel, gbm, lgbm, xgbreg]
clf_names = ["ridge ", "lasso ", "elastic", "kernel ", "gbm ", "lgbm ", "xgbreg "]
predictions_exp = []
for clf_name, clf in zip(clf_names, classifiers):
print(f"{clf_name} {np.sqrt(-cross_val_score(clf, X, y, scoring=metric ).mean()):.5f}")
 clf.fit(X, y)
 preds = clf.predict(X_test)
 predictions_exp.append(np.expm1(preds))
pred_rf=np.expm1(rf_model.predict(test_sc))
pred_rf<import_modules>
weighted =(1 * predictions_exp[0] + \
1 * predictions_exp[1] + \
1 * predictions_exp[2] + \
1 * predictions_exp[3] + \
1 * predictions_exp[4] + \
1 * predictions_exp[5] + \
1 * predictions_exp[6])/ 7
<choose_model_class><EOS> 
prediction_final = pd.DataFrame(weighted, columns=["SalePrice"])
 submission = pd.DataFrame({'Id': df_test.Id.values, 'SalePrice': prediction_final.SalePrice.values})
 submission.to_csv(f"submission.csv", index=False )