Kaggle competition: House Prices - Advanced Regression Techniques
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head()<data_type_conversions>
df_train = pd.read_csv(r".. /input/train.csv")
 df_test = pd.read_csv(r".. /input/test.csv")
 pd.set_option('display.width', 700)
 df_train.head(5 )
all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 all_data['OverallCond'] = all_data['OverallCond'].astype(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str)
<categorify>
count_nans = len(df_train)- df_train.count()
df_count_nans = count_nans.to_frame()
df_count_nans.columns=["train_nan_count"]
df_count_nans["%_train_nans"]=(df_count_nans["train_nan_count"]/df_train.shape[0])* 100
df_count_nans["test_nan_count"] = len(df_test)- df_test.count()
df_count_nans["%_test_nans"]=(df_count_nans["test_nan_count"]/df_test.shape[0])* 100
df_count_nans.sort_values("train_nan_count", ascending=False, inplace=True)
 df_count_nans.query('train_nan_count > 0 or test_nan_count > 0' )
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[c].values))
all_data[c] = lbl.transform(list(all_data[c].values))
print('Shape all_data: {}'.format(all_data.shape)) 
<feature_engineering>
y_train = df_train.SalePrice.values
print(y_train)
 df_train.drop("SalePrice", inplace=True, axis=1 )
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF'] 
<feature_engineering>
df_all_data = pd.concat([df_train, df_test])
 df_all_data.reset_index(inplace=True, drop=True)
 print(df_all_data.shape)
 df_all_data.columns
df_all_data.head()
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
all_data[feat] = boxcox1p(all_data[feat], lam)
<categorify>
df_all_data.drop(["PoolQC", "Alley", "MiscFeature", "Fence"], axis=1, inplace=True )
all_data = pd.get_dummies(all_data)
 print(all_data.shape )<split>
df_all_data["FireplaceQu"].fillna("None", inplace=True)
 df_all_data[["GarageCond", "GarageType", "GarageFinish", "GarageQual"]] = df_all_data[["GarageCond", "GarageType", "GarageFinish", "GarageQual"]].fillna("None")
 df_all_data["GarageYrBlt"].fillna(0, inplace=True)
 df_all_data[["BsmtExposure","BsmtFinType1", "BsmtFinType2", "BsmtCond", "BsmtQual"]] = df_all_data[["BsmtExposure", "BsmtFinType1", "BsmtFinType2", "BsmtCond", "BsmtQual"]].fillna("None")
 df_all_data[["BsmtFullBath", "BsmtFinSF1", "BsmtFinSF2", "BsmtUnfSF", "TotalBsmtSF", "BsmtHalfBath"]] = df_all_data[["BsmtFullBath", "BsmtFinSF1", "BsmtFinSF2", "BsmtUnfSF", "TotalBsmtSF", "BsmtHalfBath"]].fillna(0)
 df_all_data["MasVnrArea"].fillna(0, inplace=True)
 df_all_data["MasVnrType"].fillna("None", inplace=True)
 most_common = df_all_data["Electrical"].value_counts().index[0]
df_all_data["Electrical"].fillna(most_common, inplace=True)
 df_all_data["Functional"].fillna("Typ", inplace=True)
 df_all_data["KitchenQual"].fillna("TA", inplace=True)
 most_common = df_all_data["SaleType"].value_counts().index[0]
df_all_data["SaleType"].fillna(most_common, inplace=True)
 df_all_data["Utilities"].fillna("None", inplace=True)
 df_all_data["Exterior1st"].fillna("None", inplace=True)
 df_all_data["Exterior2nd"].fillna("None", inplace=True)
 most_common = df_all_data["MSZoning"].value_counts().index[0]
df_all_data["MSZoning"].fillna(most_common, inplace=True )
train = all_data[:ntrain]
test = all_data[ntrain:] 
<import_modules>
grp=df_all_data.groupby("Neighborhood")["GarageCars"].mean()
nan_idx = df_all_data[df_all_data["GarageCars"].isnull() ==True].index.tolist()
for idx in nan_idx:
df_all_data.loc[idx, "GarageCars"] = int(round(grp.loc[df_all_data.iloc[idx]["Neighborhood"]]))
grp=df_all_data.groupby("Neighborhood")["GarageArea"].mean()
nan_idx = df_all_data[df_all_data["GarageArea"].isnull() ==True].index.tolist()
for idx in nan_idx:
df_all_data.loc[idx, "GarageArea"] = int(round(grp.loc[df_all_data.iloc[idx]["Neighborhood"]]))
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb 
<compute_train_metric>
df_all_data["LotFrontage"] = df_all_data.groupby("BldgType")["LotFrontage"].transform(lambda x: x.fillna(x.mean()))
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )<choose_model_class>
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column] >= 0.75)]
print(to_drop )
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))<choose_model_class>
df_all_data.drop(to_drop, axis=1, inplace=True )
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))<choose_model_class>
drop_points = df_train.sort_values(by = 'GrLivArea', ascending = False)[:2]["GrLivArea"]
drop_points_list = drop_points.index.tolist()
df_all_data.drop(drop_points_list, inplace=True)
 y_train = np.delete(y_train, drop_points_list)
 df_train_last_index=df_train.shape[0]-len(drop_points_list )
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5 )<choose_model_class>
df_all_data['MSSubClass'] = df_all_data['MSSubClass'].apply(str)
 df_all_data["MSSubClass"] = LabelEncoder().fit_transform(df_all_data["MSSubClass"] )
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5 )<choose_model_class>
skew_thresh = 0.5
skewed = df_all_data.skew().sort_values(ascending=False)
 a=skewed[abs(skewed)>skew_thresh]
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1)
<choose_model_class>
skewed_cols = skewed[abs(skewed)>skew_thresh].index.tolist()
print(len(skewed_cols))
print(skewed_cols)
 df_all_data[skewed_cols] = df_all_data[skewed_cols].apply(np.log1p)
 df_all_data[skewed_cols].head()
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )<compute_test_metric>
categorical_data_cols = df_all_data.select_dtypes(include=['object'])
 print(categorical_data_cols.columns.tolist())
 df_all_data = pd.get_dummies(df_all_data )
score = rmsle_cv(lasso)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
y_train = np.log1p(y_train )
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
df_all_data =(df_all_data - df_all_data.mean())/(df_all_data.max() - df_all_data.min() )
score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
def _ApplyLinearAlgo(model_obj, df_tr, df_te, y_train):
model_obj.fit(df_tr, y_train)
 y_predict = model_obj.predict(df_tr)
 print("r2 score train " + str(r2_score(y_train, y_predict)))
 print("rmse score train " + str(mean_squared_error(y_train, y_predict)))
 print(df_tr.shape)
 print(df_te.shape)
 y_te_pred = np.expm1(model_obj.predict(df_te))
return y_te_pred
score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
print("
")
 print("ElasticNetCV")
 lr = ElasticNetCV(alphas=[0.0001, 0.0005, 0.001, 0.01, 0.1, 1, 10], l1_ratio=[.01,.1,.5,.9,.99], max_iter=5000)
 y_pred_Elastic = _ApplyLinearAlgo(lr, df_tr, df_te, y_train)
 print("
")
 print("
RidgeCV")
 lr=RidgeCV(alphas=[0.0001, 0.0005, 0.001, 0.01, 0.1, 1, 10])
 y_te_Ridge = _ApplyLinearAlgo(lr, df_tr, df_te, y_train)
 print("
")
 print("RandomForestRegressor")
 lr = RandomForestRegressor()
y_te_RF = _ApplyLinearAlgo(lr, df_tr, df_te, y_train )
score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
idx = pd.read_csv(".. /input/test.csv" ).Id
my_submission = pd.DataFrame({'Id': idx, 'SalePrice': y_pred_Elastic})
 my_submission.to_csv('submission.csv', index=False)
 my_submission.head()