Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<count_missing_values>
%matplotlib inline
print(os.listdir(".. /input"))
warnings.filterwarnings('ignore')
hp.isna().sum().sum()<feature_engineering>
def NormalizeData(df):

 num_cols = df._get_numeric_data().columns
skew_features = df[num_cols].apply(lambda x: x.skew() ).sort_values(ascending=False)
 high_skew = skew_features[skew_features > 0.5]
skew_index = high_skew.index
for i in skew_index:
df[i] = boxcox1p(df[i], boxcox_normmax(df[i] + 1))
return df
hp[apply_log] = np.log(hp[apply_log] )<feature_engineering>
def OrdinalsToNumerics(df):

 df['ExterQual'] = df['ExterQual'].map(lambda x: {'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5}.get(x, 0))
df['BsmtQual'] = df['BsmtQual'].map(lambda x: {'NA':1,'Po':2, 'Fa':3, 'TA':4, 'Gd':5, 'Ex':6}.get(x, 0))
df['KitchenQual'] = df['KitchenQual'].map(lambda x: {'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5}.get(x, 0))
df['GarageFinish'] = df['GarageFinish'].map(lambda x: {'NA':1, 'Unf':2, 'RFn':3, 'Fin':4}.get(x, 0))
df['FireplaceQu'] = df['FireplaceQu'].map(lambda x: {'NA':1,'Po':2, 'Fa':3, 'TA':4, 'Gd':5, 'Ex':6}.get(x, 0))
df['HeatingQC'] = df['HeatingQC'].map(lambda x: {'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5}.get(x, 0))
df['OverallQual'] = pd.to_numeric(df['OverallQual'])
 df['ExterQual'] = pd.to_numeric(df['ExterQual'])
 df['BsmtQual'] = pd.to_numeric(df['BsmtQual'])
 df['KitchenQual'] = pd.to_numeric(df['KitchenQual'])
 df['GarageFinish'] = pd.to_numeric(df['GarageFinish'])
 df['FireplaceQu'] = pd.to_numeric(df['FireplaceQu'])
 df['HeatingQC'] = pd.to_numeric(df['HeatingQC'] )
def create_features(df):
df['TotalSF'] = df['TotalBsmtSF'] + df['1stFlrSF'] + df['2ndFlrSF']
df['TotalFeet'] = df['BsmtFinSF1'] + df['BsmtFinSF2'] + df['1stFlrSF'] + df['2ndFlrSF']
df['TotalBath'] = df['FullBath'] +(0.5 * df['HalfBath'])+ df['BsmtFullBath'] +(0.5 * df['BsmtHalfBath'])
 df['TotalPorch'] = df['OpenPorchSF'] + df['3SsnPorch'] + df['EnclosedPorch'] + df['ScreenPorch'] + df['WoodDeckSF']
df['HasPool'] = df['PoolArea'].apply(bigger_than_zero)
 df['Has2ndFlr'] = df['2ndFlrSF'].apply(bigger_than_zero)
 df['HasGarage'] = df['GarageArea'].apply(bigger_than_zero)
 df['HasBsmt'] = df['TotalBsmtSF'].apply(bigger_than_zero)
 df['HasFireplace'] = df['Fireplaces'].apply(bigger_than_zero)
<feature_engineering>
def EncodeData(df):

 code_df = pd.get_dummies(df, drop_first=True)
 test = code_df[df_train.shape[0]:]
X = code_df[:df_train.shape[0]]
return test, X
create_features(hp )<categorify>
def RMSE(test_y, y_pred):

 return np.sqrt(mean_squared_error(test_y, y_pred))
one_hot_encoder = p.OneHotEncoder(sparse=False, handle_unknown='ignore')
 hp_oh = pd.DataFrame(one_hot_encoder.fit_transform(hp[categorical_oh_features]), columns=one_hot_encoder.get_feature_names(categorical_oh_features))
hp_oh<drop_column>
def EvaluateModel(model, grid_param):

 gs = GridSearchCV(estimator=model, param_grid=grid_param, cv=5)
 gs.fit(X_train, y_train)
 keys = list(grid_param[0].keys())
 length = [len(grid_param[0][x])for x in keys]
scores = gs.cv_results_['mean_test_score']
scores = np.array(scores ).reshape(tuple(length))
if len(keys)> 1:
plt.figure(figsize=(10,5))
for ind, i in enumerate(grid_param[0]['alpha']):
plt.plot(grid_param[0]['l1_ratio'], scores[ind])
 plt.xlabel('l1_ratio')
 plt.ylabel('Mean score')
 plt.show()
else:
plt.figure(figsize=(10,5))
plt.plot(scores)
 plt.xlabel('alpha')
 plt.ylabel('Mean score')
 plt.show()
best_model = gs.best_estimator_
train_rmse = RMSE(y_train, best_model.predict(X_train))
test_rmse = RMSE(y_test, best_model.predict(X_test))
print('Best params: {}'.format(gs.best_params_))
print("RMSE train: {}".format(train_rmse))
print("RMSE test : {}".format(test_rmse))
return [train_rmse, test_rmse]
hp_oh.index = hp.index
hp.drop(categorical_oh_features, axis=1, inplace=True)
 hp = pd.concat([hp, hp_oh], axis=1)
 hp<categorify>
def Submit(model, test, csv_name='submission.csv'):

 predictions = model.predict(test)
 rescaled = inv_boxcox(predictions, maxlog)
 submit_df = pd.DataFrame({'Id': test_id, 'SalePrice': rescaled})
 submit_df.to_csv(csv_name, index=False)
 return print('Submissions printed to', csv_name )
ordinal_encoder = p.OrdinalEncoder()
hp[categorical_l_features] = pd.DataFrame(ordinal_encoder.fit_transform(hp[categorical_l_features]), columns=hp[categorical_l_features].columns)
 hp[categorical_l_features]<count_missing_values>

 df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )
hp.isna().sum().sum()<choose_model_class>

 train_y = df_train['SalePrice']
df = pd.concat(( df_train, df_test), sort=False ).reset_index(drop=True)
 df.drop('SalePrice', axis=1, inplace=True )
isolation_forest = e.IsolationForest()
isolation_forest.fit(hp)
 outliers_res = isolation_forest.predict(hp )<filter>

 df['MSSubClass'] = df['MSSubClass'].apply(str)
 df['OverallCond'] = df['OverallCond'].astype(str)
 df['YrSold'] = df['YrSold'].astype(str)
 df['MoSold'] = df['MoSold'].astype(str )
outliers = hp.iloc[outliers_res == -1].index<count_missing_values>
OrdinalsToNumerics(df )
t = hp.drop(outliers, axis=0)
 t.isna().sum().sum()
hp = t<count_missing_values>

 fill_none =('FireplaceQu', 'Fence', 'Alley', 'MiscFeature', 'PoolQC', 'GarageType', 'GarageFinish',
'GarageQual', 'GarageCond', 'MasVnrType', 'MSSubClass', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1',
'BsmtFinType2')
 fill_zero =('GarageArea', 'GarageCars', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF',
'BsmtFullBath', 'BsmtHalfBath', 'MasVnrArea')
 fill_mode =('Exterior1st', 'Exterior2nd', 'SaleType','Functional','Electrical','KitchenQual')
 for col in fill_none:
df[col] = df[col].fillna('None')
 for col in fill_zero:
df[col] = df[col].fillna(0)
 for col in fill_mode:
df[col] = df[col].fillna(df[col].mode() [0] )
hp.isna().sum().sum()<prepare_x_and_y>

 df["LotFrontage"] = df.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
df['MSZoning'] = df.groupby('MSSubClass')['MSZoning'].transform(
 lambda x: x.fillna(x.mode() [0]))
df['GarageYrBlt'][df['GarageYrBlt'].isnull() ] = df['YearBuilt'][df['GarageYrBlt'].isnull() ]
y = hp.SalePrice<drop_column>

 df['TotalSF'] = df['1stFlrSF']+df['2ndFlrSF']+df['TotalBsmtSF']
df['YrBltAndRemod']=df['YearBuilt']+df['YearRemodAdd']
df['TotalBathrooms'] =(df['FullBath'] +(0.5 * df['HalfBath'])+
df['BsmtFullBath'] +(0.5 * df['BsmtHalfBath']))
df['TotalPorchSF'] =(df['OpenPorchSF'] + df['3SsnPorch'] +
df['EnclosedPorch'] + df['ScreenPorch'] + df['WoodDeckSF'])
 df['IsPool'] = df['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 df['Is2ndFloor'] = df['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 df['IsGarage'] = df['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 df['IsBsmt'] = df['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 df['IsFireplace'] = df['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
X = hp.drop(columns='SalePrice' )<split>
drop_cols = ["Alley", "ExterCond", "BsmtFinSF2", "Heating", "Electrical",\
"BsmtHalfBath", "PavedDrive", "Fence", "MiscFeature", "MoSold",\
"BsmtFinType2", "LandContour", "EnclosedPorch",\
"SaleType", "Functional", "LotConfig",\
"ScreenPorch", "YrSold", "LandSlope", 'Utilities', 'PoolQC', 'Street']
for col in [*drop_cols]:
df.drop([col], axis=1, inplace=True )
X_train, X_hold_out, X_val = np.split(X, [int (.7*len(X)) , int (.9*len(X)) ])
 y_train, y_hold_out, y_val = np.split(y, [int (.7*len(y)) , int (.9*len(y)) ] )<count_missing_values>
df_norm = NormalizeData(df )
hp.isna().sum().sum()<split>
test, X = EncodeData(df_norm )
<normalization>
X_train, X_test, y_train, y_test = train_test_split(X, train_y, test_size=0.2, random_state=80 )
scaler = p.StandardScaler()
X_train = pd.DataFrame(scaler.fit_transform(X_train),columns=X_train.columns)
 X_hold_out = pd.DataFrame(scaler.transform(X_hold_out),columns=X_hold_out.columns)
 X_val = pd.DataFrame(scaler.transform(X_val),columns=X_val.columns)
 X = pd.DataFrame(scaler.transform(X),columns=X.columns)
 X<train_on_grid>
lm = FitModel(LinearRegression() , X_train, y_train )
<choose_model_class>
lm_rmse_train = RMSE(y_train, lm.predict(X_train))
lm_rmse_test = RMSE(y_test, lm.predict(X_test))
lm_list = [lm_rmse_train, lm_rmse_test]
print("RMSE train: {}".format(lm_rmse_train))
print("RMSE test: {}".format(lm_rmse_test))
<choose_model_class>
random = 80
params = [{'alpha': np.logspace(-4, 4, 20)}]
ridge_list = EvaluateModel(Ridge(random_state = random), grid_param = params )
regressor_svr = svm.SVR(kernel='rbf', C=2, epsilon=0.01, gamma=0.0005)
 regressor_en = lm.ElasticNet(alpha=0.1, l1_ratio=0.025)
 regressor_gbr = e.GradientBoostingRegressor(learning_rate=0.05, max_depth=3, n_estimators=1200)
 regressor_lgbm = lgb.LGBMRegressor(lambda_l1=0.01, lambda_l2=0.1, learning_rate=0.05,
min_data_in_leaf=25, n_estimators=2000, num_leaves=4)
 regressor_xgb = xgb.XGBRegressor(learning_rate=0.01, max_depth=4, n_estimators=4000)
 ensemble_of_regressors = [regressor_svr, regressor_en, regressor_gbr, regressor_lgbm, regressor_xgb]<train_model>
random = 80
params = [{'alpha': np.logspace(-3, 4, 20)}]
lasso_list = EvaluateModel(Lasso(random_state = random, normalize=False), grid_param = params )
for clf in ensemble_of_regressors:
clf.fit(X_train, y_train )<predict_on_test>
random = 80
params = [{'alpha': np.logspace(-2, 4, 20), 'l1_ratio': np.linspace(0, 1, 20)}]
enet_list = EvaluateModel(ElasticNet(random_state = random), grid_param = params )
def get_predictions(x, ensemble_of_regressors, y=None):
pred_result = pd.DataFrame()
i = 1
for clf in ensemble_of_regressors:
y_pred = clf.predict(x)
 if y is not None:
print(clf.__class__.__name__, m.r2_score(y, y_pred))
pred_result.insert(i - 1, 'y_pred_' + str(i), y_pred)
 i += 1
return pred_result<predict_on_test>
index = ['train_RMSE', 'test_RMSE']
compare_models = pd.DataFrame({'LinearRegression': lm_list,
'Ridge': ridge_list,
'LASSO': lasso_list,
'ElasticNet': enet_list}, index=index)
 compare_models
pred_result = get_predictions(X_hold_out, ensemble_of_regressors, y_hold_out )<prepare_x_and_y>
ridge = FitModel(Ridge(alpha=11.288378916846883, random_state=80), X, train_y )
X_stack_train = pred_result
y_stack_train = y_hold_out<train_model>
Submit(ridge, test, 'final_submit.csv' )