Kaggle competition: House Prices - Advanced Regression Techniques
train['MSSubClass'] = train['MSSubClass'].apply(str)
 test['MSSubClass'] = test['MSSubClass'].apply(str)
 train['OverallCond'] = train['OverallCond'].astype(str)
 test['OverallCond'] = test['OverallCond'].astype(str)
 train['YrSold'] = train['YrSold'].astype(str)
 train['MoSold'] = train['MoSold'].astype(str)
 test['YrSold'] = test['YrSold'].astype(str)
 test['MoSold'] = test['MoSold'].astype(str )<categorify>
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 test_id = test['Id']
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(set(train[c].unique().tolist() + test[c].unique().tolist())))
 train[c] = lbl.transform(list(train[c].values))
test[c] = lbl.transform(list(test[c].values))<feature_engineering>
missing_values = train.isnull().sum()
display(missing_values[missing_values>0].sort_values(ascending=False ).to_frame(name='Missing Values'))
train['TotalSF'] = train['TotalBsmtSF'] + train['1stFlrSF'] + train['2ndFlrSF']
test['TotalSF'] = test['TotalBsmtSF'] + test['1stFlrSF'] + test['2ndFlrSF']<prepare_x_and_y>
train['OverallQual'].value_counts()
y_train = train.SalePrice
train.drop(['SalePrice','Id'],axis=1,inplace=True)
 test_Ids = test['Id']
test.drop('Id',axis=1,inplace=True )<feature_engineering>
imp_ftr = corrmat['SalePrice'].sort_values(ascending=False ).head(11 ).to_frame()
imp_ftr
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features in train data to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
train[feat] = boxcox1p(train[feat], lam )<feature_engineering>
train.drop(columns=['GarageArea','TotRmsAbvGrd','GarageYrBlt','1stFlrSF'],axis=1,inplace=True)
 test.drop(columns=['GarageArea','TotRmsAbvGrd','GarageYrBlt','1stFlrSF'],axis=1,inplace=True )
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features in test data to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
test[feat] = boxcox1p(test[feat], lam )<categorify>
train['MSSubClass'] = train['MSSubClass'].apply(str)
 test['MSSubClass'] = test['MSSubClass'].apply(str)
 train['OverallCond'] = train['OverallCond'].astype(str)
 test['OverallCond'] = test['OverallCond'].astype(str)
 train['YrSold'] = train['YrSold'].astype(str)
 test['YrSold'] = test['YrSold'].astype(str)
 train['MoSold'] = train['MoSold'].astype(str)
 test['MoSold'] = test['MoSold'].astype(str )
train = pd.get_dummies(train)
 test = pd.get_dummies(test)
 missing_cols = set(train.columns)- set(test.columns)
 for c in missing_cols:
test[c] = 0
missing_cols = set(test.columns)- set(train.columns)
 for c in missing_cols:
train[c] = 0
test = test[train.columns.tolist() ]
train.shape,test.shape<import_modules>
total_missing = train.isnull().sum().sort_values(ascending=False)
 total_missing = total_missing[total_missing>0]
percent =(( train.isnull().sum() /train.isnull().count())* 100 ).sort_values(ascending=False)
 percent = percent[percent>0]
data_missing = pd.concat([total_missing, percent], axis=1, keys=['Total', 'Percent'])
 data_missing
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb<compute_train_metric>
object_cols = [col for col in train.columns if train[col].dtype == "object"]
object_cols
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )<choose_model_class>
object_nunique = list(map(lambda col: train[col].nunique() , object_cols))
d = dict(zip(object_cols, object_nunique))
sorted(d.items() , key=lambda x: x[1] )
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))<choose_model_class>
for df in [train, test]:
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1',
'BsmtFinType2', 'Neighborhood', 'BldgType', 'HouseStyle', 'MasVnrType', 'FireplaceQu', 'Fence', 'PoolQC', 'MiscFeature', 'Alley'):
df[col] = df[col].fillna('None' )
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))<choose_model_class>
train['MasVnrArea'] = train['MasVnrArea'].fillna(0)
 test['MasVnrArea'] = test['MasVnrArea'].fillna(0 )
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5 )<choose_model_class>
missing = train.isnull().sum().sort_values(ascending=False)
 missing = missing[missing>0]
missing
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5 )<choose_model_class>
train['Electrical'] = train['Electrical'].fillna(train['Electrical'].mode() [0])
 test['Electrical'] = test['Electrical'].fillna(test['Electrical'].mode() [0] )
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, verbosity=0,
random_state =7, nthread = -1 )<choose_model_class>
train['LotFrontage'] = train.groupby('Neighborhood')['LotFrontage'].apply(lambda x: x.fillna(x.median()))
test['LotFrontage'] = test.groupby('Neighborhood')['LotFrontage'].apply(lambda x: x.fillna(x.median()))
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )<compute_test_metric>
missing = train.isnull().sum().sort_values(ascending=False)
 missing = missing[missing>0]
missing
score = rmsle_cv(lasso)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
missing = test.select_dtypes(include='object' ).isnull().sum().sort_values(ascending=False)
 missing = missing[missing>0]
missing
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
missing = test.select_dtypes(exclude='object' ).isnull().sum().sort_values(ascending=False)
 missing[missing>0]
score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
train['MSZoning'] = train.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
test['MSZoning'] = test.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
test = test.drop(['Utilities'], axis=1)
 train = train.drop(['Utilities'], axis=1 )
score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
for df in [train, test]:
for col in('Functional', 'SaleType', 'Exterior2nd', 'Exterior1st', 'KitchenQual'):
df[col] = df[col].fillna(df[col].mode() [0] )
score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<predict_on_test>
missing = test.select_dtypes(include='object' ).isnull().sum().sort_values(ascending=False)
 missing = missing[missing>0]
missing
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )<compute_train_metric>
for df in [train, test]:
for col in('MasVnrArea', 'BsmtFullBath', 'BsmtHalfBath', 'GarageCars', 'TotalBsmtSF',
'BsmtUnfSF', 'BsmtFinSF2', 'BsmtFinSF1'):
df[col] = df[col].fillna(0 )
averaged_score = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_score)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<train_model>
missing = test.select_dtypes(exclude='object' ).isnull().sum().sort_values(ascending=False)
 missing[missing>0]
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y.iloc[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )<compute_train_metric>
sf = [col for col in train.columns if 'SF' in col]
sf
stacked_averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR),
meta_model = lasso)
 score = rmsle_cv(stacked_averaged_models)
 print("Stacking Averaged models score: {:.4f}({:.4f})".format(score.mean() , score.std()))<compute_test_metric>
train['TotalSF'] = train['TotalBsmtSF'] + train['2ndFlrSF']
test['TotalSF'] = test['TotalBsmtSF'] + test['2ndFlrSF']
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))<predict_on_test>
bath = [col for col in train.columns if 'Bath' in col]
bath
stacked_averaged_models.fit(train.values, y_train)
 stacked_train_pred = stacked_averaged_models.predict(train.values)
 stacked_pred = np.expm1(stacked_averaged_models.predict(test.values))
print(rmsle(y_train, stacked_train_pred))<predict_on_test>
train['TotalBath'] = train['BsmtFullBath'] + train['BsmtHalfBath'] + train['FullBath'] + train['HalfBath']
test['TotalBath'] = test['BsmtFullBath'] + test['BsmtHalfBath'] + test['FullBath'] + test['HalfBath']
model_xgb.fit(train, y_train)
 xgb_train_pred = model_xgb.predict(train)
 xgb_pred = np.expm1(model_xgb.predict(test))
print(rmsle(y_train, xgb_train_pred))<predict_on_test>
porch = [col for col in train.columns if 'Porch' in col]
porch
model_lgb.fit(train, y_train)
 lgb_train_pred = model_lgb.predict(train)
 lgb_pred = np.expm1(model_lgb.predict(test.values))
print(rmsle(y_train, lgb_train_pred))<compute_test_metric>
train['PorchSF'] = train['OpenPorchSF'] + train['EnclosedPorch'] + train['3SsnPorch'] + train['ScreenPorch'] + train['WoodDeckSF']
test['PorchSF'] = test['OpenPorchSF'] + test['EnclosedPorch'] + test['3SsnPorch'] + test['ScreenPorch'] + test['WoodDeckSF']
print('RMSLE score on train data:')
 print(rmsle(y_train,stacked_train_pred*0.70 +
xgb_train_pred*0.15 + lgb_train_pred*0.15))<prepare_output>
pool = [col for col in train.columns if 'Pool' in col]
pool
Predictions = stacked_pred*0.70 + xgb_pred*0.15 + lgb_pred*0.15<save_to_csv>
train['HasBsmt'] = train['TotalBsmtSF'].apply(lambda x: 1 if x>0 else 0)
 train['HasPool'] = train['PoolArea'].apply(lambda x: 1 if x>0 else 0)
 train['Has2ndFlr'] = train['2ndFlrSF'].apply(lambda x: 1 if x>0 else 0)
 train['HasFirePlace'] = train['Fireplaces'].apply(lambda x: 1 if x>0 else 0)
 train['HasGarage'] = train['GarageCars'].apply(lambda x: 1 if x>0 else 0)
 test['HasBsmt'] = test['TotalBsmtSF'].apply(lambda x: 1 if x>0 else 0)
 test['HasPool'] = test['PoolArea'].apply(lambda x: 1 if x>0 else 0)
 test['Has2ndFlr'] = test['2ndFlrSF'].apply(lambda x: 1 if x>0 else 0)
 test['HasFirePlace'] = test['Fireplaces'].apply(lambda x: 1 if x>0 else 0)
 test['HasGarage'] = test['GarageCars'].apply(lambda x: 1 if x>0 else 0 )
subm = pd.DataFrame()
subm['Id'] = test_Ids
subm['SalePrice'] = Predictions
subm.to_csv('submission.csv',index=False )<set_options>
train['MasVnrArea'] = train['MasVnrArea'].astype(int)
 test['MasVnrArea'] = test['MasVnrArea'].astype(int )
warnings.filterwarnings('ignore')
 pd.set_option('display.float_format', lambda x: '%.3f' % x)
 %matplotlib inline 
<load_from_csv>
label_enc_variables = ['FireplaceQu', 'LotShape', 'OverallCond', 'ExterQual',
'ExterCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1',
'BsmtFinType2', 'HeatingQC', 'KitchenQual', 'GarageFinish', 'GarageQual',
'GarageCond', 'PavedDrive', 'PoolQC', 'Fence']
label_encoder = LabelEncoder()
for col in label_enc_variables:
train[col] = label_encoder.fit_transform(train[col])
 test[col] = label_encoder.transform(test[col] )
train = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv" )<concatenate>
X = train.drop(['SalePrice'], axis=1)
 y = train['SalePrice']
whole_df = pd.concat([train, test] )<prepare_x_and_y>
num_cols = [col for col in X.columns if X[col].dtype!='object']
X.update(X[num_cols].fillna(0))
test.update(test[num_cols].fillna(0))
cat_cols = [col for col in X.columns if X[col].dtype=='object']
X.update(X[cat_cols].fillna('None'))
test.update(test[cat_cols].fillna('None'))
train_salePrice = train.loc[:, 'SalePrice']
train_ID = train.loc[:, 'Id']
test_ID = test.loc[:, 'Id']<drop_column>
from sklearn.model_selection import train_test_split
from sklearn.impute import SimpleImputer
from xgboost import XGBRegressor
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from mlxtend.regressor import StackingCVRegressor
from sklearn.pipeline import make_pipeline
from lightgbm import LGBMRegressor
import lightgbm as lgb
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.linear_model import Ridge, RidgeCV, ElasticNetCV
from sklearn.preprocessing import RobustScaler
from sklearn.svm import SVR
whole_df.set_index('Id', drop = True, inplace = True )<drop_column>
X_train, X_valid, y_train, y_valid = train_test_split(X, y, train_size=0.8, test_size=0.2, random_state=2 )
whole_df.drop(['GarageArea', '1stFlrSF', 'TotRmsAbvGrd', 'GarageYrBlt', 'Utilities','MiscVal'], axis = 1, inplace = True )<sort_values>
numerical_cols = [col for col in X_train.columns
if X_train[col].dtype in ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']]
categorical_cols= [col for col in X_train.columns
if X_train[col].nunique() <=30 and X_train[col].dtype=='object']
final_cols = numerical_cols + categorical_cols
X_train = X_train[final_cols].copy()
X_valid = X_valid[final_cols].copy()
test = test[final_cols].copy()
total = whole_df.isnull().sum().sort_values(ascending = False)
 percent =(whole_df.isnull().sum() /whole_df.isnull().count() ).sort_values()
missing = pd.concat([total, percent], axis =1, keys = ['Total', 'Percent'])
 print('Total Missing Values')
 miss_cols = missing.iloc[:20].index
missing.iloc[:20] 
<feature_engineering>
X_train = pd.get_dummies(X_train)
 X_valid = pd.get_dummies(X_valid)
 test = pd.get_dummies(test)
 X_train, X_valid = X_train.align(X_valid, join='left', axis=1)
 X_train, test = X_train.align(test, join='left', axis=1 )
whole_df.loc[:, "Alley"] = whole_df.loc[:, "Alley"].fillna("None")
 whole_df.loc[:, "BedroomAbvGr"] = whole_df.loc[:, "BedroomAbvGr"].fillna(0)
 whole_df.loc[:, "BsmtQual"] = whole_df.loc[:, "BsmtQual"].fillna("None")
 whole_df.loc[:, "BsmtCond"] = whole_df.loc[:, "BsmtCond"].fillna("None")
 whole_df.loc[:, "BsmtExposure"] = whole_df.loc[:, "BsmtExposure"].fillna("None")
 whole_df.loc[:, "BsmtFinType1"] = whole_df.loc[:, "BsmtFinType1"].fillna("None")
 whole_df.loc[:, "BsmtFinType2"] = whole_df.loc[:, "BsmtFinType2"].fillna("None")
 whole_df.loc[:, "BsmtFullBath"] = whole_df.loc[:, "BsmtFullBath"].fillna(0)
 whole_df.loc[:, "BsmtHalfBath"] = whole_df.loc[:, "BsmtHalfBath"].fillna(0)
 whole_df.loc[:, "BsmtUnfSF"] = whole_df.loc[:, "BsmtUnfSF"].fillna(0)
 whole_df.loc[:, "CentralAir"] = whole_df.loc[:, "CentralAir"].fillna("N")
 whole_df.loc[:, "Condition1"] = whole_df.loc[:, "Condition1"].fillna("Norm")
 whole_df.loc[:, "Condition2"] = whole_df.loc[:, "Condition2"].fillna("Norm")
 whole_df.loc[:, "EnclosedPorch"] = whole_df.loc[:, "EnclosedPorch"].fillna(0)
 whole_df.loc[:, "ExterCond"] = whole_df.loc[:, "ExterCond"].fillna("TA")
 whole_df.loc[:, "ExterQual"] = whole_df.loc[:, "ExterQual"].fillna("TA")
 whole_df.loc[:, "Fence"] = whole_df.loc[:, "Fence"].fillna("None")
 whole_df.loc[:, "FireplaceQu"] = whole_df.loc[:, "FireplaceQu"].fillna("None")
 whole_df.loc[:, "Fireplaces"] = whole_df.loc[:, "Fireplaces"].fillna(0)
 whole_df.loc[:, "Functional"] = whole_df.loc[:, "Functional"].fillna("Typ")
 whole_df.loc[:, "GarageType"] = whole_df.loc[:, "GarageType"].fillna("None")
 whole_df.loc[:, "GarageFinish"] = whole_df.loc[:, "GarageFinish"].fillna("None")
 whole_df.loc[:, "GarageQual"] = whole_df.loc[:, "GarageQual"].fillna("None")
 whole_df.loc[:, "GarageCond"] = whole_df.loc[:, "GarageCond"].fillna("None")
 whole_df.loc[:, "GarageCars"] = whole_df.loc[:, "GarageCars"].fillna(0)
 whole_df.loc[:, "HalfBath"] = whole_df.loc[:, "HalfBath"].fillna(0)
 whole_df.loc[:, "HeatingQC"] = whole_df.loc[:, "HeatingQC"].fillna("TA")
 whole_df.loc[:, "KitchenAbvGr"] = whole_df.loc[:, "KitchenAbvGr"].fillna(0)
 whole_df.loc[:, "KitchenQual"] = whole_df.loc[:, "KitchenQual"].fillna("TA")
 whole_df.loc[:, "LotFrontage"] = whole_df.loc[:, "LotFrontage"].fillna(0)
 whole_df.loc[:, "LotShape"] = whole_df.loc[:, "LotShape"].fillna("Reg")
 whole_df.loc[:, "MasVnrType"] = whole_df.loc[:, "MasVnrType"].fillna("None")
 whole_df.loc[:, "MasVnrArea"] = whole_df.loc[:, "MasVnrArea"].fillna(0)
 whole_df.loc[:, "MiscFeature"] = whole_df.loc[:, "MiscFeature"].fillna("None")
 whole_df.loc[:, "MSZoning"] = whole_df.loc[:, "MSZoning"].fillna(whole_df.loc[:, "MSZoning"].mode() [0])
 whole_df.loc[:, "OpenPorchSF"] = whole_df.loc[:, "OpenPorchSF"].fillna(0)
 whole_df.loc[:, "PavedDrive"] = whole_df.loc[:, "PavedDrive"].fillna("N")
 whole_df.loc[:, "PoolQC"] = whole_df.loc[:, "PoolQC"].fillna("None")
 whole_df.loc[:, "PoolArea"] = whole_df.loc[:, "PoolArea"].fillna(0)
 whole_df.loc[:, "SaleCondition"] = whole_df.loc[:, "SaleCondition"].fillna("Normal")
 whole_df.loc[:, "ScreenPorch"] = whole_df.loc[:, "ScreenPorch"].fillna(0)
 whole_df.loc[:, "WoodDeckSF"] = whole_df.loc[:, "WoodDeckSF"].fillna(0 )<drop_column>
estimators = [2000, 2500, 3000, 3500]
for n in estimators:
my_dict = dict()
model = XGBRegressor(n_estimators=n, learning_rate=0.01, colsample_bytree=0.45, max_depth=3)
 model.fit(X_train, y_train)
 preds = model.predict(X_valid)
 mae = mean_absolute_error(rev_y(preds), rev_y(y_valid))
my_dict[f"n_est {n}"] = mae
print(my_dict )
whole_df.drop('SalePrice', axis = 1, inplace = True )<sort_values>
my_dict = dict()
n=3500
model = XGBRegressor(n_estimators=n, learning_rate=0.01, colsample_bytree=0.45, max_depth=3,
gamma=0, subsample=0.4, reg_alpha=0, reg_lambda=1, objective='reg:squarederror')
 model.fit(X_train, y_train)
 preds = model.predict(X_valid)
 mae = mean_absolute_error(rev_y(preds), rev_y(y_valid))
my_dict[f"n_est {n}"] = mae
print(my_dict )
total = whole_df.isnull().sum().sort_values(ascending = False)
 percent =(whole_df.isnull().sum() /whole_df.isnull().count() ).sort_values()
missing = pd.concat([total, percent], axis = 1, keys = ['Total', 'Percent'])
 print('Total Missing Values')
 miss_cols = missing.iloc[:20].index
missing.iloc[:20]<feature_engineering>
n = 3500
my_dict = dict()
xgb = XGBRegressor(n_estimators=n,
learning_rate=0.01,
colsample_bytree=0.45,
max_depth=3,
gamma=0,
subsample=0.4,
reg_alpha=0,
reg_lambda=1,
objective='reg:squarederror')
 lgb = LGBMRegressor(objective='regression',
num_leaves=6,
learning_rate=0.01,
n_estimators=n,
max_bin=200,
bagging_fraction=0.8,
bagging_freq=4,
bagging_seed=8,
feature_fraction=0.2,
feature_fraction_seed=8,
min_sum_hessian_in_leaf = 11,
verbose=-1,
random_state=42)
 ridge_alphas = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=ridge_alphas))
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
elasticnet = make_pipeline(RobustScaler() , ElasticNetCV(max_iter=1e7, alphas=e_alphas, l1_ratio=e_l1ratio))
svr = make_pipeline(RobustScaler() , SVR(C= 20, epsilon= 0.008, gamma=0.0003))
gbr = GradientBoostingRegressor(n_estimators=n,
learning_rate=0.01,
max_depth=4,
max_features='sqrt',
min_samples_leaf=15,
min_samples_split=10,
loss='huber',
random_state=42)
 stack = StackingCVRegressor(regressors=(ridge, gbr, xgb, elasticnet, svr, lgb),
meta_regressor=xgb,
use_features_in_secondary=True )
for i in miss_cols[:7]:
whole_df.loc[:, i] = whole_df.loc[:, i].fillna(whole_df.loc[:, i].mode() [0] )<sort_values>
Scores = {}
total = whole_df.isnull().sum().sort_values(ascending = False)
 percent =(whole_df.isnull().sum() /whole_df.isnull().count() ).sort_values()
missing = pd.concat([total, percent], axis = 1, keys = ['Total', 'Percent'])
 print('Total Missing Values')
 missing.iloc[:20] 
<count_missing_values>
def rmse(model, X, y):
scores = np.sqrt(-1 * cross_val_score(model, X, y,
cv=10,
scoring='neg_mean_squared_error'))
return scores
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
whole_df.isnull().values.any()<define_variables>
scores = rmse(xgb, X_train, y_train)
 print("XGBoost Regressor
")
 print("Root Mean Square Error(RMSE)", str(scores.mean()))
print("Error Standard Deviation", str(scores.std()))
Scores['XGB'] = scores.mean()
ord_fields = ['MSSubClass','ExterQual','LotShape','BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1',
'BsmtFinType2','HeatingQC','Functional','FireplaceQu','KitchenQual', 'GarageFinish',
'GarageQual','GarageCond','PoolQC','Fence']
orders = [
['20','30','40','45','50','60','70','75','80','85', '90','120','150','160','180','190'],
['Fa','TA','Gd','Ex'],
['Reg','IR1' ,'IR2','IR3'],
['None','Fa','TA','Gd','Ex'],
['None','Po','Fa','TA','Gd','Ex'],
['None','No','Mn','Av','Gd'],
['None','Unf','LwQ', 'Rec','BLQ','ALQ' , 'GLQ' ],
['None','Unf','LwQ', 'Rec','BLQ','ALQ' , 'GLQ' ],
['Po','Fa','TA','Gd','Ex'],
['Sev','Maj2','Maj1','Mod','Min2','Min1','Typ'],
['None','Po','Fa','TA','Gd','Ex'],
['Fa','TA','Gd','Ex'],
['None','Unf','RFn','Fin'],
['None','Po','Fa','TA','Gd','Ex'],
['None','Po','Fa','TA','Gd','Ex'],
['None','Fa','Gd','Ex'],
['None','MnWw','GdWo','MnPrv','GdPrv']]
for i in range(len(orders)) :
ord_en = OrdinalEncoder(categories = {0 : orders[i]})
 whole_df.loc[:, ord_fields[i]] = ord_en.fit_transform(whole_df.loc[:, ord_fields[i]].values.reshape(-1,1))<count_unique_values>
scores = rmse(lgb, X_train, y_train)
 print("Light Gradient Boosting Regressor
")
 print("Root Mean Square Error(RMSE)", str(scores.mean()))
print("Error Standard Deviation", str(scores.std()))
Scores['LGB'] = scores.mean()
cat_df.nunique()<categorify>
scores = rmse(ridge, X_train, y_train)
 print("Ridge Regressor
")
 print("Root Mean Square Error(RMSE)", str(scores.mean()))
print("Error Standard Deviation", str(scores.std()))
Scores['Ridge'] = scores.mean()
cat_cols = cat_df.columns
for one in cat_cols:
lab_enc = LabelEncoder()
lab_enc.fit(list(whole_df[one].values))
whole_df[one] = lab_enc.transform(list(whole_df[one].values))<prepare_x_and_y>
scores = rmse(svr, X_train, y_train)
 print("Support Vector Regressor
")
 print("Root Mean Square Error(RMSE)", str(scores.mean()))
print("Error Standard Deviation", str(scores.std()))
Scores['SVR'] = scores.mean()
lam = 0.15
train_log_SP = boxcox1p(train_salePrice, lam )<feature_engineering>
scores = rmse(gbr, X_train, y_train)
 print("Gradient Boosting Regressor
")
 print("Root Mean Square Error(RMSE)", str(scores.mean()))
print("Error Standard Deviation", str(scores.std()))
Scores['GBR'] = scores.mean()
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
for feat in skewed_features:
whole_df[feat] = boxcox1p(whole_df[feat], lam )<define_variables>
scores = rmse(elasticnet, X_train, y_train)
 print("ElasticNet Regressor
")
 print("Root Mean Square Error(RMSE)", str(scores.mean()))
print("Error Standard Deviation", str(scores.std()))
Scores['ElasticNet'] = scores.mean()
ntrain = len(train.index)
 ntrain<prepare_x_and_y>
xgb.fit(X_train, y_train )
train = whole_df.iloc[ : ntrain]
test = whole_df.iloc[ntrain : ]
y = train_log_SP
X = train<import_modules>
lgb.fit(X_train, y_train )
from sklearn.model_selection import train_test_split<import_modules>
ridge.fit(X_train, y_train )
from sklearn.model_selection import train_test_split<split>
svr.fit(X_train, y_train )
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state = 0 )<import_modules>
gbr.fit(X_train, y_train )
from sklearn.linear_model import LinearRegression<choose_model_class>
elasticnet.fit(X_train, y_train )
lm = LinearRegression()<train_model>
stack.fit(np.array(X_train), np.array(y_train))
lm.fit(X_train, y_train )<predict_on_test>
def blended_predictions(X):
return(( 0.2 * ridge.predict(X)) + \ 
(0.2 * elasticnet.predict(X)) + \ 
(0.05 * svr.predict(X)) + \ 
(0.1 * gbr.predict(X)) + \ 
(0.1 * xgb.predict(X)) + \ 
(0.1 * lgb.predict(X)) + \ 
(0.25 * stack.predict(np.array(X))))
predictions = lm.predict(X_test )<compute_test_metric>
X_valid = X_valid.fillna(0)
 test = test.fillna(0 )
print('RMSE: ', np.sqrt(metrics.mean_squared_error(y_test,predictions)) )<compute_test_metric>
blended_score = rmsle(y_valid, blended_predictions(X_valid))
Scores['Blended'] = blended_score
blended_score
metrics.explained_variance_score(y_test,predictions )<choose_model_class>
submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.shape
lm_test = LinearRegression()<train_model>
submission.iloc[:,1] = np.floor(rev_y(blended_predictions(test)) )
lm_test.fit(X,y )<predict_on_test>
q1 = submission['SalePrice'].quantile(0.0045)
 q2 = submission['SalePrice'].quantile(0.99)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x > q1 else x*0.77)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x < q2 else x*1.1 )
test_predictions = lm_test.predict(test )<compute_test_metric>
submission['SalePrice'] *= 1.001619
submission.to_csv("new_submission.csv", index=False )