Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<filter>
color = sns.color_palette()
sns.set_style('darkgrid')
 def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
pd.set_option('display.float_format', lambda x: '{:.3f}'.format(x))
pd.options.display.max_rows = 80
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train_ID = train['Id']
test_ID = test['Id']
train.drop("Id", axis = 1, inplace = True)
 test.drop("Id", axis = 1, inplace = True )
min(dataset.GarageYrBlt[dataset["GarageYrBlt"] != 'NA'].values )<feature_engineering>
train["SalePrice"] = transform_log
bins = [1890, 1920, 1940, 1950, 1960, 1970, 1980, 1990, 2000, 2011]
names = ['-1920', '1920-1940', '1940-1950', '1950-1960', '1960-1970',
'1970-1980', '1980-1990', '1990-2000', '2000+']
dataset.GarageYrBlt[dataset["GarageYrBlt"] != 'NA'] = pd.cut(dataset.GarageYrBlt[dataset["GarageYrBlt"] != 'NA'],
bins,
labels=names )<data_type_conversions>
train = train.drop(train[(train['GrLivArea']>4000)&(np.expm1(train['SalePrice'])<300000)].index )
dataset.GarageYrBlt = dataset.GarageYrBlt.astype(str )<categorify>
all_data = pd.concat(( train, test)).reset_index(drop=True )
bins = [1949, 1960, 1970, 1980, 1990, 2000, 2011]
names = ['1950-1960', '1960-1970',
'1970-1980', '1980-1990', '1990-2000', '2000+']
dataset.YearRemodAdd = pd.cut(dataset.YearRemodAdd, bins, labels=names )<categorify>
temp = all_data.copy()
temp.drop(['SalePrice'], axis=1, inplace=True)
 all_data_na = temp.isnull().sum()
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)
 all_data_missing = pd.DataFrame({'Missing Numbers' :all_data_na})
 all_data_missing
dataset.YearRemodAdd = dataset.YearRemodAdd.astype(str )<data_type_conversions>
all_data['SaleType'].value_counts()
dataset.YearBuilt = dataset.YearBuilt.astype(str )<data_type_conversions>
all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0])
 all_data['SaleType'].value_counts()
dataset.YrSold = dataset.YrSold.astype(str )<data_type_conversions>
all_data['KitchenQual'].value_counts()
dataset.MoSold = dataset.MoSold.astype(str )<define_variables>
all_data['KitchenQual'] = all_data['KitchenQual'].fillna(all_data['KitchenQual'].mode() [0])
 all_data['KitchenQual'].value_counts()
num_continuous = ['LotFrontage','LotArea','MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF',
'1stFlrSF','2ndFlrSF','LowQualFinSF','GrLivArea','GarageArea','WoodDeckSF','OpenPorchSF',
'EnclosedPorch','3SsnPorch','ScreenPorch','PoolArea','MiscVal']<drop_column>
all_data['Exterior2nd'].value_counts()
dataset.LotFrontage = boxcox1p(dataset.LotFrontage, 0.35 )<feature_engineering>
all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0])
 all_data['Exterior2nd'].value_counts()
dataset.LotArea = boxcox1p(dataset.LotArea, 0.15 )<concatenate>
all_data['Exterior1st'].value_counts()
dataset.MasVnrArea = boxcox1p(dataset.MasVnrArea, 0.15 )<data_type_conversions>
all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Exterior1st'].value_counts()
dataset.BsmtFinSF1 = dataset.BsmtFinSF1.astype(str )<feature_engineering>
all_data['Electrical'].value_counts()
dataset.BsmtFinSF2 = dataset.BsmtFinSF2.apply(lambda x: 1 if x > 0 else 0 )<data_type_conversions>
all_data['Electrical'] = all_data['Electrical'].fillna(all_data['Electrical'].mode() [0])
 all_data['Electrical'].value_counts()
dataset.BsmtUnfSF = dataset.BsmtUnfSF.astype(str )<feature_engineering>
all_data['Functional'].value_counts()
dataset["FlrSF"] = dataset["1stFlrSF"] + dataset["2ndFlrSF"] + dataset["TotalBsmtSF"]<feature_engineering>
all_data['Functional'] = all_data['Functional'].fillna(all_data['Functional'].mode() [0])
 all_data['Functional'].value_counts()
dataset.FlrSF = boxcox1p(dataset.FlrSF, 0.35)
 dataset["1stFlrSF"] = boxcox1p(dataset["1stFlrSF"], 0.15 )<data_type_conversions>
all_data['Utilities'].value_counts()
dataset['2ndFlrSF'] = dataset['2ndFlrSF'].astype(str )<feature_engineering>
test['Utilities'].value_counts()
dataset.LowQualFinSF = dataset.LowQualFinSF.apply(lambda x: 1 if x > 0 else 0 )<concatenate>
all_data.drop(['Utilities'], axis=1, inplace=True )
dataset.GrLivArea = boxcox1p(dataset.GrLivArea,0.15 )<concatenate>
all_data['MSZoning'].value_counts()
dataset.WoodDeckSF = boxcox1p(dataset.WoodDeckSF,0.15 )<drop_column>
all_data['MSZoning']=all_data['MSZoning'].fillna(all_data['MSZoning'].mode() [0])
 all_data['MSZoning'].value_counts()
dataset["PorchSF"] = dataset["OpenPorchSF"] + dataset["EnclosedPorch"] + dataset["3SsnPorch"] + dataset["ScreenPorch"]
dataset = dataset.drop(columns=["OpenPorchSF","EnclosedPorch","3SsnPorch","ScreenPorch"] )<concatenate>
all_data['MasVnrType'].value_counts()
dataset.PorchSF = boxcox1p(dataset.PorchSF, 0.35 )<drop_column>
all_data['MasVnrType']=all_data['MasVnrType'].fillna('None')
 all_data['MasVnrType'].value_counts()
dataset = dataset.drop(columns=["MiscVal"] )<define_variables>
all_data['BsmtFinType1'].value_counts()
cat_ordinal = ['LotShape','Utilities','LandSlope','OverallQual','OverallCond','ExterQual','ExterCond',
'BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','HeatingQC','CentralAir',
'Electrical','KitchenQual','Functional','FireplaceQu','GarageFinish','GarageQual','GarageCond',
'PavedDrive','PoolQC','Fence']<define_variables>
all_data['BsmtFinType1']=all_data['BsmtFinType1'].fillna('None')
 all_data['BsmtFinType1'].value_counts()
dict_ = {"PoolQC": {"NA": 0, "Fa": 1, "TA": 2, "Gd": 3, "Ex": 4},
"Fence": {"NA": 0, "MnWw": 1, "GdWo": 2, "MnPrv": 3, "GdPrv": 4},
"FireplaceQu": {"NA": 0, "Po": 1, "Fa": 2, "TA": 3, "Gd": 4, "Ex": 5},
"GarageCond": {"NA": 0, "Po": 1, "Fa": 2, "TA": 3, "Gd": 4, "Ex": 5},
"GarageQual": {"NA": 0, "Po": 1, "Fa": 2, "TA": 3, "Gd": 4, "Ex": 5},
"GarageFinish": {"NA": 0, "Unf": 1, "RFn": 2, "Fin": 3},
"BsmtExposure": {"NA": 0, "No": 1, "Mn": 2, "Av": 3, "Gd": 4},
"BsmtCond": {"NA": 0, "Po": 1, "Fa": 2, "TA": 3, "Gd": 4, "Ex": 5},
"BsmtQual": {"NA": 0, "Po": 1, "Fa": 2, "TA": 3, "Gd": 4, "Ex": 5},
"BsmtFinType2": {"NA": 0, "Unf": 1, "LwQ": 2, "Rec": 3, "BLQ": 4, "ALQ": 5, "GLQ": 6},
"BsmtFinType1": {"NA": 0, "Unf": 1, "LwQ": 2, "Rec": 3, "BLQ": 4, "ALQ": 5, "GLQ": 6},
"Electrical": {"Mix": 0, "FuseP": 1, "FuseF": 2, "FuseA": 3, "SBrkr": 4},
"Functional": {"Sal": 0, "Sev": 1, "Maj2": 2, "Maj1": 3, "Mod": 4, "Min2": 5, "Min1": 6, "Typ": 7},
"KitchenQual": {"Po": 0, "Fa": 1, "TA": 2, "Gd": 3, "Ex": 4},
"Utilities": {"ELO": 0, "NoSeWa": 1, "NoSewr": 2, "AllPub": 3},
"LotShape": {"IR3": 0, "IR2": 1, "IR1": 2, "Reg": 3},
"LandSlope": {"Sev": 0, "Mod": 1, "Gtl": 2},
"ExterQual": {"Po": 0, "Fa": 1, "TA": 2, "Gd": 3, "Ex": 4},
"ExterCond": {"Po": 0, "Fa": 1, "TA": 2, "Gd": 3, "Ex": 4},
"HeatingQC": {"Po": 0, "Fa": 1, "TA": 2, "Gd": 3, "Ex": 4},
"CentralAir": {"N": 0, "Y": 1},
"FireplaceQu": {"NA": 0, "Po": 1, "Fa": 2, "TA": 3, "Gd": 4, "Ex": 5},
"PavedDrive": {"N": 0, "P": 1, "Y": 2}}
dataset.replace(dict_, inplace=True )<data_type_conversions>
all_data['BsmtFinType2'].value_counts()
dataset.BsmtCond = dataset.BsmtCond.astype(int )<feature_engineering>
all_data['BsmtFinType2']=all_data['BsmtFinType2'].fillna('None')
 all_data['BsmtFinType2'].value_counts()
dataset.LotShape = dataset.LotShape.apply(lambda x: 1 if x == 0 else x )<drop_column>
all_data['BsmtQual'].value_counts()
dataset = dataset.drop(columns='Utilities' )<feature_engineering>
all_data['BsmtQual']=all_data['BsmtQual'].fillna('None')
 all_data['BsmtQual'].value_counts()
dataset.LandSlope = dataset.LandSlope.apply(lambda x: 1 if x == 2 else 0 )<feature_engineering>
all_data['BsmtCond'].value_counts()
dataset.OverallQual = dataset.OverallQual.apply(lambda x: 2 if x == 1 else x )<choose_model_class>
all_data['BsmtCond']=all_data['BsmtCond'].fillna('None')
 all_data['BsmtCond'].value_counts()
dataset.OverallCond = dataset.OverallCond.apply(lambda x: 3 if x < 3 else x )<feature_engineering>
all_data['BsmtExposure'].value_counts()
dataset.ExterCond = dataset.ExterCond.apply(lambda x: 0 if x == 1 else x)
 dataset.ExterCond = dataset.ExterCond.apply(lambda x: 1 if x >= 2 else x )<feature_engineering>
all_data['BsmtExposure']=all_data['BsmtExposure'].fillna('None')
 all_data['BsmtExposure'].value_counts()
dataset.BsmtCond = dataset.BsmtCond.apply(lambda x: 0 if x == 1 else x )<data_type_conversions>
all_data['GarageType'].value_counts()
dataset.BsmtFinType1 = dataset.BsmtFinType1.astype(str )<data_type_conversions>
all_data['GarageType']=all_data['GarageType'].fillna('None')
 all_data['GarageType'].value_counts()
dataset.BsmtFinType2 = dataset.BsmtFinType2.astype(str )<feature_engineering>
all_data['GarageYrBlt'][all_data['GarageYrBlt']>2150]
dataset.HeatingQC = dataset.HeatingQC.apply(lambda x: 1 if x == 0 else x )<feature_engineering>
all_data['GarageFinish'].value_counts()
dataset.Electrical = dataset.Electrical.apply(lambda x: 2 if x < 2 else x )<feature_engineering>
all_data['GarageFinish']=all_data['GarageFinish'].fillna('None')
 all_data['GarageFinish'].value_counts()
dataset.Functional = dataset.Functional.apply(lambda x: 2 if x < 2 else x )<data_type_conversions>
all_data['GarageCond'].value_counts()
dataset.Functional = dataset.Functional.astype(str )<feature_engineering>
all_data['GarageCond']=all_data['GarageCond'].fillna('None')
 all_data['GarageCond'].value_counts()
dataset.GarageQual = dataset.GarageQual.apply(lambda x: 0 if x == 1 else x)
 dataset.GarageQual = dataset.GarageQual.apply(lambda x: 4 if x == 5 else x )<feature_engineering>
all_data['GarageQual'].value_counts()
dataset.GarageCond = dataset.GarageCond.apply(lambda x: 0 if x == 1 else x)
 dataset.GarageCond = dataset.GarageCond.apply(lambda x: 3 if x > 3 else x )<data_type_conversions>
all_data['GarageQual']=all_data['GarageQual'].fillna('None')
 all_data['GarageQual'].value_counts()
dataset.GarageCond = dataset.GarageCond.astype(str )<drop_column>
all_data['FireplaceQu'].value_counts()
dataset = dataset.drop(columns='PoolQC' )<data_type_conversions>
all_data['FireplaceQu']=all_data['FireplaceQu'].fillna('None')
 all_data['FireplaceQu'].value_counts()
dataset.Fence = dataset.Fence.astype(str )<define_variables>
all_data['Fence'].value_counts()
cat_nominal = ['MSSubClass','MSZoning','Street','Alley','LandContour','LotConfig','Neighborhood',
'Condition1','Condition2','BldgType','HouseStyle','RoofStyle','RoofMatl','Exterior1st',
'Exterior2nd','MasVnrType','Foundation','Heating','GarageType','MiscFeature',
'SaleType','SaleCondition']<data_type_conversions>
all_data['Fence']=all_data['Fence'].fillna('None')
 all_data['Fence'].value_counts()
dataset.MSSubClass = dataset.MSSubClass.astype(str )<drop_column>
all_data['Alley'].value_counts()
dataset = dataset.drop(columns='Street' )<feature_engineering>
all_data['Alley']=all_data['Alley'].fillna('None')
 all_data['Alley'].value_counts()
dataset.LotConfig = dataset.LotConfig.apply(lambda x: 'FR2' if x == 'FR3' else x )<feature_engineering>
all_data['MiscFeature'].value_counts()
dataset.Condition1 = dataset.Condition1.apply(lambda x: 'RRs' if(x == 'RRNe' or x == 'RRAe' or x == 'RRNn' or x == 'RRAn')else x )<drop_column>
all_data['MiscFeature']=all_data['MiscFeature'].fillna('None')
 all_data['MiscFeature'].value_counts()
dataset = dataset.drop(columns='Condition2' )<set_options>
all_data['PoolQC'].value_counts()
dataset.RoofStyle = dataset.RoofStyle.apply(lambda x: 'Other' if(x == 'Gambrel' or x == 'Mansard' or x == 'Flat' or x == 'Shed')else x )<drop_column>
all_data['PoolQC']=all_data['PoolQC'].fillna('None')
 all_data['PoolQC'].value_counts()
dataset = dataset.drop(columns='RoofMatl' )<drop_column>
all_data['MSSubClass'] = all_data['MSSubClass'].astype(str)
 all_data['OverallCond'] = all_data['OverallCond'].astype(str)
 all_data['BsmtFullBath'] = all_data['BsmtFullBath'].astype(str)
 all_data['BsmtHalfBath'] = all_data['BsmtHalfBath'].astype(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str)
 all_data['GarageYrBlt'] = all_data['GarageYrBlt'].astype(str)
 all_data['YearBuilt'] = all_data['YearBuilt'].astype(str)
 all_data['YearRemodAdd'] = all_data['YearRemodAdd'].astype(str)
 all_data['GarageCars'] = all_data['GarageCars'].astype(str )
dataset.Exterior1st = dataset.Exterior1st.apply(lambda x: 'Other' if(x == 'BrkComm' or x == 'AsphShn' or x == 'Stone' or x == 'ImStucc' or x== 'CBlock')else x )<drop_column>
all_data['TotalSF']=all_data['TotalBsmtSF']+all_data['1stFlrSF']+all_data['2ndFlrSF']
dataset = dataset.drop(columns = 'Exterior2nd' )<feature_engineering>
cols =('ExterCond','HeatingQC', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold','GarageYrBlt','YearBuilt','YearRemodAdd', 'BsmtHalfBath','BsmtFullBath', 'GarageCars')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[c].values))
all_data[c] = lbl.transform(list(all_data[c].values))
dataset.MiscFeature = dataset.MiscFeature.apply(lambda x: 0 if x == 'NA' else 1 )<feature_engineering>
all_data.FireplaceQu = all_data.FireplaceQu.astype('category', ordered=True, categories=['None','Po','Fa','TA','Gd','Ex'] ).cat.codes
all_data.BsmtQual = all_data.BsmtQual.astype('category', ordered=True, categories=['None','Fa','TA','Gd','Ex'] ).cat.codes
all_data.BsmtCond = all_data.BsmtCond.astype('category', ordered=True, categories=['None','Po','Fa','TA','Gd'] ).cat.codes
all_data.GarageQual = all_data.GarageQual.astype('category', ordered=True, categories=['None','Po','Fa','TA','Gd','Ex'] ).cat.codes
all_data.GarageCond = all_data.GarageCond.astype('category', ordered=True, categories=['None','Po','Fa','TA','Gd','Ex'] ).cat.codes
all_data.ExterQual = all_data.ExterQual.astype('category', ordered=True, categories=['Fa','TA','Gd','Ex'] ).cat.codes
all_data.PoolQC = all_data.PoolQC.astype('category', ordered=True, categories=['None','Fa','Gd','Ex'] ).cat.codes
all_data.KitchenQual = all_data.KitchenQual.astype('category', ordered=True, categories=['Fa','TA','Gd','Ex'] ).cat.codes
dataset.SaleType = dataset.SaleType.apply(lambda x: x if(x == 'WD' or x == 'New')else 'Other' )<feature_engineering>
for feat in skewness[abs(skewness)>0.5].index:
all_data[feat] = boxcox1p(all_data[feat], 0.15 )
dataset.SaleCondition = dataset.SaleCondition.apply(lambda x: x if(x == 'Normal' or x == 'Abnormal' or x == 'Partial')else 'Other' )<prepare_x_and_y>
all_data = pd.get_dummies(all_data )
X = dataset.loc[:,dataset.columns.difference(['SalePrice', 'Id'])]
Y = dataset.SalePrice
X = pd.get_dummies(X )<normalization>
new_train = all_data[all_data['SalePrice'].notnull() ]
new_test=all_data[all_data['SalePrice'].isnull() ]
new_test.drop('SalePrice', axis=1, inplace=True )
scaler = RobustScaler()
X = pd.DataFrame(scaler.fit_transform(X))<prepare_x_and_y>
y_train = new_train.SalePrice.values
new_train.drop('SalePrice', axis=1, inplace=True )
X_train = X.iloc[0:1457].as_matrix()
Y_train = Y.iloc[0:1457].as_matrix()
X_test = X.iloc[1457:].as_matrix()
test_ids = dataset.Id.iloc[1457:]<choose_model_class>
nfold = 5
def cv_score(model):
kf = KFold(nfold, shuffle=True, random_state=42 ).get_n_splits(new_train.values)
 return np.sqrt(-cross_val_score(model, new_train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
lasso = Lasso(alpha= 0.0005 )<choose_model_class>
class AveragedModel(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models, weights):
self.models = models
self.weights = weights
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.dot(predictions,self.weights)
 class StackedAveragedModel(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, nfold=5):
self.base_models = base_models
self.meta_model = meta_model
self.nfold = nfold
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.nfold, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )
elastic = ElasticNet(alpha=0.0005, l1_ratio=.9 )<define_search_space>
flag=True
if flag:
steps = [('scaler',RobustScaler()),('select',SelectFromModel(Lasso(alpha =5e-4, random_state=1,selection='cyclic'))), 
('lasso', Lasso(alpha =5e-4, random_state=1,selection='cyclic')) ]
lasso_p = Pipeline(steps)
 alphas=(np.linspace(1e-4,1e-3,num=10))
selections=['random','cyclic']
gscv = GridSearchCV(lasso_p, cv=nfold, param_grid={'lasso__alpha': alphas, 'lasso__selection': selections}, n_jobs=1, verbose=1)
 gscv.fit(new_train.values, y_train)
 lasso_ = gscv.best_estimator_.named_steps.lasso
score = cv_score(lasso_)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
print('Best Lasso: ',lasso_)
 lasso = make_pipeline(RobustScaler() ,SelectFromModel(lasso_), lasso_)
 steps = [('scaler',RobustScaler()),('select',SelectFromModel(ElasticNet(alpha=1e-3,
l1_ratio=5e-1, random_state=3, selection='random'))),('enet', ElasticNet(alpha=1e-3,
l1_ratio=5e-1, random_state=3, selection='random')) ]
enet_p = Pipeline(steps)
 alphas=np.linspace(1e-3,1e-2,num=10)
 l1_ratios=np.linspace(0,1,num=11)
 selections=['random','cyclic']
gscv = GridSearchCV(enet_p, cv=nfold,
param_grid={'enet__alpha': alphas, 'enet__l1_ratio': l1_ratios, 'enet__selection':selections }, n_jobs=1, verbose=1)
 gscv.fit(new_train.values, y_train)
 enet_ = gscv.best_estimator_.named_steps.enet
score = cv_score(enet_)
 print("
ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
print('Best ElasticNet: ',enet_)
 enet = make_pipeline(RobustScaler() ,SelectFromModel(enet_), enet_)
 steps = [('scaler',RobustScaler()),('select',SelectFromModel(lasso_)) ,('krr',KernelRidge(alpha=6e-1,
kernel='polynomial', degree=2, coef0=4)) ]
krr_p = Pipeline(steps)
 alphas=np.linspace(1e-1,1,num=10)
 degrees=[1,2,3]
coef0s = [1,2,4,8]
gscv = GridSearchCV(krr_p, cv=nfold,
param_grid={'krr__alpha': alphas, 'krr__degree': degrees, 'krr__coef0':coef0s }, n_jobs=1, verbose=1)
 gscv.fit(new_train.values, y_train)
 krr_ = gscv.best_estimator_.named_steps.krr
score = cv_score(krr_)
 print("
Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
print('Best Kernel Ridge: ',krr_)
 krr = make_pipeline(RobustScaler() ,SelectFromModel(lasso_), krr_)
 steps = [('scaler',RobustScaler()),('select',SelectFromModel(lasso_)) ,('gboost',GradientBoostingRegressor(n_estimators=2000,
learning_rate=2e-2, max_depth=5, max_features='log2',
min_samples_leaf=5, min_samples_split=15, loss='huber', random_state =5)) ]
gboost_p = Pipeline(steps)
 n_estimatorss=[1000,2000,3000]
learning_rates=np.linspace(1e-2,1e-1,num=10)
 max_depths=[1,2,3,4,5]
max_featuress=['sqrt','log2']
min_samples_leafs=[5,10,15,20]
min_samples_splits=[5,10,15,20]
rscv = RandomizedSearchCV(gboost_p, cv=nfold,
param_distributions={'gboost__n_estimators': n_estimatorss, 'gboost__learning_rate': learning_rates, 'gboost__max_depth': max_depths,
'gboost__min_samples_leaf': min_samples_leafs, 'gboost__min_samples_split': min_samples_splits, 'gboost__max_features': max_featuress},
n_jobs=1, verbose=1, random_state=7)
 rscv.fit(new_train.values, y_train)
 gboost_ = rscv.best_estimator_.named_steps.gboost
score = cv_score(gboost_)
 print("
Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
print('Best Gradient Boosting: ',gboost_)
 gboost = make_pipeline(RobustScaler() ,SelectFromModel(lasso_), gboost_)
 steps = [('scaler',RobustScaler()),('select',SelectFromModel(lasso_)) ,('xgboost', xgb.XGBRegressor(colsample_bytree=6e-1, gamma=1e-2,
learning_rate=4e-2, max_depth=3, min_child_weight=1.5, n_estimators=1000, reg_alpha=4e-1, reg_lambda=9e-1, subsample=4e-1,
silent=1, random_state =7,n_jobs = 1)) ]
xgboost_p = Pipeline(steps)
 n_estimatorss=[1000,2000,3000]
learning_rates=np.linspace(1e-2,1e-1,num=10)
 max_depths=[1,2,3,4,5]
gammas=np.linspace(1e-2,1e-1,num=10)
 min_child_weights=[1.5,1.6,1.7,1.8,1.9]
colsample_bytrees=np.linspace(1e-1,1,num=10)
 reg_alphas=[0.3,0.4,0.5]
reg_lambdas=[0.7,0.8,0.9]
subsamples=[0.3,0.4,0.5,0.6,0.7]
rscv = RandomizedSearchCV(xgboost_p, cv=nfold,
param_distributions={'xgboost__n_estimators': n_estimatorss, 'xgboost__learning_rate': learning_rates,
'xgboost__max_depth': max_depths, 'xgboost__gamma': gammas, 'xgboost__min_child_weight': min_child_weights,
'xgboost__colsample_bytree': colsample_bytrees, 'xgboost__reg_alpha' : reg_alphas, 'xgboost__reg_lambda': reg_lambdas,
'xgboost__subsample': subsamples},
n_jobs=1, verbose=1, random_state=7)
 rscv.fit(new_train.values, y_train)
 xgboost_ = rscv.best_estimator_.named_steps.xgboost
score = cv_score(xgboost_)
 print("
X Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
print('Best X Gradient Boosting: ',xgboost_)
 xgboost = make_pipeline(RobustScaler() ,SelectFromModel(lasso_), xgboost_)
 steps = [('scaler',RobustScaler()),('select',SelectFromModel(lasso_)) ,('lgbm',
lgb.LGBMRegressor(objective='regression',num_leaves=5, learning_rate=6e-2, n_estimators=500, max_bin = 60,
bagging_fraction = 1, bagging_freq = 5, feature_fraction = 1e-1, feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 10, verbose=-1)) ]
lgbm_p = Pipeline(steps)
 n_estimatorss=[500,600,700,800,900,1000]
learning_rates=np.linspace(1e-2,1e-1,num=10)
 num_leavess=[5,10,15]
max_bins=[10,20,30,40,50,60,70,80,90,100]
bagging_fractions=[0.2,0.4,0.6,0.8,1]
bagging_freqs=[1,3,5,7,9]
feature_fractions=np.linspace(1e-1,1,num=10)
 min_data_in_leafs=[2,4,6,8,10]
min_sum_hessian_in_leafs=[10,20,30]
rscv = RandomizedSearchCV(lgbm_p, cv=nfold,
param_distributions={'lgbm__n_estimators': n_estimatorss, 'lgbm__learning_rate': learning_rates,
'lgbm__num_leaves': num_leavess, 'lgbm__max_bin': max_bins, 'lgbm__bagging_fraction': bagging_fractions,
'lgbm__bagging_freq': bagging_freqs, 'lgbm__feature_fraction' : feature_fractions, 'lgbm__min_data_in_leaf': min_data_in_leafs,
'lgbm__min_sum_hessian_in_leaf': min_sum_hessian_in_leafs},
n_jobs=1, verbose=1, random_state=7)
 rscv.fit(new_train.values, y_train)
 lgbm_ = rscv.best_estimator_.named_steps.lgbm
score = cv_score(lgbm_)
 print("
Light Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
print('Best Light Gradient Boosting: ',lgbm_)
 lgbm = make_pipeline(RobustScaler() ,SelectFromModel(lasso_), lgbm_ )
<choose_model_class>
if flag:
score = cv_score(lasso)
 lasso_w = 1.0/score.std()
print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = cv_score(enet)
 enet_w = 1.0/score.std()
print("
ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = cv_score(krr)
 krr_w = 1.0/score.std()
print("
Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = cv_score(gboost)
 gboost_w = 1.0/score.std()
print("
Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = cv_score(xgboost)
 xgboost_w = 1.0/score.std()
print("
X Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = cv_score(lgbm)
 lgbm_w = 1.0/score.std()
print("
Light Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
weights = [enet_w, gboost_w, krr_w, lasso_w, xgboost_w, lgbm_w]
averaged_model = AveragedModel(models =(enet, gboost, krr, lasso, xgboost, lgbm), weights=weights/sum(weights))
score = cv_score(averaged_model)
 averaged_model_w = 1.0/score.std()
print("
Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
stacked_averaged_model = StackedAveragedModel(base_models =(lasso, enet, gboost, krr, xgboost, lgbm), meta_model = lasso)
 score=cv_score(stacked_averaged_model)
 stacked_averaged_model_w = 1.0/score.std()
print("
Stacked Averaged model score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
<define_search_model><EOS> 
if flag:
stacked_averaged_model.fit(new_train.values, y_train)
 stacked_train_pred = stacked_averaged_model.predict(new_train.values)
 stacked_pred = stacked_averaged_model.predict(new_test.values)
 print("
Stacked Averaged model train prediction error: {:.4f}
".format(np.sqrt(mean_squared_error(y_train, stacked_train_pred))))
averaged_model.fit(new_train.values, y_train)
 averaged_train_pred = averaged_model.predict(new_train.values)
 averaged_pred = averaged_model.predict(new_test.values)
 print("
Averaged base model train prediction error: {:.4f}
".format(np.sqrt(mean_squared_error(y_train, averaged_train_pred))))
total_weight = stacked_averaged_model_w+averaged_model_w
train_pred =(stacked_train_pred*stacked_averaged_model_w + averaged_train_pred*averaged_model_w)/total_weight
print("
Stacked and Averaged train prediction error: {:4f}
".format(np.sqrt(mean_squared_error(y_train, train_pred))))
prediction = stacked_pred
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = np.expm1(prediction)
 sub.to_csv('submission.csv',index=False )