Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<filter>
Image(".. /input/kernel-files/model_training_advanced_regression.png" )
train_test = train_test[columns_to_models]<normalization>
pd.set_option('display.max_columns', None)
 warnings.filterwarnings(action="ignore")
 pd.options.display.max_seq_items = 8000
pd.options.display.max_rows = 8000
print(os.listdir(".. /input/kernel-files"))
scaler = StandardScaler()<define_variables>
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 train.shape, test.shape
new_numeric = []
for col in numeric:
if col in columns_to_models:
new_numeric.append(col )<define_variables>
train_ID = train['Id']
test_ID = test['Id']
train.drop(['Id'], axis=1, inplace=True)
 test.drop(['Id'], axis=1, inplace=True)
 train.shape, test.shape
new_cat = []
for col in categorical:
if col in columns_to_models:
new_cat.append(col )<normalization>
train["SalePrice"] = np.log1p(train["SalePrice"] )
train_test[new_numeric] = scaler.fit_transform(train_test[new_numeric] )<categorify>
train.drop(train[(train['OverallQual']<5)&(train['SalePrice']>200000)].index, inplace=True)
 train.drop(train[(train['GrLivArea']>4500)&(train['SalePrice']<300000)].index, inplace=True)
 train.reset_index(drop=True, inplace=True )
for col in new_cat:
if col != 'SalePrice':
train_test[col] = LE.fit_transform(train_test[col] )<define_variables>
train_labels = train['SalePrice'].reset_index(drop=True)
 train_features = train.drop(['SalePrice'], axis=1)
 test_features = test
all_features = pd.concat([train_features, test_features] ).reset_index(drop=True)
 all_features.shape
train_test_lin = train_test.copy()<categorify>
def percent_missing(df):
data = pd.DataFrame(df)
 df_cols = list(pd.DataFrame(data))
dict_x = {}
for i in range(0, len(df_cols)) :
dict_x.update({df_cols[i]: round(data[df_cols[i]].isnull().mean() *100,2)})
 return dict_x
missing = percent_missing(all_features)
 df_miss = sorted(missing.items() , key=lambda x: x[1], reverse=True)
 print('Percent of missing data')
 df_miss[0:10]
train_test_lin = pd.concat([train_test_lin,
pd.get_dummies(train_test_lin['BedroomAbvGr'], prefix='BedroomAbvGr', drop_first=True),
pd.get_dummies(train_test_lin['GarageFinish'], prefix='GarageFinish', drop_first=True),
pd.get_dummies(train_test_lin['BsmtQual'], prefix='BsmtQual', drop_first=True),
pd.get_dummies(train_test_lin['HalfBath'], prefix='HalfBath', drop_first=True),
pd.get_dummies(train_test_lin['MoSold'], prefix='MoSold', drop_first=True),
pd.get_dummies(train_test_lin['SaleCondition'], prefix='SaleCondition', drop_first=True),
pd.get_dummies(train_test_lin['BsmtFinType1'], prefix='BsmtFinType1', drop_first=True),
pd.get_dummies(train_test_lin['KitchenQual'], prefix='KitchenQual', drop_first=True),
pd.get_dummies(train_test_lin['OverallQual'], prefix='OverallQual', drop_first=True),
pd.get_dummies(train_test_lin['LandContour'], prefix='LandContour', drop_first=True),
pd.get_dummies(train_test_lin['TotRmsAbvGrd'], prefix='TotRmsAbvGrd', drop_first=True),
pd.get_dummies(train_test_lin['ExterQual'], prefix='ExterQual', drop_first=True),
pd.get_dummies(train_test_lin['Exterior2nd'], prefix='Exterior2nd', drop_first=True),
pd.get_dummies(train_test_lin['OverallCond'], prefix='OverallCond', drop_first=True),
pd.get_dummies(train_test_lin['GarageType'], prefix='GarageType', drop_first=True),
pd.get_dummies(train_test_lin['Fireplaces'], prefix='Fireplaces', drop_first=True),
pd.get_dummies(train_test_lin['CentralAir'], prefix='CentralAir', drop_first=True),
pd.get_dummies(train_test_lin['LotShape'], prefix='LotShape', drop_first=True),
pd.get_dummies(train_test_lin['Exterior1st'], prefix='Exterior1st', drop_first=True),
pd.get_dummies(train_test_lin['BsmtExposure'], prefix='BsmtExposure', drop_first=True),
pd.get_dummies(train_test_lin['Neighborhood'], prefix='Neighborhood', drop_first=True),
pd.get_dummies(train_test_lin['FullBath'], prefix='FullBath', drop_first=True),
pd.get_dummies(train_test_lin['GarageCars'], prefix='GarageCars', drop_first=True),
pd.get_dummies(train_test_lin['MSZoning'], prefix='MSZoning', drop_first=True)
 ],
axis=1 )<drop_column>
all_features['MSSubClass'] = all_features['MSSubClass'].apply(str)
 all_features['YrSold'] = all_features['YrSold'].astype(str)
 all_features['MoSold'] = all_features['MoSold'].astype(str )
train_test_lin.drop(new_cat, axis=1, inplace=True )<split>
def handle_missing(features):
features['Functional'] = features['Functional'].fillna('Typ')
 features['Electrical'] = features['Electrical'].fillna("SBrkr")
 features['KitchenQual'] = features['KitchenQual'].fillna("TA")
 features['Exterior1st'] = features['Exterior1st'].fillna(features['Exterior1st'].mode() [0])
 features['Exterior2nd'] = features['Exterior2nd'].fillna(features['Exterior2nd'].mode() [0])
 features['SaleType'] = features['SaleType'].fillna(features['SaleType'].mode() [0])
 features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
features["PoolQC"] = features["PoolQC"].fillna("None")
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
features[col] = features[col].fillna(0)
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
features[col] = features[col].fillna('None')
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
features[col] = features[col].fillna('None')
 features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
objects = []
for i in features.columns:
if features[i].dtype == object:
objects.append(i)
 features.update(features[objects].fillna('None'))
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numeric = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numeric.append(i)
 features.update(features[numeric].fillna(0))
return features
all_features = handle_missing(all_features )
train_to_model = train_test[train_test['SalePrice'].isna() == False]
test_to_model = train_test[train_test['SalePrice'].isna() == True]
train_to_model_lin = train_test_lin[train_test_lin['SalePrice'].isna() == False]
test_to_model_lin = train_test_lin[train_test_lin['SalePrice'].isna() == True]<drop_column>
missing = percent_missing(all_features)
 df_miss = sorted(missing.items() , key=lambda x: x[1], reverse=True)
 print('Percent of missing data')
 df_miss[0:10]
test_to_model.drop(['SalePrice'], axis=1, inplace=True)
 test_to_model_lin.drop(['SalePrice'], axis=1, inplace=True )<split>
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numeric = []
for i in all_features.columns:
if all_features[i].dtype in numeric_dtypes:
numeric.append(i )
X_train, X_test, y_train, y_test = train_test_split(train_to_model.drop(['SalePrice'], axis=1), \
train_to_model['SalePrice'], test_size=0.1, random_state=42 )<split>
for i in skew_index:
all_features[i] = boxcox1p(all_features[i], boxcox_normmax(all_features[i] + 1))
X_train_lin, X_test_lin, y_train_lin, y_test_lin = train_test_split(train_to_model_lin.drop(['SalePrice'], axis=1), \
train_to_model_lin['SalePrice'], test_size=0.1, random_state=42 )<choose_model_class>
all_features['BsmtFinType1_Unf'] = 1*(all_features['BsmtFinType1'] == 'Unf')
 all_features['HasWoodDeck'] =(all_features['WoodDeckSF'] == 0)* 1
all_features['HasOpenPorch'] =(all_features['OpenPorchSF'] == 0)* 1
all_features['HasEnclosedPorch'] =(all_features['EnclosedPorch'] == 0)* 1
all_features['Has3SsnPorch'] =(all_features['3SsnPorch'] == 0)* 1
all_features['HasScreenPorch'] =(all_features['ScreenPorch'] == 0)* 1
all_features['YearsSinceRemodel'] = all_features['YrSold'].astype(int)- all_features['YearRemodAdd'].astype(int)
 all_features['Total_Home_Quality'] = all_features['OverallQual'] + all_features['OverallCond']
all_features = all_features.drop(['Utilities', 'Street', 'PoolQC',], axis=1)
 all_features['TotalSF'] = all_features['TotalBsmtSF'] + all_features['1stFlrSF'] + all_features['2ndFlrSF']
all_features['YrBltAndRemod'] = all_features['YearBuilt'] + all_features['YearRemodAdd']
all_features['Total_sqr_footage'] =(all_features['BsmtFinSF1'] + all_features['BsmtFinSF2'] +
all_features['1stFlrSF'] + all_features['2ndFlrSF'])
 all_features['Total_Bathrooms'] =(all_features['FullBath'] +(0.5 * all_features['HalfBath'])+
all_features['BsmtFullBath'] +(0.5 * all_features['BsmtHalfBath']))
all_features['Total_porch_sf'] =(all_features['OpenPorchSF'] + all_features['3SsnPorch'] +
all_features['EnclosedPorch'] + all_features['ScreenPorch'] +
all_features['WoodDeckSF'])
 all_features['TotalBsmtSF'] = all_features['TotalBsmtSF'].apply(lambda x: np.exp(6)if x <= 0.0 else x)
 all_features['2ndFlrSF'] = all_features['2ndFlrSF'].apply(lambda x: np.exp(6.5)if x <= 0.0 else x)
 all_features['GarageArea'] = all_features['GarageArea'].apply(lambda x: np.exp(6)if x <= 0.0 else x)
 all_features['GarageCars'] = all_features['GarageCars'].apply(lambda x: 0 if x <= 0.0 else x)
 all_features['LotFrontage'] = all_features['LotFrontage'].apply(lambda x: np.exp(4.2)if x <= 0.0 else x)
 all_features['MasVnrArea'] = all_features['MasVnrArea'].apply(lambda x: np.exp(4)if x <= 0.0 else x)
 all_features['BsmtFinSF1'] = all_features['BsmtFinSF1'].apply(lambda x: np.exp(6.5)if x <= 0.0 else x)
 all_features['haspool'] = all_features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 all_features['has2ndfloor'] = all_features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 all_features['hasgarage'] = all_features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 all_features['hasbsmt'] = all_features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 all_features['hasfireplace'] = all_features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )
kfolds = KFold(n_splits=10, shuffle=True, random_state=42 )<choose_model_class>
def logs(res, ls):
m = res.shape[1]
for l in ls:
res = res.assign(newcol=pd.Series(np.log(1.01+res[l])).values)
 res.columns.values[m] = l + '_log'
m += 1
return res
log_features = ['LotFrontage','LotArea','MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF',
'TotalBsmtSF','1stFlrSF','2ndFlrSF','LowQualFinSF','GrLivArea',
'BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','BedroomAbvGr','KitchenAbvGr',
'TotRmsAbvGrd','Fireplaces','GarageCars','GarageArea','WoodDeckSF','OpenPorchSF',
'EnclosedPorch','3SsnPorch','ScreenPorch','PoolArea','MiscVal','YearRemodAdd','TotalSF']
all_features = logs(all_features, log_features )
%%time
lgb_reg = lgbm.LGBMRegressor(objective='regression', learning_rate=0.01, n_estimators=100, n_jobs=-1, subsample=0.5)
 lgb_param_grid = {"learning_rate":[0.1, 0.01, 0.001],
"n_estimators":[250, 500, 750, 1000],
"subsample":[0.3, 0.4, 0.5, 0.6],
"max_depth":[4, 5, 6, 7]
}
grid_search = GridSearchCV(lgb_reg, param_grid=lgb_param_grid, cv=kfolds, scoring="neg_mean_squared_error", n_jobs=-1, verbose=1)
 grid_search.fit(X_train, y_train)
 lgb_reg = grid_search.best_estimator_
print(grid_search.best_params_)
 y_pred = lgb_reg.predict(X_test)
 print('-' * 10 + 'LGBM' + '-' * 10)
 print('R square Accuracy: ', r2_score(y_test, y_pred))
print('Mean Absolute Error: ', mean_absolute_error(y_test, y_pred))
print('Mean Squared Error: ', mean_squared_error(y_test, y_pred))<choose_model_class>
def squares(res, ls):
m = res.shape[1]
for l in ls:
res = res.assign(newcol=pd.Series(res[l]*res[l] ).values)
 res.columns.values[m] = l + '_sq'
m += 1
return res
squared_features = ['YearRemodAdd', 'LotFrontage_log',
'TotalBsmtSF_log', '1stFlrSF_log', '2ndFlrSF_log', 'GrLivArea_log',
'GarageCars_log', 'GarageArea_log']
all_features = squares(all_features, squared_features )
%%time
xgb_reg = xgb.XGBRegressor(learning_rate=0.01, n_estimators=100, n_jobs=-1, booster='gbtree', random_state=42, subsample=0.5)
 xgb_param_grid = {"learning_rate":[0.1, 0.01, 0.001],
"n_estimators":[250, 500, 750, 1000],
"subsample":[0.3, 0.4, 0.5, 0.6],
"max_depth":[4, 5, 6, 7]
}
grid_search = GridSearchCV(xgb_reg, param_grid=xgb_param_grid, cv=kfolds, scoring="neg_mean_squared_error", n_jobs=-1, verbose=1)
 grid_search.fit(X_train, y_train)
 xgb_reg = grid_search.best_estimator_
print(grid_search.best_params_)
 y_pred = xgb_reg.predict(X_test)
 print('-' * 10 + 'XGB' + '-' * 10)
 print('R square Accuracy: ', r2_score(y_test, y_pred))
print('Mean Absolute Error: ', mean_absolute_error(y_test, y_pred))
print('Mean Squared Error: ', mean_squared_error(y_test, y_pred))<choose_model_class>
all_features = pd.get_dummies(all_features ).reset_index(drop=True)
 all_features.shape
%%time
cat_reg = cat.CatBoostRegressor(learning_rate=0.01, n_estimators=100, objective='RMSE', loss_function='R2', random_state=42, subsample=0.5)
 cat_param_grid = {"learning_rate":[0.1, 0.01, 0.001],
"n_estimators":[250, 500, 750, 1000],
"subsample":[0.3, 0.4, 0.5, 0.6],
"max_depth":[4, 5, 6, 7]
}
grid_search = GridSearchCV(cat_reg, param_grid=cat_param_grid, cv=kfolds, scoring="neg_mean_squared_error", n_jobs=-1, verbose=1)
 grid_search.fit(X_train, y_train)
 cat_reg = grid_search.best_estimator_
print(grid_search.best_params_)
 y_pred = cat_reg.predict(X_test)
 print('-' * 10 + 'CatBoost' + '-' * 10)
 print('R square Accuracy: ', r2_score(y_test, y_pred))
print('Mean Absolute Error: ', mean_absolute_error(y_test, y_pred))
print('Mean Squared Error: ', mean_squared_error(y_test, y_pred))<choose_model_class>
all_features = all_features.loc[:,~all_features.columns.duplicated() ]
%%time
gb_reg = GradientBoostingRegressor(n_estimators=100, learning_rate=0.01, random_state=42, subsample=0.5)
 gbdt_param_grid = {"learning_rate":[0.1, 0.01, 0.001],
"n_estimators":[250, 500, 750, 1000],
"subsample":[0.3, 0.4, 0.5, 0.6],
"max_depth":[4, 5, 6, 7]
}
grid_search = GridSearchCV(gb_reg, param_grid=gbdt_param_grid, cv=kfolds, scoring="neg_mean_squared_error", n_jobs=-1, verbose=1)
 grid_search.fit(X_train, y_train)
 gb_reg = grid_search.best_estimator_
print(grid_search.best_params_)
 y_pred = gb_reg.predict(X_test)
 print('-' * 10 + 'GBM' + '-' * 10)
 print('R square Accuracy: ', r2_score(y_test, y_pred))
print('Mean Absolute Error: ', mean_absolute_error(y_test, y_pred))
print('Mean Squared Error: ', mean_squared_error(y_test, y_pred))<compute_train_metric>
kf = KFold(n_splits=12, random_state=42, shuffle=True )
alphas = [1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 0.1, 1, 10]
ridgecv_reg = make_pipeline(RidgeCV(alphas=alphas, cv=kfolds))
ridgecv_reg.fit(X_train_lin, y_train_lin)
 y_pred = ridgecv_reg.predict(X_test_lin)
 print('-' * 10 + 'RidgeCV' + '-' * 10)
 print('R square: ', r2_score(y_test_lin, y_pred))
print('Mean Absolute Error: ', mean_absolute_error(y_test_lin, y_pred))
print('Mean Squared Error: ', mean_squared_error(y_test_lin, y_pred))<compute_train_metric>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, train_labels, scoring="neg_mean_squared_error", cv=kf))
return(rmse )
lassocv_reg = make_pipeline(LassoCV(alphas=alphas, cv=kfolds))
lassocv_reg.fit(X_train_lin, y_train_lin)
 y_pred = lassocv_reg.predict(X_test_lin)
 print('-' * 10 + 'LassoCV' + '-' * 10)
 print('R square: ', r2_score(y_test_lin, y_pred))
print('Mean Absolute Error: ', mean_absolute_error(y_test_lin, y_pred))
print('Mean Squared Error: ', mean_squared_error(y_test_lin, y_pred))<compute_train_metric>
lightgbm = LGBMRegressor(objective='regression',
num_leaves=6,
learning_rate=0.01,
n_estimators=7000,
max_bin=200,
bagging_fraction=0.8,
bagging_freq=4,
bagging_seed=8,
feature_fraction=0.2,
feature_fraction_seed=8,
min_sum_hessian_in_leaf = 11,
verbose=-1,
random_state=42)
 xgboost = XGBRegressor(learning_rate=0.01,
n_estimators=6000,
max_depth=4,
min_child_weight=0,
gamma=0.6,
subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear',
nthread=-1,
scale_pos_weight=1,
seed=27,
reg_alpha=0.00006,
random_state=42)
 ridge_alphas = [1e-15, 1e-10, 1e-8, 9e-4, 7e-4, 5e-4, 3e-4, 1e-4, 1e-3, 5e-2, 1e-2, 0.1, 0.3, 1, 3, 5, 10, 15, 18, 20, 30, 50, 75, 100]
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=ridge_alphas, cv=kf))
svr = make_pipeline(RobustScaler() , SVR(C= 20, epsilon= 0.008, gamma=0.0003))
gbr = GradientBoostingRegressor(n_estimators=6000,
learning_rate=0.01,
max_depth=4,
max_features='sqrt',
min_samples_leaf=15,
min_samples_split=10,
loss='huber',
random_state=42)
 rf = RandomForestRegressor(n_estimators=1200,
max_depth=15,
min_samples_split=5,
min_samples_leaf=5,
max_features=None,
oob_score=True,
random_state=42)
 stack_gen = StackingCVRegressor(regressors=(xgboost, lightgbm, svr, ridge, gbr, rf),
meta_regressor=xgboost,
use_features_in_secondary=True )
alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008, 0.0009, 0.001]
l1ratio = [0.8, 0.83, 0.85, 0.87, 0.9, 0.92, 0.95, 0.97, 0.99, 1]
elasticv_reg = make_pipeline(ElasticNetCV(alphas=alphas, cv=kfolds, l1_ratio=l1ratio))
elasticv_reg.fit(X_train_lin, y_train_lin)
 y_pred = elasticv_reg.predict(X_test_lin)
 print('-' * 10 + 'ElasticNetCV' + '-' * 10)
 print('R square: ', r2_score(y_test_lin, y_pred))
print('Mean Absolute Error: ', mean_absolute_error(y_test_lin, y_pred))
print('Mean Squared Error: ', mean_squared_error(y_test_lin, y_pred))<predict_on_test>
scores = {}
score = cv_rmse(lightgbm)
 print("lightgbm: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['lgb'] =(score.mean() , score.std() )
ensemble = lgb_reg.predict(test_to_model)* 0.2 + xgb_reg.predict(test_to_model)* 0.2 + cat_reg.predict(test_to_model)* 0.2 \
+ gb_reg.predict(test_to_model)* 0.2 + lassocv_reg.predict(test_to_model_lin)* 0.05 + \
ridgecv_reg.predict(test_to_model_lin)* 0.05 + elasticv_reg.predict(test_to_model_lin)* 0.1<import_modules>
score = cv_rmse(xgboost)
 print("xgboost: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['xgb'] =(score.mean() , score.std() )
import keras
from keras import models, layers, regularizers<choose_model_class>
score = cv_rmse(svr)
 print("SVR: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['svr'] =(score.mean() , score.std() )
model = models.Sequential()
model.add(layers.Dense(256, activation='relu', input_shape=(X_train.shape[1],)))
 model.add(layers.Dense(256, activation='relu'))
model.add(layers.Dense(1))
optimizer = keras.optimizers.RMSprop(0.001)
 model.compile(optimizer=optimizer,
loss='mae',
metrics=['mae', 'mse'] )<train_model>
score = cv_rmse(ridge)
 print("ridge: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['ridge'] =(score.mean() , score.std() )
history = model.fit(X_train, y_train,
epochs=250,
batch_size=5,
validation_data=(X_test, y_test))<compute_test_metric>
score = cv_rmse(rf)
 print("rf: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['rf'] =(score.mean() , score.std() )
y_pred = model.predict(X_test)
 print('-' * 10 + 'NN' + '-' * 10)
 print('R square Accuracy: ', r2_score(y_test, y_pred))
print('Mean Absolute Error: ', mean_absolute_error(y_test, y_pred))
print('Mean Squared Error: ', mean_squared_error(y_test, y_pred))<choose_model_class>
score = cv_rmse(gbr)
 print("gbr: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['gbr'] =(score.mean() , score.std() )
model = models.Sequential()
model.add(layers.Dense(256, activation='relu', input_shape=(X_train.shape[1],)))
 model.add(layers.Dense(256, activation='relu'))
model.add(layers.Dense(1))
optimizer = keras.optimizers.RMSprop(0.001)
 model.compile(optimizer=optimizer,
loss='mae',
metrics=['mae', 'mse'] )<train_model>
print('stack_gen')
 stack_gen_model = stack_gen.fit(np.array(X), np.array(train_labels))
history = model.fit(train_to_model.drop(['SalePrice'], axis=1), train_to_model['SalePrice'],
epochs=300,
batch_size=5,
validation_split=0.05 )<predict_on_test>
print('lightgbm')
 lgb_model_full_data = lightgbm.fit(X, train_labels )
y_pred_model = model.predict(test_to_model )<define_variables>
print('xgboost')
 xgb_model_full_data = xgboost.fit(X, train_labels )
model_pred = []
for pred in range(len(y_pred_model.tolist())) :
model_pred.append(y_pred_model.tolist() [pred][0] )<save_to_csv>
print('Svr')
 svr_model_full_data = svr.fit(X, train_labels )
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = model_pred
sub.to_csv('submission_NN.csv',index=False )<prepare_output>
print('Ridge')
 ridge_model_full_data = ridge.fit(X, train_labels )
model_pred = np.array(model_pred)
 model_pred<define_variables>
print('RandomForest')
 rf_model_full_data = rf.fit(X, train_labels )
final_blend = ensemble * 0.3 + model_pred * 0.7<save_to_csv>
print('GradientBoosting')
 gbr_model_full_data = gbr.fit(X, train_labels )
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = final_blend
sub.to_csv('submission_blend_NN7.csv',index=False )<save_to_csv>
def blended_predictions(X):
return(( 0.1 * ridge_model_full_data.predict(X)) + \ 
(0.2 * svr_model_full_data.predict(X)) + \ 
(0.1 * gbr_model_full_data.predict(X)) + \ 
(0.1 * xgb_model_full_data.predict(X)) + \ 
(0.1 * lgb_model_full_data.predict(X)) + \ 
(0.05 * rf_model_full_data.predict(X)) + \ 
(0.35 * stack_gen_model.predict(np.array(X))))
final_blend = ensemble * 0.4 + model_pred * 0.6
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = final_blend
sub.to_csv('submission_blend_NN6.csv',index=False )<save_to_csv>
blended_score = rmsle(train_labels, blended_predictions(X))
scores['blended'] =(blended_score, 0)
 print('RMSLE score on train data:')
 print(blended_score )
final_blend = ensemble * 0.5 + model_pred * 0.5
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = final_blend
sub.to_csv('submission_blend_NN5.csv',index=False )<save_to_csv>
submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.shape
final_blend = ensemble * 0.6 + model_pred * 0.4
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = final_blend
sub.to_csv('submission_blend_NN4.csv',index=False )<save_to_csv>
submission.iloc[:,1] = np.floor(np.expm1(blended_predictions(X_test)) )
final_blend = ensemble * 0.2 + model_pred * 0.8
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = final_blend
sub.to_csv('submission_blend_NN8.csv',index=False )<load_from_csv>
q1 = submission['SalePrice'].quantile(0.0045)
 q2 = submission['SalePrice'].quantile(0.99)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x > q1 else x*0.77)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x < q2 else x*1.1)
 submission.to_csv("submission_regression1.csv", index=False )
<set_options><EOS> 
submission['SalePrice'] *= 1.001619
submission.to_csv("submission_regression2.csv", index=False )