Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<import_modules>
pd.set_option('display.max_columns', None)
 warnings.filterwarnings(action="ignore")
 pd.options.display.max_seq_items = 8000
pd.options.display.max_rows = 8000
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt<load_from_csv>
train=pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test=pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )
dataset = pd.read_csv('.. /input/train.csv' )<count_missing_values>
train.drop('Id',axis=1,inplace=True)
 test.drop('Id',axis=1,inplace=True )
dataset.isnull().sum()<feature_engineering>
train.rename(columns={'3SsnPorch':'TSsnPorch'}, inplace=True)
 test.rename(columns={'3SsnPorch':'TSsnPorch'}, inplace=True )
dataset['Age'] = dataset['Age'].fillna(dataset['Age'].median() )<count_values>
train = train[train.GrLivArea < 4500]
train.reset_index(drop=True, inplace=True)
 train["SalePrice"] = np.log1p(train["SalePrice"])
 y = train['SalePrice'].reset_index(drop=True )
dataset['Embarked'].value_counts()<drop_column>
train_labels = train['SalePrice'].reset_index(drop=True)
 train_features = train.drop(['SalePrice'], axis=1)
 test_features = test
ntrain = train.shape[0]
ntest = test.shape[0]
features = pd.concat([train_features, test_features] ).reset_index(drop=True)
 features.shape
dataset['Embarked'].fillna('S', inplace = True )<count_values>
features['MSSubClass'] = features['MSSubClass'].apply(str)
 features['YrSold'] = features['YrSold'].astype(str)
 features['MoSold'] = features['MoSold'].astype(str )
dataset['Sex'].value_counts()<feature_engineering>
print(features["Functional"].unique())
 features["Functional"].value_counts()
dataset['Sex'] = dataset['Sex'].apply(lambda x: 1 if x=='male' else 0 )<categorify>
print(features['Electrical'].unique())
 features["Electrical"].value_counts()
cat_columns = ['Embarked']
dataset = pd.get_dummies(dataset,prefix_sep = '__', columns = cat_columns )<drop_column>
def handle_missing(features):
features['Functional'] = features['Functional'].fillna('Typ')
 features['Electrical'] = features['Electrical'].fillna("SBrkr")
 features['KitchenQual'] = features['KitchenQual'].fillna("TA")
 features['Exterior1st'] = features['Exterior1st'].fillna(features['Exterior1st'].mode() [0])
 features['Exterior2nd'] = features['Exterior2nd'].fillna(features['Exterior2nd'].mode() [0])
 features['SaleType'] = features['SaleType'].fillna(features['SaleType'].mode() [0])
 features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
features["PoolQC"] = features["PoolQC"].fillna("None")
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
features[col] = features[col].fillna(0)
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
features[col] = features[col].fillna('None')
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
features[col] = features[col].fillna('None')
 features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
objects = []
for i in features.columns:
if features[i].dtype == object:
objects.append(i)
 features.update(features[objects].fillna('None'))
numeric_dtypes = ['int64', 'float64']
numeric = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numeric.append(i)
 features.update(features[numeric].fillna(0))
return features
features = handle_missing(features )
dataset.drop(['PassengerId', 'Name', 'Ticket', 'Cabin'], axis=1, inplace=True )<prepare_x_and_y>
features.isna().sum()
x = dataset.iloc[:, 1:].values
y = dataset.iloc[:,:1].values<split>
numeric_dtypes = ['int64', 'float64']
numeric = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numeric.append(i )
x_train, x_test, y_train, y_test = train_test_split(x,y, test_size = 0.2 )<define_variables>
skew_features = features[numeric].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skew = skew_features[skew_features > 0.5]
skew_index = high_skew.index
print("There are {} numerical features with Skew > 0.5 :".format(high_skew.shape[0]))
skewness = pd.DataFrame({'Skew' :high_skew})
 skew_features.head(10 )
N,D = x.shape
M = 10<init_hyperparams>
for i in skew_index:
features[i] = boxcox1p(features[i], boxcox_normmax(features[i] + 1))
w1= np.random.randn(D,M)/np.sqrt(D+M)
 b1 = np.zeros(M)
 w2 = np.random.randn(M,1)/np.sqrt(M+1)
 b2 = 0<compute_test_metric>
features = features.drop(['Utilities', 'Street', 'PoolQC',], axis=1)
 features['YrBltAndRemod']=features['YearBuilt']+features['YearRemodAdd']
features['TotalSF']=features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']
features['Total_sqr_footage'] =(features['BsmtFinSF1'] + features['BsmtFinSF2'] +
features['1stFlrSF'] + features['2ndFlrSF'])
 features['Total_Bathrooms'] =(features['FullBath'] +(0.5 * features['HalfBath'])+
features['BsmtFullBath'] +(0.5 * features['BsmtHalfBath']))
features['Total_porch_sf'] =(features['OpenPorchSF'] + features['TSsnPorch'] +
features['EnclosedPorch'] + features['ScreenPorch'] +
features['WoodDeckSF'])
 features['haspool'] = features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 features['has2ndfloor'] = features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasgarage'] = features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 features['hasbsmt'] = features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasfireplace'] = features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
def sigmoid(z):
return 1/(1+np.exp(-z))<data_type_conversions>
features['TotalSF'] = features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']
d_relu = lambda x:(x>0 ).astype(x.dtype )<compute_test_metric>
all_features = pd.get_dummies(features ).reset_index(drop=True)
 all_features.shape
def forward(x,w1,b1,w2,b2):
z = np.maximum(( x.dot(w1)+b1), 0)
 return sigmoid(z.dot(w2)+b2), z 
<compute_test_metric>
outliers = [30, 88, 462, 631, 1322]
X = X.drop(X.index[outliers])
 y = y.drop(y.index[outliers])
 overfit = []
for i in X.columns:
counts = X[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(X)* 100 > 99.94:
overfit.append(i)
 overfit = list(overfit)
 X = X.drop(overfit, axis=1)
 X_sub = X_sub.drop(overfit, axis=1 )
def cross_en(T,Y):
return -np.mean(T*np.log(Y)+(1-T)*np.log(1-Y)) 
<define_variables>
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb
cost_train = []
cost_test = []
lr = 0.001<compute_train_metric>
kf = KFold(n_splits=12, random_state=42, shuffle=True)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, y, scoring="neg_mean_squared_error", cv=kf))
return(rmse)
for i in range(100000):
yprep, ztrain = forward(x_train,w1,b1,w2,b2)
 train_cost = cross_en(y_train, yprep)
 yprep_test, ztest = forward(x_test, w1,b1,w2,b2)
 test_cost = cross_en(y_test, yprep_test)
 if i%5000==0:
print('train_cost = {}'.format(train_cost))
print('test_cost = {}'.format(test_cost))
E =(yprep-y_train)
 w2-= lr*(ztrain.T.dot(E)/len(x_train))
b2-= lr*(E.sum() /len(x_train))
dz = E.dot(w2.T)*d_relu(ztrain)
 w1-= lr*(x_train.T.dot(dz)/len(x_train))
b1-= lr*(dz.sum() /len(x_train))
cost_train.append(train_cost)
 cost_test.append(test_cost)
<load_from_csv>
lightgbm = LGBMRegressor(objective='regression',
num_leaves=6,
learning_rate=0.01,
n_estimators=7000,
max_bin=200,
bagging_fraction=0.8,
bagging_freq=4,
bagging_seed=8,
feature_fraction=0.2,
feature_fraction_seed=8,
min_sum_hessian_in_leaf = 11,
verbose=-1,
random_state=42)
 xgboost = XGBRegressor(learning_rate=0.01,
n_estimators=6000,
max_depth=4,
min_child_weight=0,
gamma=0.6,
subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear',
nthread=-1,
scale_pos_weight=1,
seed=27,
reg_alpha=0.00006,
random_state=42)
 ridge_alphas = [1e-15, 1e-10, 1e-8, 9e-4, 7e-4, 5e-4, 3e-4, 1e-4, 1e-3, 5e-2, 1e-2, 0.1, 0.3, 1, 3, 5, 10, 15, 18, 20, 30, 50, 75, 100]
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=ridge_alphas, cv=kf))
svr = make_pipeline(RobustScaler() , SVR(C= 20, epsilon= 0.008, gamma=0.0003))
gbr = GradientBoostingRegressor(n_estimators=6000,
learning_rate=0.01,
max_depth=4,
max_features='sqrt',
min_samples_leaf=15,
min_samples_split=10,
loss='huber',
random_state=42)
 rf = RandomForestRegressor(n_estimators=1200,
max_depth=15,
min_samples_split=5,
min_samples_leaf=5,
max_features=None,
oob_score=True,
random_state=42)
 stack_gen = StackingCVRegressor(regressors=(xgboost, lightgbm, svr, ridge, gbr, rf),
meta_regressor=xgboost,
use_features_in_secondary=True )
test_dataset = pd.read_csv('.. /input/test.csv' )<count_missing_values>
scores = {}
score = cv_rmse(lightgbm)
 print("lightgbm: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['lgb'] =(score.mean() , score.std())
 score = cv_rmse(xgboost)
 print("xgboost: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['xgb'] =(score.mean() , score.std())
 score = cv_rmse(svr)
 print("SVR: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['svr'] =(score.mean() , score.std())
 score = cv_rmse(ridge)
 print("ridge: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['ridge'] =(score.mean() , score.std())
 score = cv_rmse(rf)
 print("rf: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['rf'] =(score.mean() , score.std())
 score = cv_rmse(gbr)
 print("gbr: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['gbr'] =(score.mean() , score.std() )
test_dataset.isnull().sum()<data_type_conversions>
print('stack_gen')
 stack_gen_model = stack_gen.fit(np.array(X), np.array(y))
print('Ridge')
 ridge_model_full_data = ridge.fit(X, y)
 print('Svr')
 svr_model_full_data = svr.fit(X, y)
 print('GradientBoosting')
 gbr_model_full_data = gbr.fit(X, y)
 print('xgboost')
 xgb_model_full_data = xgboost.fit(X, y)
 print('lightgbm')
 lgb_model_full_data = lightgbm.fit(X, y )
test_dataset['Age']= test_dataset['Age'].fillna(test_dataset['Age'].median())
 test_dataset['Fare']= test_dataset['Fare'].fillna(test_dataset['Fare'].median() )<feature_engineering>
print('RandomForest')
 rf_model_full_data = rf.fit(X, y )
test_dataset['Sex'] = test_dataset['Sex'].apply(lambda x: 1 if x=='male' else 0 )<categorify>
def blend_models_predict(X):
return(( 0.1 * ridge_model_full_data.predict(X)) + \ 
(0.2 * svr_model_full_data.predict(X)) + \ 
(0.1 * gbr_model_full_data.predict(X)) + \ 
(0.1 * xgb_model_full_data.predict(X)) + \ 
(0.1 * lgb_model_full_data.predict(X)) + \ 
(0.05 * rf_model_full_data.predict(X)) + \ 
(0.35 * stack_gen_model.predict(np.array(X))))
test_dataset = pd.get_dummies(test_dataset,prefix_sep = '__', columns = cat_columns )<drop_column>
blended_score = rmsle(y, blend_models_predict(X))
scores['blended'] =(blended_score, 0)
 print('RMSLE score on train data:')
 print(blended_score )
passengerID = test_dataset['PassengerId']
test_dataset.drop(['PassengerId', 'Name', 'Ticket', 'Cabin'], axis=1, inplace=True )<prepare_x_and_y>
print('Predict submission')
 submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.iloc[:,1] = np.floor(np.expm1(blend_models_predict(X_sub)) )
test_data_pred,z_test_data = forward(test_dataset,w1,b1,w2,b2)
 test_data_pred =(np.round(test_data_pred))
submission = pd.DataFrame({'PassengerId': passengerID, 'Survived':test_data_pred[0]} ).astype(int )<save_to_csv>
submission.to_csv("submission.csv", index=False )
submission.to_csv('submission.csv', index=False)
<load_from_csv>
submission.to_csv("submission.csv", index=False )
train=pd.read_csv('.. /input/train.csv')
 test=pd.read_csv('.. /input/test.csv')
 sub=pd.read_csv('.. /input/sample_submission.csv' )<drop_column>
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 print("Train set size:", train.shape)
 print("Test set size:", test.shape)
 print('START data processing', datetime.now() , )
x=train.drop(['label'],axis=1)
 x_test=test.copy()<feature_engineering>
train_ID = train['Id']
test_ID = test['Id']
x=x/255
x_test=x_test/255<prepare_x_and_y>
train.drop(['Id'], axis=1, inplace=True)
 test.drop(['Id'], axis=1, inplace=True )
y=train['label']<create_dataframe>
train = train[train.GrLivArea < 4500]
train.reset_index(drop=True, inplace=True )
y=pd.Categorical(y )<categorify>
train["SalePrice"] = np.log1p(train["SalePrice"])
 y = train.SalePrice.reset_index(drop=True)
 train_features = train.drop(['SalePrice'], axis=1)
 test_features = test
y=pd.get_dummies(y )<split>
features = pd.concat([train_features, test_features] ).reset_index(drop=True)
 print(features.shape )
x_train,x_val,y_train,y_val=train_test_split(x,y,test_size=0.1,random_state=0 )<split>
features['MSSubClass'] = features['MSSubClass'].apply(str)
 features['YrSold'] = features['YrSold'].astype(str)
 features['MoSold'] = features['MoSold'].astype(str)
 features['Functional'] = features['Functional'].fillna('Typ')
 features['Electrical'] = features['Electrical'].fillna("SBrkr")
 features['KitchenQual'] = features['KitchenQual'].fillna("TA")
 features['Exterior1st'] = features['Exterior1st'].fillna(features['Exterior1st'].mode() [0])
 features['Exterior2nd'] = features['Exterior2nd'].fillna(features['Exterior2nd'].mode() [0])
 features['SaleType'] = features['SaleType'].fillna(features['SaleType'].mode() [0])
 features["PoolQC"] = features["PoolQC"].fillna("None")
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
features[col] = features[col].fillna(0)
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
features[col] = features[col].fillna('None')
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
features[col] = features[col].fillna('None')
 features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
objects = []
for i in features.columns:
if features[i].dtype == object:
objects.append(i)
 features.update(features[objects].fillna('None'))
features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
x_train=x_train.values
y_train=y_train.values
x_val=x_val.values
y_val=y_val.values
x_test=x_test.values<train_model>
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numerics.append(i)
 features.update(features[numerics].fillna(0))
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics2 = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numerics2.append(i)
 skew_features = features[numerics2].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skew = skew_features[skew_features > 0.5]
skew_index = high_skew.index
for i in skew_index:
features[i] = boxcox1p(features[i], boxcox_normmax(features[i] + 1))
features = features.drop(['Utilities', 'Street', 'PoolQC',], axis=1)
 features['YrBltAndRemod']=features['YearBuilt']+features['YearRemodAdd']
features['TotalSF']=features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']
features['Total_sqr_footage'] =(features['BsmtFinSF1'] + features['BsmtFinSF2'] +
features['1stFlrSF'] + features['2ndFlrSF'])
 features['Total_Bathrooms'] =(features['FullBath'] +(0.5 * features['HalfBath'])+
features['BsmtFullBath'] +(0.5 * features['BsmtHalfBath']))
features['Total_porch_sf'] =(features['OpenPorchSF'] + features['3SsnPorch'] +
features['EnclosedPorch'] + features['ScreenPorch'] +
features['WoodDeckSF'])
x_train=x_train.reshape(37800,28,28,1)
 x_val=x_val.reshape(4200,28,28,1)
 x_test=x_test.reshape(28000,28,28,1 )<choose_model_class>
features['haspool'] = features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 features['has2ndfloor'] = features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasgarage'] = features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 features['hasbsmt'] = features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasfireplace'] = features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 print(features.shape)
 final_features = pd.get_dummies(features ).reset_index(drop=True)
 print(final_features.shape)
 X = final_features.iloc[:len(y), :]
X_sub = final_features.iloc[len(X):, :]
print('X', X.shape, 'y', y.shape, 'X_sub', X_sub.shape )
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))<choose_model_class>
outliers = [30, 88, 462, 631, 1322]
X = X.drop(X.index[outliers])
 y = y.drop(y.index[outliers])
 overfit = []
for i in X.columns:
counts = X[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(X)* 100 > 99.94:
overfit.append(i)
 overfit = list(overfit)
 overfit.append('MSZoning_C(all)')
 X = X.drop(overfit, axis=1 ).copy()
X_sub = X_sub.drop(overfit, axis=1 ).copy()
print('X', X.shape, 'y', y.shape, 'X_sub', X_sub.shape)
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adadelta() ,
metrics=['accuracy'] )<train_model>
kfolds = KFold(n_splits=10, shuffle=True, random_state=42)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, y,
scoring="neg_mean_squared_error",
cv=kfolds))
return(rmse)
 alphas_alt = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas2 = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
ridge = make_pipeline(RobustScaler() ,
RidgeCV(alphas=alphas_alt, cv=kfolds))
lasso = make_pipeline(RobustScaler() ,
LassoCV(max_iter=1e7, alphas=alphas2,
random_state=42, cv=kfolds))
elasticnet = make_pipeline(RobustScaler() ,
ElasticNetCV(max_iter=1e7, alphas=e_alphas,
cv=kfolds, l1_ratio=e_l1ratio))
svr = make_pipeline(RobustScaler() ,
SVR(C= 20, epsilon= 0.008, gamma=0.0003,))
gbr = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =42)
 lightgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=5000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
verbose=-1, 
)
 xgboost = XGBRegressor(learning_rate=0.01, n_estimators=3460,
max_depth=3, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear', nthread=-1,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006)
 stack_gen = StackingCVRegressor(regressors=(ridge, lasso, elasticnet,
gbr, xgboost, lightgbm),
meta_regressor=xgboost,
use_features_in_secondary=True)
 print('TEST score on CV')
 score = cv_rmse(ridge)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(lasso)
 print("Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(elasticnet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(svr)
 print("SVR score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(lightgbm)
 print("Lightgbm score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(gbr)
 print("GradientBoosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(xgboost)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() , )
model.fit(x_train, y_train,
batch_size=128,
epochs=15,
verbose=1,
validation_data=(x_val, y_val))<save_to_csv>
stack_gen_model = stack_gen.fit(np.array(X), np.array(y))
print(datetime.now() , 'elasticnet')
 elastic_model_full_data = elasticnet.fit(X, y)
 print(datetime.now() , 'lasso')
 lasso_model_full_data = lasso.fit(X, y)
 print(datetime.now() , 'ridge')
 ridge_model_full_data = ridge.fit(X, y)
 print(datetime.now() , 'svr')
 svr_model_full_data = svr.fit(X, y)
 print(datetime.now() , 'GradientBoosting')
 gbr_model_full_data = gbr.fit(X, y)
 print(datetime.now() , 'xgboost')
 xgb_model_full_data = xgboost.fit(X, y)
 print(datetime.now() , 'lightgbm')
 lgb_model_full_data = lightgbm.fit(X, y )
pred=model.predict(x_test,verbose=0)
 new_pred = [np.argmax(y, axis=None, out=None)for y in pred]
output=pd.DataFrame({'ImageId':sub['ImageId'],'Label':new_pred})
 output.to_csv('Digit_recognizer.csv', index=False )<load_from_csv>
def blend_models_predict(X):
return(( 0.1 * elastic_model_full_data.predict(X)) + \ 
(0.1 * lasso_model_full_data.predict(X)) + \ 
(0.1 * ridge_model_full_data.predict(X)) + \ 
(0.1 * svr_model_full_data.predict(X)) + \ 
(0.1 * gbr_model_full_data.predict(X)) + \ 
(0.15 * xgb_model_full_data.predict(X)) + \ 
(0.1 * lgb_model_full_data.predict(X)) + \ 
(0.25 * stack_gen_model.predict(np.array(X))))
print('RMSLE score on train data:')
 print(rmsle(y, blend_models_predict(X)) )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
<prepare_x_and_y>
submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.iloc[:,1] = np.floor(np.expm1(blend_models_predict(X_sub)))
 print('Blend with Top Kernals submissions', datetime.now() ,)
 sub_1 = pd.read_csv('.. /input/top-10-0-10943-stacking-mice-and-brutal-force/House_Prices_submit.csv')
 sub_2 = pd.read_csv('.. /input/hybrid-svm-benchmark-approach-0-11180-lb-top-2/hybrid_solution.csv')
 sub_3 = pd.read_csv('.. /input/lasso-model-for-regression-problem/lasso_sol22_Median.csv')
 submission.iloc[:,1] = np.floor(( 0.25 * np.floor(np.expm1(blend_models_predict(X_sub)))) + 
(0.25 * sub_1.iloc[:,1])+ 
(0.25 * sub_2.iloc[:,1])+ 
(0.25 * sub_3.iloc[:,1]))
label = np.array(train.iloc[:,0],np.str)
 data = np.array(train.iloc[:,1:],np.float32)
 label_test = np.array([])
 data_test = np.array(test.iloc[:,:],np.float32)
<categorify>
q1 = submission['SalePrice'].quantile(0.0045)
 q2 = submission['SalePrice'].quantile(0.99)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x > q1 else x*0.77)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x < q2 else x*1.1)
 submission.to_csv("new_submission.csv", index=False)
 print('Save submission', datetime.now() , )