Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<set_options>
InteractiveShell.ast_node_interactivity = "all"
%matplotlib inline 
<load_from_csv>
import numpy as np
import pandas as pd
from datetime import datetime
from scipy.stats import skew
from scipy.special import boxcox1p
from scipy.stats import boxcox_normmax
from sklearn.linear_model import ElasticNetCV, LassoCV, RidgeCV, Ridge
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.svm import SVR
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler, StandardScaler
from sklearn.model_selection import KFold, cross_val_score
from sklearn.metrics import mean_squared_error as mse
from sklearn.metrics import make_scorer
from sklearn.neighbors import LocalOutlierFactor
from sklearn.linear_model import LinearRegression
from mlxtend.regressor import StackingCVRegressor
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
import matplotlib.pyplot as plt
import seaborn as sns
import os
mnist_train = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 mnist_test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv" )<count_missing_values>
print(os.listdir(".. /input"))
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 print("Train set size:", train.shape)
 print("Test set size:", test.shape)
 print('start data processing', datetime.now() , )
mnist_train.isna().any().any()<categorify>
outs = detect_outliers(train['GrLivArea'], train['SalePrice'],top=5)
 outs
mnist_train_data = mnist_train.loc[:, "pixel0":]
mnist_train_label = mnist_train.loc[:, "label"]
mnist_train_data = mnist_train_data/255.0
mnist_test = mnist_test/255.0<normalization>
outs = detect_outliers(train['LowQualFinSF'], train['SalePrice'],top=5)
 outs
standardized_scalar = StandardScaler()
standardized_data = standardized_scalar.fit_transform(mnist_train_data)
 standardized_data.shape<train_model>
outliers = [30, 88, 462, 523, 632, 1298, 1324]
cov_matrix = np.matmul(standardized_data.T, standardized_data)
 cov_matrix.shape<compute_train_metric>
train = train.drop(train.index[outliers])
 train.shape
lambdas, vectors = eigh(cov_matrix, eigvals=(782, 783))
vectors.shape<concatenate>
y = train.SalePrice.reset_index(drop=True)
 train_features = train.drop(['SalePrice'], axis=1)
 test_features = test
features = pd.concat([train_features, test_features] ).reset_index(drop=True)
 features.drop(['Id'], axis=1, inplace=True)
 print(features.shape )
new_coordinates = np.matmul(vectors, standardized_data.T)
 print(new_coordinates.shape)
 new_coordinates = np.vstack(( new_coordinates, mnist_train_label)).T<prepare_output>
features['MSSubClass'] = features['MSSubClass'].apply(str)
 features['YrSold'] = features['YrSold'].astype(str)
 features['MoSold'] = features['MoSold'].astype(str)
df_new = pd.DataFrame(new_coordinates, columns=["f1", "f2", "labels"])
 df_new.head()<normalization>
numeric_features = features.dtypes[features.dtypes != 'object'].index
numeric_features
len(numeric_features)
 category_features = features.dtypes[features.dtypes == 'object'].index
category_features
len(category_features )
pca = decomposition.PCA()
pca.n_components = 2
pca_data = pca.fit_transform(standardized_data)
 pca_data.shape<concatenate>
special_features = [
'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2',
'FireplaceQu', 'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond',
'PoolQC', 'Fence'
]
len(special_features )
pca_data = np.vstack(( pca_data.T, mnist_train_label)).T<prepare_output>
features['Functional'] = features['Functional'].fillna('Typ')
 features['Electrical'] = features['Electrical'].fillna("SBrkr")
 features['KitchenQual'] = features['KitchenQual'].fillna("TA")
 features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
features["PoolQC"] = features["PoolQC"].fillna("None")
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
features[col] = features[col].fillna('None')
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
features[col] = features[col].fillna('None')
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
features[col] = features[col].fillna(0)
 features['Exterior1st'] = features['Exterior1st'].fillna(features['Exterior1st'].mode() [0])
 features['Exterior2nd'] = features['Exterior2nd'].fillna(features['Exterior2nd'].mode() [0])
 features['SaleType'] = features['SaleType'].fillna(features['SaleType'].mode() [0])
 features[numeric_features] = features[numeric_features].apply(
 lambda x: x.fillna(0))
features[category_features] = features[category_features].apply(
 lambda x: x.fillna('None'))
df_PCA = pd.DataFrame(new_coordinates, columns=["f1", "f2", "labels"])
 df_PCA.head()<create_dataframe>
skew_features = features[numeric_features].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skew = skew_features[skew_features > 0.15]
skew_index = high_skew.index
for i in skew_index:
features[i] = boxcox1p(features[i], boxcox_normmax(features[i] + 1))
mnist_train_data = np.array(mnist_train_data)
 mnist_train_label = np.array(mnist_train_label )<import_modules>
features = features.drop(['Utilities', 'Street', 'PoolQC',], axis=1)
 features['YrBltAndRemod']=features['YearBuilt']+features['YearRemodAdd']
features['TotalSF']=features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']
features['Total_sqr_footage'] =(features['BsmtFinSF1'] + features['BsmtFinSF2'] +
features['1stFlrSF'] + features['2ndFlrSF'])
 features['Total_Bathrooms'] =(features['FullBath'] +(0.5 * features['HalfBath'])+
features['BsmtFullBath'] +(0.5 * features['BsmtHalfBath']))
features['Total_porch_sf'] =(features['OpenPorchSF'] + features['3SsnPorch'] +
features['EnclosedPorch'] + features['ScreenPorch'] +
features['WoodDeckSF'])
 features['haspool'] = features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 features['has2ndfloor'] = features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasgarage'] = features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 features['hasbsmt'] = features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasfireplace'] = features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Lambda, Flatten, BatchNormalization
from tensorflow.keras.layers import Conv2D, MaxPool2D, AvgPool2D
from tensorflow.keras.optimizers import Adadelta
from keras.utils.np_utils import to_categorical
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.callbacks import ReduceLROnPlateau
from tensorflow.keras.callbacks import LearningRateScheduler<categorify>
print("before get_dummies:",features.shape)
 final_features = pd.get_dummies(features ).reset_index(drop=True)
 print("after get_dummies:",final_features.shape)
 X = final_features.iloc[:len(y), :]	
X_sub = final_features.iloc[len(y):, :]
print("after get_dummies, the dataset size:",'X', X.shape, 'y', y.shape, 'X_sub', X_sub.shape )
nclasses = mnist_train_label.max() - mnist_train_label.min() + 1
mnist_train_label = to_categorical(mnist_train_label, num_classes = nclasses)
 print("Shape of ytrain after encoding: ", mnist_train_label.shape )<choose_model_class>
overfit = []
for i in X.columns:
counts = X[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(X)* 100 > 99.94:
overfit.append(i)
 overfit = list(overfit)
 overfit.append('MSZoning_C(all)')
 X = np.array(X.drop(overfit, axis=1 ).copy())
 y = np.array(y)
 X_sub = np.array(X_sub.drop(overfit, axis=1 ).copy())
 print('X', X.shape, 'y', y.shape, 'X_sub', X_sub.shape)
 print('feature engineering finished!', datetime.now() )
def build_model(input_shape=(28, 28, 1)) :
model = Sequential()
model.add(Conv2D(32, kernel_size = 3, activation='relu', input_shape = input_shape))
model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(64, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(128, kernel_size = 4, activation='relu'))
model.add(BatchNormalization())
 model.add(Flatten())
 model.add(Dropout(0.4))
model.add(Dense(10, activation='softmax'))
return model
def compile_model(model, optimizer='adam', loss='categorical_crossentropy'):
model.compile(optimizer=optimizer, loss=loss, metrics=["accuracy"])
 def train_model(model, train, test, epochs, split):
history = model.fit(train, test, shuffle=True, epochs=epochs, validation_split=split)
 return history<train_model>
kfolds = KFold(n_splits=10, shuffle=True, random_state=42)
 def rmsle(y, y_pred):
return np.sqrt(mse(y, y_pred))
def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, y, scoring="neg_mean_squared_error", cv=kfolds))
return(rmse )
cnn_model = build_model(( 28, 28, 1))
compile_model(cnn_model, 'adam', 'categorical_crossentropy')
 model_history = train_model(cnn_model, mnist_train_data, mnist_train_label, 80, 0.2 )<predict_on_test>
alphas_alt = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas2 = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
predictions = cnn_model.predict(mnist_test_arr )<define_variables>
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=alphas_alt, cv=kfolds))
lasso = make_pipeline(
 RobustScaler() ,
LassoCV(max_iter=1e7, alphas=alphas2, random_state=42, cv=kfolds))
elasticnet = make_pipeline(
 RobustScaler() ,
ElasticNetCV(max_iter=1e7, alphas=e_alphas, cv=kfolds, l1_ratio=e_l1ratio))
svr = make_pipeline(RobustScaler() , SVR(
 C=20,
epsilon=0.008,
gamma=0.0003, 
))
gbr = GradientBoostingRegressor(n_estimators=3000,
learning_rate=0.05,
max_depth=4,
max_features='sqrt',
min_samples_leaf=15,
min_samples_split=10,
loss='huber',
random_state=42)
 lightgbm = LGBMRegressor(
 objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=5000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
verbose=-1, 
)
 xgboost = XGBRegressor(learning_rate=0.01,
n_estimators=3460,
max_depth=3,
min_child_weight=0,
gamma=0,
subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear',
nthread=-1,
scale_pos_weight=1,
seed=27,
reg_alpha=0.00006 )
predictions_test = []
for i in predictions:
predictions_test.append(np.argmax(i))<save_to_csv>
stack_gen = StackingCVRegressor(regressors=(ridge, lasso, elasticnet, gbr, xgboost, lightgbm),
meta_regressor=xgboost,
use_features_in_secondary=True )
submission = pd.DataFrame({
"ImageId": mnist_test.index+1,
"Label": predictions_test
})
 submission.to_csv('my_submission.csv', index=False )<set_options>
print('TEST score on CV')
 score = cv_rmse(ridge)
 print("Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(lasso)
 print("Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(elasticnet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(svr)
 print("SVR score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(lightgbm)
 print("Lightgbm score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(gbr)
 print("GradientBoosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(xgboost)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() , )
%matplotlib inline 
<load_from_csv>
print('START Fit')
 print(datetime.now() , 'StackingCVRegressor')
 stack_gen_model = stack_gen.fit(X, y )
mnist_train = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 mnist_test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv" )<count_missing_values>
print('Predict submission', datetime.now() ,)
 submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.iloc[:,1] = np.floor(np.expm1(stack_gen_model.predict(X_sub)))
 submission.head()
submission.to_csv("submission_stacking.csv", index=False )
mnist_train.isna().any().any()<categorify>
print(datetime.now() , 'ridge')
 ridge_model_full_data = ridge.fit(X, y)
 print(datetime.now() , 'lasso')
 lasso_model_full_data = lasso.fit(X, y)
 print(datetime.now() , 'elasticnet')
 elastic_model_full_data = elasticnet.fit(X, y)
 print(datetime.now() , 'GradientBoosting')
 gbr_model_full_data = gbr.fit(X, y)
 print(datetime.now() , 'xgboost')
 xgb_model_full_data = xgboost.fit(X, y)
 print(datetime.now() , 'lightgbm')
 lgb_model_full_data = lightgbm.fit(X, y)
 print(datetime.now() , 'svr')
 svr_model_full_data = svr.fit(X, y )
mnist_train_data = mnist_train.loc[:, "pixel0":]
mnist_train_label = mnist_train.loc[:, "label"]
mnist_train_data = mnist_train_data/255.0
mnist_test = mnist_test/255.0<normalization>
models = [
ridge_model_full_data, lasso_model_full_data, elastic_model_full_data,
gbr_model_full_data, xgb_model_full_data, lgb_model_full_data,
svr_model_full_data, stack_gen_model
]
len(models )
standardized_scalar = StandardScaler()
standardized_data = standardized_scalar.fit_transform(mnist_train_data)
 standardized_data.shape<train_model>
public_coefs = [0.1, 0.1, 0.1, 0.1, 0.15, 0.1, 0.1, 0.25]
bias = 0
cov_matrix = np.matmul(standardized_data.T, standardized_data)
 cov_matrix.shape<compute_train_metric>
def linear_blend_models_predict(data_x,models,coefs, bias):
tmp=[model.predict(data_x)for model in models]
tmp = [c*d for c,d in zip(coefs,tmp)]
pres=np.array(tmp ).swapaxes(0,1)
 pres=np.sum(pres,axis=1)
 return pres
lambdas, vectors = eigh(cov_matrix, eigvals=(782, 783))
vectors.shape<concatenate>
print('blending models RMSLE score on train data:')
 print(rmsle(y, linear_blend_models_predict(X,models,public_coefs, bias)))
 print('Predict submission', datetime.now() ,)
 submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.iloc[:,1] = np.floor(np.expm1(linear_blend_models_predict(X_sub,models,public_coefs, bias)))
 submission.head()
submission.to_csv("submission_blend_models_with_public_coefs.csv", index=False )
new_coordinates = np.matmul(vectors, standardized_data.T)
 print(new_coordinates.shape)
 new_coordinates = np.vstack(( new_coordinates, mnist_train_label)).T<prepare_output>
alphas3 = [70]
def blend_models(train_x, train_y, models):
tmp = [model.predict(train_x)for model in models]
pres = np.array(tmp ).swapaxes(0,1)
 print(pres.shape)
 linear = RidgeCV(alphas=alphas3,
cv=kfolds,
fit_intercept=False,
scoring=make_scorer(rmsle, greater_is_better=False)
)
 linear = linear.fit(pres, train_y)
 print('linear coefficient:')
 print(linear.coef_)
 print('linear bias:')
 print(linear.intercept_)
 print('best alpha: %f'%(linear.alpha_))
print('best score: %f'%(rmsle(linear.predict(pres), train_y)))
 return linear.coef_, linear.intercept_
df_new = pd.DataFrame(new_coordinates, columns=["f1", "f2", "labels"])
 df_new.head()<normalization>
print('blending models RMSLE score on train data:')
 print(rmsle(y, linear_blend_models_predict(X,models,coefs,bias)))
 print('Predict submission', datetime.now() ,)
 submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.iloc[:,1] = np.floor(np.expm1(linear_blend_models_predict(X_sub,models,coefs,bias)))
 submission.head()
submission.to_csv("submission_blend_models_with_regression_coefs.csv", index=False )
pca = decomposition.PCA()
pca.n_components = 2
pca_data = pca.fit_transform(standardized_data)
 pca_data.shape<concatenate>
print('Blend with Top Kernals submissions', datetime.now() ,)
 sub_1 = pd.read_csv('.. /input/top-10-0-10943-stacking-mice-and-brutal-force/House_Prices_submit.csv')
 sub_2 = pd.read_csv('.. /input/hybrid-svm-benchmark-approach-0-11180-lb-top-2/hybrid_solution.csv')
 sub_3 = pd.read_csv('.. /input/lasso-model-for-regression-problem/lasso_sol.csv')
 submission.iloc[:,1] = np.floor(( 0.25 * np.floor(np.expm1(linear_blend_models_predict(X_sub,models,public_coefs, bias)))) + 
(0.25 * sub_1.iloc[:,1])+ 
(0.25 * sub_2.iloc[:,1])+ 
(0.25 * sub_3.iloc[:,1]))
submission.to_csv("submission_blend_top.csv", index=False)
 print('Save submission', datetime.now() )
pca_data = np.vstack(( pca_data.T, mnist_train_label)).T<prepare_output>
submission = pd.read_csv('.. /input/house-price-best/submission_best.csv')
 submission.to_csv("submission_best.csv", index=False)
 print('Save submission best', datetime.now() )
df_PCA = pd.DataFrame(new_coordinates, columns=["f1", "f2", "labels"])
 df_PCA.head()<create_dataframe>
mnist_train_data = np.array(mnist_train_data)
 mnist_train_label = np.array(mnist_train_label )<import_modules>
q1 = submission['SalePrice'].quantile(0.005)
 q2 = submission['SalePrice'].quantile(0.995)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x > q1 else x*1.1)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x < q2 else x*0.77)
 submission.to_csv("submission_base_blend_top_Scale extremes2.csv", index=False)
 print('Save submission', datetime.now() )