Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<data_type_conversions>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv" )
objects = []
for i in data.columns:
if data[i].dtype == object:
objects.append(i)
 data.update(data[objects].fillna('None'))
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numeric = []
for i in data.columns:
if data[i].dtype in numeric_dtypes:
numeric.append(i)
 data.update(data[numeric].fillna(0))<count_missing_values>
train_df_na =(train_df.isnull().sum() / len(train_df)) * 100
train_df_na = train_df_na.drop(train_df_na[train_df_na == 0].index ).sort_values(ascending=False)[:20]
missing_data = pd.DataFrame({'Missing Ratio' :train_df_na})
 missing_data.head(10 )
data.isna().sum()<create_dataframe>
train_df = train_df.drop(['PoolQC', 'MiscFeature', 'Alley', 'Fence'], axis=1)
 test_df = test_df.drop(['PoolQC', 'MiscFeature', 'Alley', 'Fence'], axis=1 )
df1=data.copy()<drop_column>
for dataset in [train_df, test_df]:
dataset['SF'] = dataset['TotalBsmtSF'] + dataset['1stFlrSF'] + dataset['2ndFlrSF']
dataset['TotalBath'] = dataset['BsmtFullBath'] +(1/2)* dataset['BsmtHalfBath'] + dataset['FullBath'] +(1/2)* dataset['HalfBath']
dataset['HasPool'] = dataset['PoolArea'].apply(lambda x : 1 if x>0 else 0)
 dataset['Has2ndFloor'] = dataset['2ndFlrSF'].apply(lambda x : 1 if x>0 else 0)
 dataset['HasGarage'] = dataset['GarageArea'].apply(lambda x : 1 if x>0 else 0 )
all_data_na =(df1.isnull().sum() / len(df1)) * 100
df2 = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Ratio' :df2})
 missing_data.head()<create_dataframe>
train_df = train_df.drop(['Id'], axis=1 )
df=data.copy()<define_variables>
LotShape_mapping = {"Reg": 1, "IR1": 2, "IR2": 3, "IR3": 4}
LandSlope_mapping = {"Gtl": 1, "Mod": 2, "Sev": 3}
ExterQual_mapping = {"Ex": 1, "Gd": 2, "TA": 3, "Fa": 4, "Po": 5}
ExterCond_mapping = {"Ex": 1, "Gd": 2, "TA": 3, "Fa": 4, "Po": 5}
BsmtQual_mapping = {"Ex": 1, "Gd": 2, "TA": 3, "Fa": 4, "Po": 5, "NA": 6}
BsmtCond_mapping = {"Ex": 1, "Gd": 2, "TA": 3, "Fa": 4, "Po": 5, "NA": 6}
BsmtExposure_mapping = {"Gd": 1, "Av": 2, "Mn": 3, "No": 4, "NA": 5}
BsmtFinType1_mapping = {"GLQ": 1, "ALQ": 2, "BLQ": 3, "Rec": 4, "LwQ": 5, "Unf": 6, "NA": 7}
BsmtFinType2_mapping = {"GLQ": 1, "ALQ": 2, "BLQ": 3, "Rec": 4, "LwQ": 5, "Unf": 6, "NA": 7}
HeatingQC_mapping = {"Ex": 1, "Gd": 2, "TA": 3, "Fa": 4, "Po": 5}
CentralAir_mapping = {"N": 0, "Y": 1}
KitchenQual_mapping = {"Ex": 1, "Gd": 2, "TA": 3, "Fa": 4, "Po": 5}
FireplaceQu_mapping = {"Ex": 1, "Gd": 2, "TA": 3, "Fa": 4, "Po": 5}
GarageQual_mapping = {"Ex": 1, "Gd": 2, "TA": 3, "Fa": 4, "Po": 5}
GarageCond_mapping = {"Ex": 1, "Gd": 2, "TA": 3, "Fa": 4, "Po": 5}
PavedDrive_mapping = {"Y": 1, "P": 2, "N": 3}
for dataset in [train_df, test_df]:
dataset['LotShape'] = dataset['LotShape'].map(LotShape_mapping)
 dataset['LandSlope'] = dataset['LandSlope'].map(LandSlope_mapping)
 dataset['ExterQual'] = dataset['ExterQual'].map(ExterQual_mapping)
 dataset['ExterCond'] = dataset['ExterCond'].map(ExterCond_mapping)
 dataset['BsmtQual'] = dataset['BsmtQual'].map(BsmtQual_mapping)
 dataset['BsmtCond'] = dataset['BsmtCond'].map(BsmtCond_mapping)
 dataset['BsmtExposure'] = dataset['BsmtExposure'].map(BsmtExposure_mapping)
 dataset['BsmtFinType1'] = dataset['BsmtFinType1'].map(BsmtFinType1_mapping)
 dataset['BsmtFinType2'] = dataset['BsmtFinType2'].map(BsmtFinType2_mapping)
 dataset['HeatingQC'] = dataset['HeatingQC'].map(HeatingQC_mapping)
 dataset['CentralAir'] = dataset['CentralAir'].map(CentralAir_mapping)
 dataset['KitchenQual'] = dataset['KitchenQual'].map(KitchenQual_mapping)
 dataset['GarageQual'] = dataset['GarageQual'].map(GarageQual_mapping)
 dataset['GarageCond'] = dataset['GarageCond'].map(GarageCond_mapping)
 dataset['PavedDrive'] = dataset['PavedDrive'].map(PavedDrive_mapping )
cols = ['MSZoning', 'Street', 'LotShape',
'LandContour', 'Utilities', 'LotConfig', 'LandSlope',
'Neighborhood', 'Condition1', 'Condition2', 'BldgType', 'HouseStyle',
'RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'ExterQual',
'ExterCond', 'Foundation', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1',
'BsmtFinType2', 'Heating', 'HeatingQC', 'CentralAir', 'Electrical', 'KitchenQual',
'Functional', 'FireplaceQu', 'GarageType', 'GarageFinish', 'GarageQual',
'GarageCond', 'PavedDrive', 'SaleType', 'SaleCondition']<categorify>
for attr in train_df.select_dtypes(include='object'):
print(train_df[[attr, "SalePrice"]].groupby([attr], as_index=False ).mean().sort_values(by='SalePrice', ascending=False))
print("
________________________
" )
for c in cols:
lbl = LabelEncoder()
lbl.fit(list(df[c].values))
df[c] = lbl.transform(list(df[c].values))
print('Shape all_data: {}'.format(df.shape))<feature_engineering>
cat_attrs =["MSZoning","LandContour", "LotConfig", "Neighborhood", "Condition1", "Condition2", "HouseStyle", "RoofStyle",
"RoofMatl", "Exterior1st", "Exterior2nd", "MasVnrType", "Foundation", "Heating", "Electrical", "Functional",
"GarageType", "GarageFinish", "SaleType", "SaleCondition"]
df['TotalSF'] = df['TotalBsmtSF'] +df['1stFlrSF'] + df['2ndFlrSF']<import_modules>
drop_indexes = train_df.sort_values(by='SF', ascending=False)[:2].index
train_df = train_df.drop(drop_indexes )
from scipy import stats
from scipy.stats import norm, skew<count_duplicates>
class DataFrameSelector(BaseEstimator, TransformerMixin):
def __init__(self, attr_names):
self.attr_names = attr_names
def fit(self, X, y=None):
return self
def transform(self, X):
return X[self.attr_names].values
df = df.iloc[:, ~df.columns.duplicated() ]<prepare_x_and_y>
class LabelBinarizerPipelineFriendly(MultiLabelBinarizer):
def fit(self, X, y=None):
super(LabelBinarizerPipelineFriendly,self ).fit(X)
 def transform(self, X, y=None):
return super(LabelBinarizerPipelineFriendly, self ).transform(X)
 def fit_transform(self, X, y=None):
return super(LabelBinarizerPipelineFriendly, self ).fit(X ).transform(X )
y_train = train.SalePrice.values 
<import_modules>
num_data_attrs = [x for x in num_attrs if not x=='SalePrice']
num_pipeline = Pipeline([ 
('selector', DataFrameSelector(num_data_attrs)) , 
('imputer', SimpleImputer(strategy="median")) , 
('std_scaler', StandardScaler()),
] )
from scipy import stats
from scipy.stats import norm, skew<feature_engineering>
cat_pipeline = Pipeline([ 
('selector', DataFrameSelector(cat_attrs)) , 
('imputer', SimpleImputer(strategy="most_frequent")) , 
('binarizer', LabelBinarizerPipelineFriendly()),
] )
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
df[feat] = boxcox1p(df[feat], lam )<categorify>
full_pipeline = FeatureUnion(transformer_list=[ 
('num_pipeline', num_pipeline), 
('cat_pipeline', cat_pipeline),
] )
df = pd.get_dummies(df)
 print(df.shape )<import_modules>
def find_outliers(model, X, y, sigma=3):
y_pred = pd.Series(model.predict(X), index=y.index)
 resid = y - y_pred
mean_resid = resid.mean()
std_resid = resid.std()
z =(resid - mean_resid)/std_resid
outliers = z[abs(z)>sigma].index
return outliers
import pandas_profiling<count_missing_values>
train_data = train_df.drop(['SalePrice'], axis=1)
 train_labels = train_df['SalePrice']
model = Ridge()
train_prepared = full_pipeline.fit_transform(train_data)
 model.fit(train_prepared, train_labels)
 outliers = find_outliers(model, train_prepared, train_labels)
 train_df = train_df.drop(np.asarray(outliers))
df.apply(lambda x: sum(x.isnull()),axis=0 )<import_modules>
train_data = train_df.drop(['SalePrice'], axis=1)
 train_labels = train_df['SalePrice']
import plotly.express as px<set_options>
train_prepared = full_pipeline.fit_transform(train_data )
warnings.filterwarnings("ignore", category=DeprecationWarning )<prepare_x_and_y>
def rms_score_cv(model):
rmse= np.sqrt(-cross_val_score(model, train_prepared, train_labels, scoring="neg_mean_squared_error", cv = 5))
return(rmse.mean() )
train = df[:ntrain]
test = df[ntrain:]<import_modules>
linear_reg = LinearRegression()
print("Linear Regression : ", rms_score_cv(linear_reg))
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone<import_modules>
ridge = Ridge(alpha=0.6, solver='cholesky')
 rms_score_cv(ridge )
from sklearn.linear_model import Ridge
from sklearn.linear_model import Lasso
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor, ExtraTreesRegressor
from sklearn.svm import SVR, LinearSVR
from sklearn.linear_model import ElasticNet, SGDRegressor, BayesianRidge
from sklearn.kernel_ridge import KernelRidge
from xgboost import XGBRegressor
import xgboost as xgb
import lightgbm as lgb
from sklearn.metrics import mean_squared_error 
<compute_train_metric>
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
rms_score_cv(lasso )
n_folds = 10
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=2020 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )<choose_model_class>
e_net = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
rms_score_cv(e_net )
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=42))<choose_model_class>
ransac = RANSACRegressor()
rms_score_cv(ransac )
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=9,
min_child_weight=1.7817, n_estimators=2000,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1 )<compute_test_metric>
svm_reg = LinearSVR(epsilon=0.01)
 rms_score_cv(svm_reg )
score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<choose_model_class>
svm_poly_reg = SVR(kernel='poly', degree=2, C=100, epsilon=0.1)
 rms_score_cv(svm_poly_reg )
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.03, n_estimators=1500,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )<compute_test_metric>
decision_tree = DecisionTreeRegressor(max_depth=7, random_state=42)
 rms_score_cv(decision_tree )
score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<choose_model_class>
rand_forest = RandomForestRegressor(max_depth=15, random_state=42)
 rms_score_cv(rand_forest )
Rd=Ridge()<compute_test_metric>
voting_reg = VotingRegressor(
 estimators=[('e_net', e_net),('ridge', ridge),('svm_poly_reg', svm_poly_reg),('rand_forest', rand_forest)])
 rms_score_cv(voting_reg )
score = rmsle_cv(Rd)
 print("Rd score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<choose_model_class>
bagging_reg = BaggingRegressor(e_net, n_estimators=300, bootstrap=True, n_jobs=-1)
Rf= RandomForestRegressor()<compute_test_metric>
bagging_reg = BaggingRegressor(e_net, n_estimators=500, bootstrap=False, n_jobs=-1)
score = rmsle_cv(Rf)
 print("Rf score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<choose_model_class>
extra_tree = ExtraTreesRegressor(random_state=42)
 rms_score_cv(extra_tree )
GBR= GradientBoostingRegressor()<compute_test_metric>
ada_boost = AdaBoostRegressor(e_net, n_estimators=500, learning_rate=0.5)
score = rmsle_cv(GBR)
 print("GBR score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<choose_model_class>
gbr = GradientBoostingRegressor(max_depth=5, n_estimators=15, learning_rate=0.6)
 rms_score_cv(gbr )
Ext= ExtraTreesRegressor()<compute_test_metric>
estimators = [ e_net, ridge, svm_poly_reg, rand_forest, extra_tree]
slice = int(( 0.8)*(len(train_prepared)))
 train_d = train_prepared[:slice]
train_l = train_labels[:slice]
val_d = train_prepared[slice: ]
val_l = train_labels[slice:]
score = rmsle_cv(Ext)
 print("Ext score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<choose_model_class>
for estimator in estimators:
estimator.fit(train_d, train_l )
Svmr=LinearSVR()<compute_test_metric>
val_preds = np.empty(( len(val_d), len(estimators)) , dtype=np.float32)
 for index, estimator in enumerate(estimators):
val_preds[:, index] = model.predict(val_d)
 blender = RandomForestRegressor(n_estimators=200, oob_score=True, random_state=42)
 blender.fit(val_preds, val_l)
 blender.oob_score_
score = rmsle_cv(Svmr)
 print("Svmr score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<choose_model_class>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y=None):
X = X
y = y.values
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=42)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def get_metafeatures(self, X):
return np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )
BR=BayesianRidge()<compute_test_metric>
stacked_averaged_models = StackingAveragedModels(base_models =(e_net, ridge, svm_poly_reg, rand_forest), meta_model = lasso)
score = rmsle_cv(BR)
 print("BR score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
el_net = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
el_net.fit(train_prepared, train_labels )
score = rmsle_cv(lasso)
 print("lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<train_model>
test_prepared = full_pipeline.transform(test_df )
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=2020)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )<choose_model_class>
pred = el_net.predict(test_prepared)
 predSalePrice = [np.exp(x)for x in pred]
stacked_averaged_models = StackingAveragedModels(base_models =(model_lgb,model_xgb),
meta_model = model_xgb )<compute_test_metric>
output = pd.DataFrame({
'Id' : test_df['Id'],
'SalePrice' : predSalePrice
} )
<compute_test_metric><EOS> 
output.to_csv('prediction.csv', index=False )