Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<compute_test_metric>
pd.set_option('display.max_columns', 500)
 pd.set_option('display.max_rows', 500 )
print(f'rmse on oof: {np.round(mean_squared_error(target, oof, squared=False),5)}' )<drop_column>
df_train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 df_test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 df_data = pd.concat([df_train, df_test])
 df_data.reset_index(drop=True, inplace=True)
 del df_train
del df_test
df_data.info(verbose=False, memory_usage='deep' )
<define_variables>
list_missing_feats = [a for a in df_data.columns if(df_data[a].isnull().sum() > 0)&(not a == 'SalePrice')]
list_numerical_feats = [n for n in df_data.select_dtypes(include=['number'] ).columns]
df_missing = pd.DataFrame(list_missing_feats, columns=['feature'])
 df_missing['feature_count'] = df_missing.apply(lambda x: df_data[x['feature']].isnull().sum() , axis=1)
 df_missing['feature_type'] = np.where(df_missing['feature'].isin(list_numerical_feats), 'number', 'text')
 df_missing = df_missing.sort_values('feature_count' ).reset_index(drop=True)
 df_missing['meh'] = 'x'
df_missing['feature_rank'] = df_missing.sort_values(['feature_count'], ascending=[True] ).groupby(['meh'] ).cumcount()
f, ax = plt.subplots(figsize =(20,8))
sns.set_color_codes('pastel')
 palette = ['r' if(x == 'number')else 'b' for x in df_missing['feature_type']]
z = sns.barplot(x='feature_count', y='feature', data=df_missing, label='Total', palette=palette, edgecolor='w')
 for index, row in df_missing.iterrows() :
text_col = 'b'
if row.feature_type == 'number': text_col = 'r'
z.text(row.feature_count + 15, row.feature_rank, str(row.feature_count), color=text_col, va="center")
 patch_num = mpatches.Patch(color='r', label='Numerical')
 patch_text = mpatches.Patch(color='b', label='Text')
 ax.legend(handles=[patch_num, patch_text])
 sns.despine(left = True, bottom = True)
 plt.show()
avg_pred_01 = sub_216['target'] * 1.3 - preds_folds_xgb * 0.1 - sub_222['target'] * 0.2 
<save_to_csv>
df_data['Functional'] = df_data['Functional'].fillna('Typ')
 df_data['Utilities'] = df_data['Utilities'].fillna('AllPub')
 for a in ['GarageYrBlt','GarageArea','GarageCars','TotalBsmtSF','MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtFullBath','BsmtHalfBath']: df_data[a] = df_data[a].fillna(0)
 for a in ['BsmtExposure','GarageType','GarageFinish','Alley','MiscFeature','BsmtFinType1','BsmtFinType2','BsmtQual','BsmtCond','GarageQual','GarageCond','FireplaceQu','Fence','PoolQC']: df_data[a] = df_data[a].fillna('Na' )
df_sub['target'] = avg_pred_01
df_sub.to_csv('submission_035.csv', index=False )<init_hyperparams>
dict_missing = {}
for b in [a for a in df_data.columns if(df_data[a].isnull().sum() > 0)&(not a == 'SalePrice')]: dict_missing[b] = df_data[b].isnull().sum()
dict_missing = dict(sorted(dict_missing.items() , key=lambda item: item[1]))
for f in dict_missing:
missing_estimator = KNeighborsClassifier(n_neighbors=10)
 if(df_data[f].dtype == np.float64): missing_estimator = KNeighborsRegressor(n_neighbors=10)
 n_cols = [n for n in df_data.select_dtypes(include=['number'] ).columns if(df_data[n].isnull().sum() == 0)&(n not in ['Id','SalePrice'])]
missing_estimator.fit(
 df_data[n_cols][(df_data[f].notnull())],
df_data[f][(df_data[f].notnull())] 
)
 df_data[f] = df_data.apply(lambda x: missing_estimator.predict(np.array(x[n_cols] ).reshape(1,-1)) [0] if pd.isnull(x[f])else x[f], axis=1 )
lgb_params_kfold = {
'n_estimators':3000,
'learning_rate': 0.004,
'min_data_per_group': 5,
'boosting_type': 'gbdt',
'num_leaves': 256,
'max_depth': -1,
'lambda_l1': 4.5,
'lambda_l2': 1.2,
'metric': 'rmse',
'cat_smooth': 1.0,
'silent': True,
'importance_type': 'split',
'feature_pre_filter': False,
'bagging_fraction': 0.85,
'min_data_in_leaf': 100,
'min_sum_hessian_in_leaf': 0.001,
'bagging_freq': 7,
'feature_fraction': 0.5,
'min_gain_to_split': 0.0,
'min_child_samples': 20,
'n_jobs': -1,
'random_state': SEED}
xgb_params = {
'booster':'gbtree',
'n_estimators':10000,
'max_depth':5,
'eta':0.006,
'gamma':1.2,
'objective':'reg:squarederror',
'verbosity':0,
'subsample':0.85,
'colsample_bytree':0.8,
'lambda':4.5,
'alpha':1.2,
'scale_pos_weight':1,
'objective':'reg:squarederror',
'eval_metric':'rmse',
'seed':SEED,
'tree_method':'gpu_hist',
'gpu_id':0
}
xgb_params = {
'booster':'gbtree',
'n_estimators':10000,
'max_depth':7,
'eta':0.01,
'gamma':1.8,
'objective':'reg:squarederror',
'verbosity':0,
'subsample':0.85,
'colsample_bytree':0.4,
'lambda':2.7,
'alpha':6,
'scale_pos_weight':1,
'objective':'reg:squarederror',
'eval_metric':'rmse',
'seed':SEED,
'tree_method':'gpu_hist',
'gpu_id':0
}<load_from_csv>
def _no_trans(df, list_cols, prefix='x_'):
for a in list_cols: df[prefix + a] = df[a]
return df
def _create_dummies(df, list_cols, prefix='x_'):
for a in list_cols:
df_dummies = pd.get_dummies(df[a].astype(str))
df[prefix + a + '_' + df_dummies.columns] = df_dummies
del df_dummies
return df
def _map_val(df, list_cols, prefix='x_'):
for a in list_cols: df[prefix + a] = df[a].fillna('Na' ).map({'Ex':5,'Gd':4,'TA':3,'Fa':2,'Po':1,'Na':0,'GLQ':6,'ALQ':5,'BLQ':4,'Rec':3,'LwQ':2,'Unf':1,'Y':1,'N':0,'GdPrv':4,'MnPrv':3,'GdWo':2,'MnWw':1})
 return df
def _merge_features(df, list_pairs, prefix='x_'):
for a in list_pairs:
list_options = list(set(df[a[0]].to_list() + df[a[1]].to_list()))
df[[prefix + str(a[0])+ '_' + o for o in list_options]] = df.apply(lambda x: _merge_list(list_options, x[a[0]], x[a[1]]), axis=1, result_type='expand')
 return df
def _merge_list(list_data, data_1, data_2):
list_return = [0] * len(list_data)
 list_return[list_data.index(data_1)] = 1
list_return[list_data.index(data_2)] = 1
return list_return
def _rank_features(df, list_cols, prefix='x_rank_'):
for a in list_cols:
ranked_data = df[(df[target].notnull())].groupby(a ).agg({target:'median'} ).sort_values(target ).index
dict_ranks = dict(zip(ranked_data, range(0, len(ranked_data))))
df[prefix + a] = df[a].map(dict_ranks)
 return df
train = pd.read_csv('.. /input/tabular-playground-series-feb-2021/train.csv')
 test = pd.read_csv('.. /input/tabular-playground-series-feb-2021/test.csv')
 sub = pd.read_csv('.. /input/tabular-playground-series-feb-2021/sample_submission.csv' )<prepare_x_and_y>
list_no_trans = ['LotArea','LotFrontage','MasVnrArea','BsmtFullBath','BsmtHalfBath','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF','1stFlrSF','GrLivArea','OverallQual','OverallCond','YearBuilt','YearRemodAdd','2ndFlrSF','LowQualFinSF','FullBath','HalfBath','BedroomAbvGr','KitchenAbvGr','TotRmsAbvGrd','Fireplaces','WoodDeckSF','OpenPorchSF','EnclosedPorch','3SsnPorch','ScreenPorch','PoolArea','MiscVal','YrSold','GarageYrBlt','GarageArea','GarageCars']
list_to_dummy = ['MSSubClass','MSZoning','Electrical','MasVnrType','Street','Alley','LotShape','LandContour','LotConfig','LandSlope','Neighborhood','BldgType','HouseStyle','RoofStyle','RoofMatl','Foundation','BsmtExposure','Heating','GarageType','GarageFinish','PavedDrive','MiscFeature','SaleCondition','Functional','Utilities','SaleType']
list_to_map = ['ExterQual','ExterCond','KitchenQual','BsmtQual','BsmtCond','BsmtFinType1','BsmtFinType2','HeatingQC','CentralAir','FireplaceQu','GarageQual','GarageCond','PoolQC','Fence']
list_to_merge = [('Condition1','Condition2'),('Exterior1st','Exterior2nd')]
list_to_rank = ['Heating','HouseStyle','LotShape','MSSubClass','Neighborhood','SaleCondition','SaleType','MasVnrType']
df_data =(df_data 
.pipe(_no_trans, list_no_trans)
.pipe(_create_dummies, list_to_dummy)
.pipe(_map_val, list_to_map)
.pipe(_merge_features, list_to_merge)
.pipe(_rank_features, list_to_rank)
)
 df_data['x_rank_MSSubClass'] = df_data['x_rank_MSSubClass'].fillna(8)
 df_data.info(verbose=False, memory_usage='deep' )
target = train['target'].values 
<categorify>
random_state = 42
def _evaluate(model, X_train, y_train):
cv = KFold(n_splits=4, random_state=random_state, shuffle=True)
 scores = cross_val_score(model, X_train.values, y_train.values, cv=cv, n_jobs=-1, error_score='neg_root_mean_squared_error')
 return scores
for feature in cat_features:
le = LabelEncoder()
le.fit(train[feature])
 train[feature] = le.transform(train[feature])
 test[feature] = le.transform(test[feature] )<define_variables>
df_evaluation = pd.DataFrame(columns=['Description','Score','Std'] )
train_oof = np.zeros(( 300000,))
test_preds = 0
train_oof.shape<init_hyperparams>
mask_base =(df_data[target].notnull())
 x_cols = [c for c in df_data.columns if c[:2] == 'x_']
xgb = XGBRegressor()
scores = _evaluate(xgb, df_data[x_cols][mask_base], df_data[target][mask_base])
 df_evaluation.loc[len(df_evaluation)] = 'baseline', np.mean(scores), np.std(scores)
 print('baseline: mean:', np.mean(scores), ', std:', np.std(scores))
params = {'reg_alpha': 6.147694913504962,
'reg_lambda': 0.002457826062076097,
'colsample_bytree': 0.3,
'subsample': 0.8,
'learning_rate': 0.0005,
'max_depth': 20,
'num_leaves': 111,
'min_child_samples': 285,
'random_state': 48,
'n_estimators': 320000,
'metric': 'rmse',
'cat_smooth': 39}<train_model>
df_data['x_busy_period'] = np.where(df_data['MoSold'].isin([3, 4, 5, 6, 7, 8]), 1, 0)
 mask_base =(df_data[target].notnull())
 x_cols = [c for c in df_data.columns if c[:2] == 'x_']
xgb = XGBRegressor()
scores = _evaluate(xgb, df_data[x_cols][mask_base], df_data[target][mask_base])
 df_evaluation.loc[len(df_evaluation)] = 'busy period', np.mean(scores), np.std(scores)
 print('busy period: mean:', np.mean(scores), ', std:', np.std(scores))
NUM_FOLDS = 10
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)
 for f,(train_ind, val_ind)in tqdm(enumerate(kf.split(train, target))):
train_df, val_df = train.iloc[train_ind][columns], train.iloc[val_ind][columns]
train_target, val_target = target[train_ind], target[val_ind]
model = LGBMRegressor(**params)
 model.fit(train_df, train_target, eval_set=[(val_df,val_target)],early_stopping_rounds=1600,verbose=False)
 temp_oof = model.predict(val_df)
 temp_test = model.predict(test[columns])
 train_oof[val_ind] = temp_oof
test_preds += temp_test/NUM_FOLDS
print(mean_squared_error(temp_oof, val_target, squared=False))<compute_test_metric>
df_data['x_total_sq_ft'] =(df_data['BsmtFinSF1'] + df_data['BsmtFinSF2'] + df_data['1stFlrSF'] + df_data['2ndFlrSF'])
 df_data['x_total_bath'] =(df_data['FullBath'] +(0.5 * df_data['HalfBath'])+ df_data['BsmtFullBath'] +(0.5 * df_data['BsmtHalfBath']))
df_data['x_total_porch_sq_ft'] =(df_data['OpenPorchSF'] + df_data['3SsnPorch'] + df_data['EnclosedPorch'] + df_data['ScreenPorch'] + df_data['WoodDeckSF'])
 df_data['x_total_qual'] = df_data['OverallQual'] + df_data['OverallCond']
df_data['x_total_qual_mean'] =(df_data['OverallQual'] + df_data['OverallCond'])/ 2
mask_base =(df_data[target].notnull())
 x_cols = [c for c in df_data.columns if c[:2] == 'x_']
xgb = XGBRegressor()
scores = _evaluate(xgb, df_data[x_cols][mask_base], df_data[target][mask_base])
 df_evaluation.loc[len(df_evaluation)] = 'totals', np.mean(scores), np.std(scores)
 print('totals: mean:', np.mean(scores), ', std:', np.std(scores))
mean_squared_error(train_oof, target, squared=False)
<save_model>
df_data['x_sqft_per_room'] = df_data['GrLivArea'] /(df_data['TotRmsAbvGrd'] + df_data['FullBath'] + df_data['HalfBath'] + df_data['KitchenAbvGr'])
 mask_base =(df_data[target].notnull())
 x_cols = [c for c in df_data.columns if c[:2] == 'x_']
xgb = XGBRegressor()
scores = _evaluate(xgb, df_data[x_cols][mask_base], df_data[target][mask_base])
 df_evaluation.loc[len(df_evaluation)] = 'sqft room', np.mean(scores), np.std(scores)
 print('sqft room: mean:', np.mean(scores), ', std:', np.std(scores))
np.save('train_oof', train_oof)
 np.save('test_preds', test_preds )<save_to_csv>
df_data['x_age_built'] = df_data['YrSold'] - df_data['YearBuilt']
df_data['x_age_remod'] = df_data['YrSold'] - df_data['YearRemodAdd']
df_data['x_age_garage'] = df_data['YrSold'] - df_data['GarageYrBlt']
df_data['x_build_remod_sum'] = df_data['YearBuilt'] + df_data['YearRemodAdd']
df_data['x_build_remod_sub'] = df_data['YearRemodAdd'] - df_data['YearBuilt']
mask_base =(df_data[target].notnull())
 x_cols = [c for c in df_data.columns if c[:2] == 'x_']
xgb = XGBRegressor()
scores = _evaluate(xgb, df_data[x_cols][mask_base], df_data[target][mask_base])
 df_evaluation.loc[len(df_evaluation)] = 'property ages', np.mean(scores), np.std(scores)
 print('property ages: mean:', np.mean(scores), ', std:', np.std(scores))
sub['target'] = test_preds
sub.to_csv('submission.csv', index=False )<set_options>
q1 = df_data[target].quantile(0.25)
 q3 = df_data[target].quantile(0.75)
 iqr = q3 - q1
min_quartile = q1 - 1.5 * iqr
max_quartile = q3 + 1.5 * iqr
mask_iqr =(df_data[target].notnull())&(df_data[target] >= min_quartile)&(df_data[target] <= max_quartile)
 x_cols = [c for c in df_data.columns if c[:2] == 'x_']
xgb = XGBRegressor()
scores = _evaluate(xgb, df_data[x_cols][mask_iqr], df_data[target][mask_iqr])
 df_evaluation.loc[len(df_evaluation)] = 'iqr', np.mean(scores), np.std(scores)
 print('iqr: mean:', np.mean(scores), ', std:', np.std(scores))
%matplotlib inline<load_from_csv>
from sklearn.linear_model import LinearRegression
train = pd.read_csv(DATA / "train.csv")
 test = pd.read_csv(DATA / "test.csv")
 smpl_sub = pd.read_csv(DATA / "sample_submission.csv")
 print("train: {}, test: {}, sample sub: {}".format(
 train.shape, test.shape, smpl_sub.shape 
))<compute_test_metric>
param_xgb = {'n_estimators': 2820, 'max_depth': 5, 'learning_rate': 0.014605889674267689, 'min_child_weight': 2, 'gamma': 0, 'colsample_bytree': 0.15000000000000002, 'subsample': 0.71}
param_svr = {'C': 0.23086859625727427, 'epsilon': 0.006673842511350357, 'coef0': 0.8994782389230459}
param_ridge = {'alpha': 0.9999894695819446, 'tol': 0.8042543139861018}
param_lasso = {'alpha': 0.0002789651831563553}
param_lgbm = {'n_estimators': 289, 'num_leaves': 6, 'max_depth': 8, 'min_data_in_leaf': 32, 'learning_rate': 0.10777894336456172, 'min_child_weight': 0.4011559466054581}
param_rf = {'n_estimators': 285, 'max_depth': 13, 'min_samples_split': 3, 'min_samples_leaf': 3, 'max_features': 55, 'ccp_alpha': 7.135644710065878e-07}
model_xgb = XGBRegressor(**param_xgb, random_state=random_state)
 model_svr = SVR(**param_svr)
 model_ridge = Ridge(**param_ridge, random_state=random_state)
 model_lasso = Lasso(**param_lasso, random_state=random_state, tol=0.001)
 model_lgbm = LGBMRegressor(**param_lgbm, random_state=random_state)
 model_rf = RandomForestRegressor(**param_rf, random_state=random_state)
 model_stack = StackingCVRegressor(
 regressors=[model_xgb, model_svr, model_ridge, model_lasso, model_lgbm, model_rf],
meta_regressor=LinearRegression(n_jobs=-1),
cv=KFold(n_splits=4, random_state=random_state, shuffle=True),
n_jobs=-1,
random_state=random_state 
)
 mask_base =(df_data[target].notnull())
 x_cols = [c for c in df_data.columns if c[:2] == 'x_']
scores_xgb = _evaluate(model_xgb, df_data[x_cols][mask_base], df_data[target][mask_base])
 scores_svr = _evaluate(model_svr, df_data[x_cols][mask_base], df_data[target][mask_base])
 scores_ridge = _evaluate(model_ridge, df_data[x_cols][mask_base], df_data[target][mask_base])
 scores_lasso = _evaluate(model_lasso, df_data[x_cols][mask_base], df_data[target][mask_base])
 scores_lgbm = _evaluate(model_lgbm, df_data[x_cols][mask_base], df_data[target][mask_base])
 scores_rf = _evaluate(model_rf, df_data[x_cols][mask_base], df_data[target][mask_base])
 scores_stack = _evaluate(model_stack, df_data[x_cols][mask_base], df_data[target][mask_base])
 sns.set_style('whitegrid')
 fig, ax = plt.subplots(figsize=(11, 9))
scores = [scores_xgb, scores_svr, scores_ridge, scores_lasso, scores_lgbm, scores_rf, scores_stack]
models = ['xgb','svr','ridge','lasso','lgbm','rf','stack']
sns.boxplot(y=models, x=scores)
 for i in range(0, len(scores)) :
ax.text(np.max([np.max(b)for b in [a for a in scores]])* 1.02, i, str(np.mean(scores[i])) , va='center')
 plt.show()
def rmse(y_true, y_pred):

 return np.sqrt(np.mean(( y_true - y_pred)** 2))<prepare_x_and_y>
mask_base =(df_data[target].notnull())
 x_cols = [c for c in df_data.columns if c[:2] == 'x_']
model_xgb.fit(df_data[x_cols][mask_base].values, df_data[target][mask_base].values)
 model_svr.fit(df_data[x_cols][mask_base].values, df_data[target][mask_base].values)
 model_ridge.fit(df_data[x_cols][mask_base].values, df_data[target][mask_base].values)
 model_lasso.fit(df_data[x_cols][mask_base].values, df_data[target][mask_base].values)
 model_lgbm.fit(df_data[x_cols][mask_base].values, df_data[target][mask_base].values)
 model_rf.fit(df_data[x_cols][mask_base].values, df_data[target][mask_base].values)
 model_stack.fit(df_data[x_cols][mask_base].values, df_data[target][mask_base].values)
 gc.collect()
class TreeModel:

 def __init__(self, model_type: str):
self.model_type = model_type
self.trn_data = None
self.val_data = None
self.model = None
def train(self,
params: dict,
X_train: pd.DataFrame, y_train: np.ndarray,
X_val: pd.DataFrame, y_val: np.ndarray,
train_weight: tp.Optional[np.ndarray] = None,
val_weight: tp.Optional[np.ndarray] = None,
train_params: dict = None,
cat_cols: list = None, 
):
if self.model_type == "lgb":
self.trn_data = lgb.Dataset(X_train, label=y_train, weight=train_weight)
 self.val_data = lgb.Dataset(X_val, label=y_val, weight=val_weight)
 self.model = lgb.train(params=params,
train_set=self.trn_data,
valid_sets=[self.trn_data, self.val_data],
**train_params)
 elif self.model_type == "xgb":
self.trn_data = xgb.DMatrix(X_train, y_train, weight=train_weight)
 self.val_data = xgb.DMatrix(X_val, y_val, weight=val_weight)
 self.model = xgb.train(params=params,
dtrain=self.trn_data,
evals=[(self.trn_data, "train"),(self.val_data, "val")],
**train_params)
 elif self.model_type == "cat":
self.trn_data = Pool(
 X_train, label=y_train, cat_features=cat_cols)
 self.val_data = Pool(
 X_val, label=y_val, cat_features=cat_cols)
 self.model = CatBoost(params)
 self.model.fit(
 self.trn_data, eval_set=[self.val_data], use_best_model=True, **train_params)
 else:
raise NotImplementedError
def predict(self, X: pd.DataFrame):
if self.model_type == "lgb":
return self.model.predict(
 X, num_iteration=self.model.best_iteration)
 elif self.model_type == "xgb":
X_DM = xgb.DMatrix(X)
 return self.model.predict(
 X_DM, ntree_limit=self.model.best_ntree_limit)
 elif self.model_type == "cat":
return self.model.predict(X)
 else:
raise NotImplementedError
@property
def feature_names_(self):
if self.model_type == "lgb":
return self.model.feature_name()
elif self.model_type == "xgb":
return list(self.model.get_score(importance_type="gain" ).keys())
 elif self.model_type == "cat":
return self.model.feature_names_
else:
raise NotImplementedError
@property
def feature_importances_(self):
if self.model_type == "lgb":
return self.model.feature_importance(importance_type="gain")
 elif self.model_type == "xgb":
return list(self.model.get_score(importance_type="gain" ).values())
 elif self.model_type == "cat":
return self.model.feature_importances_
else:
raise NotImplementedError<define_variables>
def _blend_predictions(X):
return(( 0.2 * model_xgb.predict(X)) + \ 
(0.1 * model_svr.predict(X)) + \ 
(0.1 * model_ridge.predict(X)) + \ 
(0.1 * model_lasso.predict(X)) + \ 
(0.1 * model_lgbm.predict(X)) + \ 
(0.1 * model_rf.predict(X)) + \ 
(0.3 * model_stack.predict(np.array(X))))
ID_COL = "id"
CAT_COLS= [f"cat{i}" for i in range(10)]
CONT_COLS = [f"cont{i}" for i in range(14)]
TGT_COL = "target"
N_SPLITS = 10
RANDOM_SEED_LIST = [
42,
] 
<define_variables>
df_submission = df_data[['Id']][(df_data[target].isnull())].reset_index(drop=True ).copy()
df_submission['Prediction'] = model_stack.predict(df_data[x_cols][(df_data[target].isnull())].values)
 df_submission['SalePrice'] = np.exp(df_submission['Prediction'])
 df_submission[['Id','SalePrice']].to_csv('submission.csv', index=False )