Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<load_from_csv>
import eli5
import numpy as np
import pandas as pd
from sklearn import set_config
from sklearn.base import TransformerMixin
from sklearn.compose import ColumnTransformer
from sklearn.impute import SimpleImputer
from sklearn.metrics import mean_absolute_error, mean_squared_log_error, make_scorer
from sklearn.model_selection import GridSearchCV, cross_validate, cross_val_score
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import LabelEncoder, OneHotEncoder, FunctionTransformer
train = pd.read_csv(".. /input/application_train.csv")
 test = pd.read_csv(".. /input/application_test.csv")
 previous = pd.read_csv(".. /input/previous_application.csv")
 bureau = pd.read_csv(".. /input/bureau.csv" )<categorify>
np.random.seed(42 )
previous['AMT_APPLICATION'].replace(0,np.nan, inplace = True)
 previous['AMT_CREDIT'].replace(0,np.nan, inplace = True)
 previous['AMT_GOODS_PRICE'].replace(0,np.nan,inplace =True)
 previous['RATE_DOWN_PAYMENT'].replace(0, np.nan, inplace = True)
 previous['AMT_ANNUITY'].replace(0, np.nan, inplace = True)
 previous['CNT_PAYMENT'].replace(0, np.nan, inplace = True )<merge>
train_df = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test_df = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 full_df = pd.concat([train_df, test_df], sort=True ).reset_index(drop=True )
for i in ['Revolving loans','Cash loans', 'Consumer loans']:
tmp = previous[(previous['NAME_CONTRACT_TYPE'] == i)&(previous['DAYS_LAST_DUE'] == 365243)]
for df in [train,test]:
tmp1 = tmp.groupby(['SK_ID_CURR'])['SK_ID_PREV'].count().reset_index()
tmp1.columns = ['SK_ID_CURR','des']
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp1, on=['SK_ID_CURR'], how='left')
 df['count_notfinish_' + "_".join(i.lower().split())] = tmp_merge['des'].fillna(0 )<groupby>
def neg_rmsle(y_true, y_pred):
y_pred = np.abs(y_pred)
 return -1 * np.sqrt(mean_squared_log_error(y_true, y_pred))
for i in ['Revolving loans','Cash loans', 'Consumer loans']:
tmp = previous[(previous['NAME_CONTRACT_TYPE'] == i)&(previous['DAYS_LAST_DUE'] == 365243)]
for df in [train,test]:
tmp1 = tmp.groupby(['SK_ID_CURR'])['AMT_CREDIT'].agg({"returns": [np.mean, np.sum]})\ 
.reset_index()
tmp1.columns = ['SK_ID_CURR','des1','des2']
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp1, on=['SK_ID_CURR'], how='left')
 df['mean_notfinish_' + "_".join(i.lower().split())] = tmp_merge['des1'].fillna(0)
 df['sum_notfinish_' + "_".join(i.lower().split())] = tmp_merge['des2'].fillna(0 )<groupby>
def score_model(model, X, Y):
scores = cross_validate(
 model, X, Y,
scoring=['r2', 'neg_mean_absolute_error', 'neg_mean_squared_error'], cv=2,
n_jobs=-1, verbose=0)
 rmsle_score = cross_val_score(model, X, Y, cv=2, scoring=make_scorer(neg_rmsle))
mse_score = np.sqrt(-1 * scores['test_neg_mean_squared_error'].mean())
 mse_std = np.sqrt(scores['test_neg_mean_squared_error'].std())
 mae_score = -1 * scores['test_neg_mean_absolute_error'].mean()
mae_std = scores['test_neg_mean_absolute_error'].std()
r2_score_mean = scores['test_r2'].mean()
r2_std = scores['test_r2'].std()
print('[CV] RMSE: %.4f(%.4f)' %(mse_score, mse_std))
print('[CV] MAE: %.4f(%.4f)' %(mae_score, mae_std))
print('[CV] R^2: %.4f(%.4f)' %(r2_score_mean, r2_std))
print('[CV] RMSLE: %.6f(%.4f)' %(-1 * rmsle_score.mean() , rmsle_score.std()))
for i in ['Revolving loans','Cash loans', 'Consumer loans']:
tmp = previous[(previous['NAME_CONTRACT_TYPE'] == i)&(previous['DAYS_TERMINATION'] == 365243)]
for df in [train,test]:
tmp1 = tmp.groupby(['SK_ID_CURR'])['AMT_ANNUITY'].agg({"returns": [np.mean, np.sum]})\ 
.reset_index()
tmp1.columns = ['SK_ID_CURR','des1','des2']
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp1, on=['SK_ID_CURR'], how='left')
 df['mean_annuity_notfinish_' + "_".join(i.lower().split())] = tmp_merge['des1'].fillna(0)
 df['sum_annuity_notfinish_' + "_".join(i.lower().split())] = tmp_merge['des2'].fillna(0 )<feature_engineering>
def get_columns_from_transformer(column_transformer, input_colums):
col_name = []
for transformer_in_columns in column_transformer.transformers_[:-1]:
raw_col_name = transformer_in_columns[2]
if isinstance(transformer_in_columns[1],Pipeline):
transformer = transformer_in_columns[1].steps[-1][1]
else:
transformer = transformer_in_columns[1]
try:
names = transformer.get_feature_names(raw_col_name)
 except AttributeError:
names = raw_col_name
if isinstance(names,np.ndarray):
col_name += names.tolist()
elif isinstance(names,list):
col_name += names
elif isinstance(names,str):
col_name.append(names)
 [_, _, reminder_columns] = column_transformer.transformers_[-1]
for col_idx in reminder_columns:
col_name.append(input_colums[col_idx])
 return col_name
previous['SELLERPLACE_AREA'].replace(0, np.nan, inplace = True)
 previous['SELLERPLACE_AREA'].replace(-1, np.nan, inplace = True)
 previous['DAYS_TERMINATION'].replace(365243, np.nan, inplace = True)
 previous['DAYS_LAST_DUE'].replace(365243, np.nan, inplace = True)
 previous['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace = True)
 previous['sooner'] =(previous['DAYS_LAST_DUE_1ST_VERSION'] - previous['DAYS_LAST_DUE'])/(previous['DAYS_LAST_DUE_1ST_VERSION']-previous['DAYS_DECISION'])
 previous['duration'] = previous['DAYS_TERMINATION'] - previous['DAYS_DECISION']
previous['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace = True )<feature_engineering>
for feature in(
 'PoolQC',
'FireplaceQu',
'Alley',
'Fence',
'MiscFeature',
'BsmtQual',
'BsmtCond',
'BsmtExposure',
'BsmtFinType1',
'BsmtFinType2',
'GarageType',
'GarageFinish',
'GarageQual',
'GarageCond',
'BsmtQual',
'BsmtCond',
'BsmtExposure',
'BsmtFinType1',
'BsmtFinType2',
'MasVnrType', 
):
train_df[feature] = train_df[feature].fillna('None')
 test_df[feature] = test_df[feature].fillna('None')
 full_df[feature] = full_df[feature].fillna('None')
 for feature in(
 'BsmtFinSF1',
'BsmtFinSF2',
'BsmtUnfSF',
'TotalBsmtSF',
'BsmtFullBath',
'BsmtHalfBath',
'MasVnrArea',
'GarageCars',
'GarageArea',
'GarageYrBlt', 
):
train_df[feature] = train_df[feature].fillna(0)
 test_df[feature] = test_df[feature].fillna(0)
 full_df[feature] = full_df[feature].fillna(0)
 for feature in(
 'Electrical',
'KitchenQual',
'Exterior1st',
'Exterior2nd',
'SaleType',
'Utilities', 
):
train_df[feature] = train_df[feature].fillna(train_df[feature].mode() [0])
 test_df[feature] = test_df[feature].fillna(test_df[feature].mode() [0])
 full_df[feature] = full_df[feature].fillna(full_df[feature].mode() [0])
 for dataframe in [train_df, test_df, full_df]:
dataframe['MSZoning'] = dataframe.groupby(['Neighborhood'])['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
dataframe['MSSubClass'] = dataframe.groupby(['HouseStyle'])['MSSubClass'].transform(lambda x: x.fillna(x.mode() [0]))
dataframe['LotFrontage'] = dataframe.groupby(['Neighborhood', 'MSSubClass'])['LotFrontage'].transform(lambda x: x.fillna(x.median()))
dataframe['Functional'] = dataframe['Functional'].fillna('Typ' )
train['DAYS_EMPLOYED'] = train['DAYS_EMPLOYED'].replace(365243, np.nan)
 test['DAYS_EMPLOYED'] = test['DAYS_EMPLOYED'].replace(365243, np.nan)
 tmp = train[train['DAYS_LAST_PHONE_CHANGE'] >= 0].index
train['DAYS_LAST_PHONE_CHANGE'].iloc[tmp] = np.nan
tmp = test[test['DAYS_LAST_PHONE_CHANGE'] >= 0].index
test['DAYS_LAST_PHONE_CHANGE'].iloc[tmp] = np.nan
for df in [train, test]:
df['ORGANIZATION_TYPE_v2'] = df['ORGANIZATION_TYPE']
for i in range(1,4):
df['ORGANIZATION_TYPE_v2'].replace('Business Entity Type ' + str(i), 'Business', inplace = True)
 for i in range(1,14):
df['ORGANIZATION_TYPE_v2'].replace('Industry: type ' + str(i), 'Industry', inplace = True)
 for i in range(1,8):
df['ORGANIZATION_TYPE_v2'].replace('Trade: type ' + str(i), 'Trade', inplace = True)
 for i in range(1,8):
df['ORGANIZATION_TYPE_v2'].replace('Transport: type ' + str(i), 'Transport', inplace = True)
 df['ORGANIZATION_TYPE_v2'].replace('Other','XNA', inplace = True )<groupby>
num_features = [f for f in train_df.columns if train_df.dtypes[f] != 'object']
num_features.remove('Id')
 num_features.remove('SalePrice')
 cat_features = [f for f in train_df.columns if train_df.dtypes[f] == 'object']
tmp = previous[(previous['NAME_CONTRACT_STATUS'] == 'Approved')&(previous['NAME_CONTRACT_TYPE'].isin(['Consumer loans','Cash loans', 'Revolving loans'])) ].groupby(['SK_ID_CURR'])['AMT_CREDIT']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_amt_credit_master'] = tmp_merge['des1']
df['max_amt_credit_master'] = tmp_merge['des2']
df['mean_amt_credit_master'] = tmp_merge['des3']<groupby>
ordinal_feature_mapping = {
'ExterQual': {'Po': 0, 'Fa': 1, 'TA': 2, 'Gd': 3, 'Ex': 4},
'ExterCond': {'Po': 0, 'Fa': 1, 'TA': 2, 'Gd': 3, 'Ex': 4},
'BsmtQual': {'None': 0, 'Po': 1, 'Fa': 2, 'TA': 3, 'Gd': 4, 'Ex': 5},
'BsmtCond': {'None': 0, 'Po': 1, 'Fa': 2, 'TA': 3, 'Gd': 4, 'Ex': 5},
'BsmtFinType1': {'None': 0, 'Unf': 1, 'LwQ': 2, 'Rec': 3, 'BLQ': 4, 'ALQ': 5, 'GLQ': 6},
'BsmtFinType2': {'None': 0, 'Unf': 1, 'LwQ': 2, 'Rec': 3, 'BLQ': 4, 'ALQ': 5, 'GLQ': 6},
'HeatingQC': {'Po': 0, 'Fa': 1, 'TA': 2, 'Gd': 3, 'Ex': 4},
'KitchenQual': {'Po': 0, 'Fa': 1, 'TA': 2, 'Gd': 3, 'Ex': 4},
'FireplaceQu': {'None': 0, 'Po': 1, 'Fa': 2, 'TA': 3, 'Gd': 4, 'Ex': 5},
'GarageFinish': {'None': 0, 'Unf': 1, 'RFn': 2, 'Fin': 3},
'GarageQual': {'None': 0, 'Po': 1, 'Fa': 2, 'TA': 3, 'Gd': 4, 'Ex': 5},
'GarageCond': {'None': 0, 'Po': 1, 'Fa': 2, 'TA': 3, 'Gd': 4, 'Ex': 5},
'PoolQC': {'None': 0, 'Po': 1, 'Fa': 2, 'TA': 3, 'Gd': 4, 'Ex': 5},
'Fence': {'None': 0, 'MnWw': 1, 'GdWo': 2, 'MnPrv': 3, 'GdPrv': 4},
'PavedDrive': {'N': 0, 'P': 1, 'Y': 2},
'CentralAir': {'N': 0, 'Y': 1},
'Alley': {'None': 0, 'Pave': 1, 'Grvl': 2},
'Street': {'Pave': 0, 'Grvl': 1},
'BsmtExposure': {'None': 0, 'No': 1, 'Mn': 2, 'Av': 3, 'Gd': 4},
'Functional': {'Sal': 0, 'Sev': 1, 'Maj2': 2, 'Maj1': 3, 'Mod': 4, 'Min2': 5, 'Min1': 6, 'Typ': 7}
}
non_ordinal_cat_features = list(set(cat_features)- set(ordinal_feature_mapping.keys()))
for cat_feature in non_ordinal_cat_features:
train_df[cat_feature + 'Enc'] = LabelEncoder().fit_transform(train_df[cat_feature])
 test_df[cat_feature + 'Enc'] = LabelEncoder().fit_transform(test_df[cat_feature])
 full_df[cat_feature + 'Enc'] = LabelEncoder().fit_transform(full_df[cat_feature])
 for ordinal_feature, feature_mapping in ordinal_feature_mapping.items() :
train_df[ordinal_feature + 'Enc'] = train_df[ordinal_feature].map(feature_mapping)
 test_df[ordinal_feature + 'Enc'] = test_df[ordinal_feature].map(feature_mapping)
 full_df[ordinal_feature + 'Enc'] = full_df[ordinal_feature].map(feature_mapping )
tmp = previous[(previous['NAME_CONTRACT_STATUS'] == 'Approved')&(previous['NAME_CONTRACT_TYPE'].isin(['Cash loans'])) ].groupby(['SK_ID_CURR'])['AMT_APPLICATION']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_amt_app'] = tmp_merge['des1']
df['max_amt_app'] = tmp_merge['des2']
df['mean_amt_app'] = tmp_merge['des3']
tmp = previous[(previous['NAME_CONTRACT_STATUS'] == 'Approved')&(previous['NAME_CONTRACT_TYPE'].isin(['Consumer loans'])) ].groupby(['SK_ID_CURR'])['AMT_APPLICATION']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_amt_app_v1'] = tmp_merge['des1']
df['max_amt_app_v1'] = tmp_merge['des2']
df['mean_amt_app_v1'] = tmp_merge['des3']
tmp = previous[(previous['NAME_CONTRACT_STATUS'] == 'Approved')&(previous['NAME_CONTRACT_TYPE'].isin(['Revolving loans'])) ].groupby(['SK_ID_CURR'])['AMT_CREDIT']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_amt_card'] = tmp_merge['des1']
df['max_amt_card'] = tmp_merge['des2']
df['mean_amt_card'] = tmp_merge['des3']<groupby>
for dataframe in [train_df, test_df, full_df]:
dataframe['AvgRoomSF'] = dataframe['GrLivArea'] / dataframe['TotRmsAbvGrd']
dataframe['OverallHouseQC'] = dataframe['OverallQual'] + dataframe['OverallCond']
dataframe['IsNeighborhoodElite'] =(dataframe['Neighborhood'].isin(['NridgHt', 'CollgeCr', 'Crawfor', 'StoreBr', 'Timber'])) * 1
dataframe['NeighborhoodGroups'] = dataframe['Neighborhood'].map({
'MeadowV': 0,
'IDOTRR': 1,
'BrDale': 1,
'OldTown': 1,
'Edwards': 1,
'BrkSide': 1,
'Sawyer': 1,
'Blueste': 1,
'SWISU': 2,
'NAmes': 2,
'NPkVill': 2,
'Mitchel': 2,
'SawyerW': 2,
'Gilbert': 2,
'NWAmes': 2,
'Blmngtn': 2,
'CollgCr': 2,
'ClearCr': 3,
'Crawfor': 3,
'Veenker': 3,
'Somerst': 3,
'Timber': 3,
'StoneBr': 4,
'NoRidge': 4,
'NridgHt': 4,
})
 dataframe['HasFireplace'] = dataframe['Fireplaces'].apply(lambda x: int(x > 0))
dataframe['TotalBathrooms'] =(dataframe['FullBath'] +(0.5 * dataframe['HalfBath'])+
dataframe['BsmtFullBath'] +(0.5 * dataframe['BsmtHalfBath']))
dataframe['IsPavedDrive'] =(dataframe['PavedDrive'] == 'Y')* 1
year_built_bins = np.linspace(1871, 2010, 10)
 dataframe['YearBuiltBin'] = pd.cut(dataframe['YearBuilt'], bins=year_built_bins, labels=range(1, 10))
dataframe['HouseAge'] = dataframe['YrSold'] - dataframe['YearBuilt'].astype('int')
 dataframe['IsRecentlyBuilt'] =(full_df['YearBuilt'] == full_df['YrSold'].astype(int)) * 1
dataframe['IsRecentlyRemod'] =(full_df['YearRemodAdd'] == full_df['YrSold'].astype(int)) * 1
dataframe['ExterQC'] = dataframe['ExterQualEnc'] + dataframe['ExterCondEnc']
dataframe['FunctionalGroup'] = dataframe['Functional'].map({
'Typ': 2,
'Min1': 1,
'Min2': 1,
'Mod': 1,
'Maj1': 0,
'Maj2': 0,
'Sev': 0,
'Sav': 0,
})
 dataframe['IsGasHeating'] = dataframe['Heating'].map({'GasA': 1, 'GasW': 1, 'Grav': 0, 'Wall': 0, 'OthW': 0, 'Floor': 0})
 dataframe['IsHeatingGood'] = dataframe['HeatingQC'].map({'Po': 0, 'Fa': 0, 'TA': 0, 'Gd': 1, 'Ex': 1})
 dataframe['IsNewElectrBreakers'] = dataframe['Electrical'].map({'SBrkr': 1, 'FuseF': 0, 'FuseA': 0, 'FuseP': 0, 'Mix': 0})
 dataframe['IsGarageCondGood'] = dataframe['GarageCond'].map({'None': 0, 'Po': 0, 'Fa': 0, 'TA': 1, 'Gd': 1, 'Ex': 1})
 dataframe['RoofMatlCost'] = dataframe['RoofMatl'].map({
'CompShg': 0,
'WdShake': 1,
'ClyTile': 1,
'WdShngl': 1,
'Roll': 0,
'Metal': 1,
'Membran': 0,
'Tar&Grv': 0,
})
 dataframe['IsWoodenRoof'] = dataframe['RoofMatl'].map({
'CompShg': 0,
'WdShake': 1,
'ClyTile': 0,
'WdShngl': 1,
'Roll': 0,
'Metal': 0,
'Membran': 0,
'Tar&Grv': 0,
})
 dataframe['IsAdjArterialStreat'] =(( dataframe['Condition1'] == 'Artery')|(dataframe['Condition2'] == 'Artery')) * 1
dataframe['IsAdjFeederStreat'] =(( dataframe['Condition1'] == 'Feedr')|(dataframe['Condition2'] == 'Feedr')) * 1
dataframe['IsNormalCondition'] =(( dataframe['Condition1'] == 'Norm')|(dataframe['Condition2'] == 'Norm')) * 1
dataframe['IsAjdOffSiteFeature'] =(( dataframe['Condition1'] == 'PosA')|(dataframe['Condition2'] == 'PosA')) * 1
dataframe['IsNearOffSiteFeature'] =(( dataframe['Condition1'] == 'PosN')|(dataframe['Condition2'] == 'PosN')) * 1
dataframe['IsNearRailroad'] =(( dataframe['Condition1'].isin(['RRNn', 'RRNe'])) |(dataframe['Condition2'].isin(['RRNn', 'RRNe'])))* 1
dataframe['IsAdjRailroad'] =(( dataframe['Condition1'].isin(['RRAn', 'RRAe'])) |(dataframe['Condition2'].isin(['RRAn', 'RRAe'])))* 1
dataframe['WoodDeckGroups'] = pd.cut(dataframe['WoodDeckSF'], bins=[-1, 1, 200, 500, 2000], labels=[0, 1, 2, 3])
 dataframe['HasEnclosedPorch'] =(dataframe['EnclosedPorch'] > 0)* 1
dataframe['HasScreenPorch'] =(dataframe['ScreenPorch'] > 0)* 1
dataframe['HasEnclosedPorch'] =(dataframe['EnclosedPorch'] > 0)* 1
dataframe['Shed'] =(dataframe['MiscFeature'] == 'Shed')* 1 * full_df['MiscVal']
dataframe['Season'] = dataframe['MoSold'].map({
12: 0, 1: 0, 2: 0,
3: 1, 4: 1, 5: 1,
6: 2, 7: 2, 8: 2,
9: 3, 10: 3, 11: 3,
})
 dataframe['InflationFactor'] = dataframe['YrSold'] - 2006
tmp = previous[(previous['NAME_CONTRACT_STATUS'].isin(['Refused','Canceled'])) &(previous['NAME_CONTRACT_TYPE'].isin(['Cash loans'])) ].groupby(['SK_ID_CURR'])['AMT_APPLICATION']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_amt_app_fail'] = tmp_merge['des1']
df['max_amt_app_fail'] = tmp_merge['des2']
df['mean_amt_app_fail'] = tmp_merge['des3']
tmp = previous[(previous['NAME_CONTRACT_STATUS'].isin(['Refused','Canceled'])) &(previous['NAME_CONTRACT_TYPE'].isin(['Consumer loans'])) ].groupby(['SK_ID_CURR'])['AMT_APPLICATION']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_amt_app_v1_fail'] = tmp_merge['des1']
df['max_amt_app_v1_fail'] = tmp_merge['des2']
df['mean_amt_app_v1_fail'] = tmp_merge['des3']
tmp = previous[(previous['NAME_CONTRACT_STATUS'].isin(['Refused','Canceled'])) &(previous['NAME_CONTRACT_TYPE'].isin(['Revolving loans'])) ].groupby(['SK_ID_CURR'])['AMT_CREDIT']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_amt_card_fail'] = tmp_merge['des1']
df['max_amt_card_fail'] = tmp_merge['des2']
df['mean_amt_card_fail'] = tmp_merge['des3']<groupby>
features = [
'GrLivArea',
'1stFlrSF',
'2ndFlrSF',
'LotArea',
'BsmtFinSF1',
'BsmtFinSF2',
'BsmtUnfSF',
'BsmtFinType1Enc',
'BsmtFinType2Enc',
'GarageCars',
'OverallCond',
'Neighborhood',
'LotShape',
'LandSlope',
'BsmtCondEnc',
'BsmtQualEnc',
'SaleCondition',
'CentralAirEnc',
'IsAdjArterialStreat',
'IsAdjFeederStreat',
'IsNormalCondition',
'IsNearOffSiteFeature',
'IsAjdOffSiteFeature',
'IsAdjRailroad',
'TotalBathrooms',
'GarageFinishEnc',
'KitchenQualEnc',
'BedroomAbvGr',
'MSZoning',
'IsRecentlyBuilt',
'LandContour',
'HasFireplace',
'FunctionalGroup',
'HouseAge',
'FenceEnc',
'IsGasHeating',
'IsHeatingGood',
'IsNewElectrBreakers',
'IsGarageCondGood',
'IsWoodenRoof',
'RoofMatlCost',
'BldgType',
'HouseStyle',
'MasVnrType',
'TotRmsAbvGrd',
'WoodDeckGroups',
'HasEnclosedPorch',
'YearBuiltBin',
'HasScreenPorch',
'AvgRoomSF',
'BsmtExposureEnc',
'Shed',
'Season',
'NeighborhoodGroups',
'OverallHouseQC',
'ExterQC',
'InflationFactor',
'IsPavedDrive',
]
X = train_df[features]
Y = train_df['SalePrice']
x_test = test_df[features]
tmp = previous[(previous['NAME_CONTRACT_TYPE'].isin(['Cash loans'])) ].groupby(['SK_ID_CURR'])['RATE_DOWN_PAYMENT']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_amt_goods'] = tmp_merge['des1']
df['max_amt_goods'] = tmp_merge['des2']
df['mean_amt_goods'] = tmp_merge['des3']
tmp = previous[(previous['NAME_CONTRACT_TYPE'].isin(['Consumer loans'])) ].groupby(['SK_ID_CURR'])['RATE_DOWN_PAYMENT']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_amt_goods_v1'] = tmp_merge['des1']
df['max_amt_goods_v1'] = tmp_merge['des2']
df['mean_amt_goods_v1'] = tmp_merge['des3'] 
<groupby>
logTransformer = FunctionTransformer(func=np.log1p, inverse_func=np.expm1)
 featureTransformer = ColumnTransformer([ 
('log_scaling', logTransformer, ['GrLivArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'LotArea', 'AvgRoomSF', 'Shed', 'TotRmsAbvGrd']), 
('neighborhood_onehot', OneHotEncoder(categories=[neighborhoodCategories]), ['Neighborhood']), 
('neighborhood_grp_onehot', OneHotEncoder() , ['NeighborhoodGroups']), 
('lot_shape_onehot', OneHotEncoder(categories=[lotShapeCategories]), ['LotShape']), 
('land_slope_onehot', OneHotEncoder(categories=[landSlopeCategories]), ['LandSlope']), 
('sale_condition_onehot', OneHotEncoder(categories=[saleCondCategories]), ['SaleCondition']), 
('land_contour_onehot', OneHotEncoder() , ['LandContour']), 
('zoning_onehot', OneHotEncoder() , ['MSZoning']), 
('bldg_type_onehot', OneHotEncoder() , ['BldgType']), 
('masvrn_type_onehot', OneHotEncoder() , ['MasVnrType']), 
('house_style_onehot', OneHotEncoder() , ['HouseStyle']), 
('season_onehot', OneHotEncoder() , ['Season']),
],
remainder='passthrough'
)
tmp = previous[(previous['NAME_CONTRACT_TYPE'].isin(['Cash loans','Consumer loans'])) ].groupby(['SK_ID_CURR'])['AMT_ANNUITY']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_amt_annuity'] = tmp_merge['des1']
df['max_amt_annuity'] = tmp_merge['des2']
df['mean_amt_annuity'] = tmp_merge['des3']
tmp = previous[(previous['NAME_CONTRACT_TYPE'].isin(['Revolving loans'])) ].groupby(['SK_ID_CURR'])['AMT_ANNUITY']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_amt_card_annuity'] = tmp_merge['des1']
df['max_amt_card_annuity'] = tmp_merge['des2']
df['mean_amt_card_annuity'] = tmp_merge['des3']<groupby>
%%time
xgb_model = XGBRegressor(
 max_depth=6,
n_estimators=8000,
learning_rate=0.01,
min_child_weight=1.5,
subsample=0.2,
gamma=0.01,
reg_alpha=1,
reg_lambda=0.325,
objective='reg:gamma',
booster='gbtree' 
)
 xgb_pipeline = Pipeline([ 
('preprocessing', featureTransformer), 
('xgb_regressor', xgb_model),
])
 print('XGB Regressor:')
 score_model(xgb_pipeline, X, Y )
tmp = previous[(previous['NAME_CONTRACT_TYPE'].isin(['Cash loans','Consumer loans'])) ].groupby(['SK_ID_CURR'])['CNT_PAYMENT']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_cntpay'] = tmp_merge['des1']
df['max_cntpay'] = tmp_merge['des2']
df['mean_cntpay'] = tmp_merge['des3']
tmp = previous[(previous['NAME_CONTRACT_TYPE'].isin(['Consumer loans'])) ].groupby(['SK_ID_CURR'])['CNT_PAYMENT']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_cntpay_v1'] = tmp_merge['des1']
df['max_cntpay_v1'] = tmp_merge['des2']
df['mean_cntpay_v1'] = tmp_merge['des3'] 
<merge>
xgb_pipeline.fit(X, Y)
 X_columns = get_columns_from_transformer(xgb_pipeline.named_steps['preprocessing'], list(X.columns))
tmp = previous[(previous['NAME_CONTRACT_STATUS'] == 'Approved')&(previous['AMT_APPLICATION'] > 0)&(previous['NAME_CONTRACT_TYPE'].isin(['Cash loans','Consumer loans'])) ].sort_values(by=['SK_ID_CURR','DAYS_DECISION'])
 tmp = tmp.groupby(['SK_ID_CURR'] ).nth(-1 ).reset_index()
tmp = tmp[['SK_ID_CURR','AMT_APPLICATION']]
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['1st_recent_app'] = tmp_merge['des']
tmp = previous[(previous['NAME_CONTRACT_STATUS'] == 'Approved')&(previous['AMT_APPLICATION'] > 0)&(previous['NAME_CONTRACT_TYPE'].isin(['Cash loans','Consumer loans'])) ].sort_values(by=['SK_ID_CURR','DAYS_DECISION'])
 tmp = tmp.groupby(['SK_ID_CURR'] ).nth(-1 ).reset_index()
tmp = tmp[['SK_ID_CURR','AMT_CREDIT']]
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['1st_recent_credit'] = tmp_merge['des']
tmp = previous[(previous['NAME_CONTRACT_STATUS'] == 'Approved')&(previous['AMT_CREDIT'] > 0)&(previous['NAME_CONTRACT_TYPE'].isin(['Revolving loans'])) ].sort_values(by=['SK_ID_CURR','DAYS_DECISION'])
 tmp = tmp.groupby(['SK_ID_CURR'] ).nth(-1 ).reset_index()
tmp = tmp[['SK_ID_CURR','AMT_CREDIT']]
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['1st_recent_card'] = tmp_merge['des']<merge>
transformed_X = xgb_pipeline.named_steps['preprocessing'].transform(X)
 permutation_importance = PermutationImportance(
 xgb_model,
scoring=make_scorer(neg_rmsle),
cv=2,
random_state=42,
).fit(transformed_X, Y)
 eli5.show_weights(permutation_importance, feature_names=X_columns, top=125 )
tmp = previous[(previous['NAME_CONTRACT_STATUS'] != 'Approved')&(previous['AMT_APPLICATION'] > 0)&(previous['NAME_CONTRACT_TYPE'].isin(['Cash loans','Consumer loans'])) ].sort_values(by=['SK_ID_CURR','DAYS_DECISION'])
 tmp = tmp.groupby(['SK_ID_CURR'] ).nth(-1 ).reset_index()
tmp = tmp[['SK_ID_CURR','AMT_APPLICATION']]
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['1st_recent_app_fail'] = tmp_merge['des']
tmp = previous[(previous['NAME_CONTRACT_STATUS'] != 'Approved')&(previous['AMT_APPLICATION'] > 0)&(previous['NAME_CONTRACT_TYPE'].isin(['Cash loans','Consumer loans'])) ].sort_values(by=['SK_ID_CURR','DAYS_DECISION'])
 tmp = tmp.groupby(['SK_ID_CURR'] ).nth(-1 ).reset_index()
tmp = tmp[['SK_ID_CURR','AMT_CREDIT']]
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['1st_recent_credit_fail'] = tmp_merge['des']
tmp = previous[(previous['NAME_CONTRACT_STATUS'] != 'Approved')&(previous['AMT_CREDIT'] > 0)&(previous['NAME_CONTRACT_TYPE'].isin(['Revolving loans'])) ].sort_values(by=['SK_ID_CURR','DAYS_DECISION'])
 tmp = tmp.groupby(['SK_ID_CURR'] ).nth(-1 ).reset_index()
tmp = tmp[['SK_ID_CURR','AMT_CREDIT']]
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['1st_recent_card_fail'] = tmp_merge['des']<merge>
%%time
parameters = {
'xgb_regressor__objective': ['reg:gamma'],
'xgb_regressor__learning_rate': [0.01],
'xgb_regressor__n_estimators': [7900, 8000, 8100],
'xgb_regressor__max_depth': [11, 12, 13],
'xgb_regressor__booster': ['gbtree'],
'xgb_regressor__min_child_weight': [1.5],
'xgb_regressor__gamma': [0],
'xgb_regressor__subsample': [0.2],
'xgb_regressor__reg_alpha': [0, 0.9, 1],
'xgb_regressor__reg_lambda': [1, 0.3],
}
paramSearch = GridSearchCV(
 estimator=xgb_pipeline,
scoring=make_scorer(neg_rmsle),
param_grid=parameters,
cv=2,
n_jobs=-1,
verbose=3 
)
<merge><EOS> 
xgb_pipeline.fit(X, Y)
 y_test_predicted = xgb_pipeline.predict(x_test)
 y_test_predicted = np.rint(y_test_predicted ).astype(int)
 submission_df = pd.DataFrame({
'Id': test_df['Id'],
'SalePrice': y_test_predicted,
})
 submission_df.to_csv('./submission_xgb.csv', index=False )