Kaggle competition: Home Credit Default Risk
plt.style.use('seaborn')
 sns.set(font_scale=2.5)
 warnings.filterwarnings('ignore')
 %matplotlib inline<load_from_csv>
def geometric_mean(x):
return np.exp(np.log(x[x>0] ).mean())
 def one_hot_encoder(df, nan_as_category = True):
original_columns = list(df.columns)
 categorical_columns = [col for col in df.columns if df[col].dtype == 'object']
df = pd.get_dummies(df, columns= categorical_columns, dummy_na= nan_as_category)
 new_columns = [c for c in df.columns if c not in original_columns]
return df, new_columns
def application_train_test(num_rows = None, nan_as_category = False):
df = pd.read_csv('.. /input/application_train.csv', nrows= num_rows)
 test_df = pd.read_csv('.. /input/application_test.csv', nrows= num_rows)
 print("Train samples: {}, test samples: {}".format(len(df), len(test_df)))
 df = df.append(test_df ).reset_index()
df = df[df['CODE_GENDER'] != 'XNA']
docs = [_f for _f in df.columns if 'FLAG_DOC' in _f]
live = [_f for _f in df.columns if('FLAG_' in _f)&('FLAG_DOC' not in _f)&('_FLAG_' not in _f)]
df['DAYS_EMPLOYED'].replace(365243, np.nan, inplace= True)
 inc_by_org = df[['AMT_INCOME_TOTAL', 'ORGANIZATION_TYPE']].groupby('ORGANIZATION_TYPE' ).median() ['AMT_INCOME_TOTAL']
df['NEW_CREDIT_TO_ANNUITY_RATIO'] = df['AMT_CREDIT'] / df['AMT_ANNUITY']
df['NEW_CREDIT_TO_GOODS_RATIO'] = df['AMT_CREDIT'] / df['AMT_GOODS_PRICE']
df['NEW_DOC_IND_KURT'] = df[docs].kurtosis(axis=1)
 df['NEW_LIVE_IND_SUM'] = df[live].sum(axis=1)
 df['NEW_INC_PER_CHLD'] = df['AMT_INCOME_TOTAL'] /(1 + df['CNT_CHILDREN'])
 df['NEW_INC_BY_ORG'] = df['ORGANIZATION_TYPE'].map(inc_by_org)
 df['NEW_EMPLOY_TO_BIRTH_RATIO'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']
df['NEW_EMPLOY_TO_BIRTH-18_RATIO'] = df['DAYS_EMPLOYED'] /(df['DAYS_BIRTH'] + 18*365)
 df['NEW_BIRTH_TO_EMPLOY_RATIO'] = df['DAYS_BIRTH'] /(1 + df['DAYS_EMPLOYED'])
 df['NEW_INCOME_TO_ANNUITY_RATIO'] = df['AMT_INCOME_TOTAL'] /(1 + df['AMT_ANNUITY'])
 df['NEW_ANNUITY_TO_INCOME_RATIO'] = df['AMT_ANNUITY'] /(1 + df['AMT_INCOME_TOTAL'])
 df['NEW_EXT_SOURCES_MEDIAN'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].median(axis=1, skipna=True)
 df['NEW_EXT_SOURCES_MEAN'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].mean(axis=1, skipna=True)
 df['NEW_EXT_SOURCES_PROD'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].prod(axis=1, skipna=True, min_count=1)
 df['NEW_EXT_SOURCES_MAX'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].max(axis=1, skipna=True)
 df['NEW_EXT_SOURCES_MIN'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].min(axis=1, skipna=True)
 df['NEW_EXT_SOURCES_STD'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].std(axis=1, skipna=True)
 df['NEW_EXT_SOURCES_MAD'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].mad(axis=1, skipna=True)
 df['NEW_EXT_SOURCES_GEO'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].apply(geometric_mean, axis=1)
 df['NEW_CAR_TO_BIRTH_RATIO'] = df['OWN_CAR_AGE'] / df['DAYS_BIRTH']
df['NEW_CAR_TO_EMPLOY_RATIO'] = df['OWN_CAR_AGE'] / df['DAYS_EMPLOYED']
df['NEW_PHONE_TO_BIRTH_RATIO'] = df['DAYS_LAST_PHONE_CHANGE'] / df['DAYS_BIRTH']
df['NEW_PHONE_TO_EMPLOYED_RATIO'] = df['DAYS_LAST_PHONE_CHANGE'] / df['DAYS_EMPLOYED']
df['NEW_CREDIT_TO_INCOME_RATIO'] = df['AMT_CREDIT'] / df['AMT_INCOME_TOTAL']
df['NEW_PAYMENT_RATE'] = df['AMT_ANNUITY'] / df['AMT_CREDIT']
df['NEW_INCOME_CREDIT_PERC'] = df['AMT_INCOME_TOTAL'] / df['AMT_CREDIT']
df['NEW_INCOME_PER_PERSON'] = df['AMT_INCOME_TOTAL'] / df['CNT_FAM_MEMBERS']
for bin_feature in ['CODE_GENDER', 'FLAG_OWN_CAR', 'FLAG_OWN_REALTY']:
df[bin_feature], uniques = pd.factorize(df[bin_feature])
 df, cat_cols = one_hot_encoder(df, nan_as_category)
 dropcolum=['FLAG_DOCUMENT_2','FLAG_DOCUMENT_4',
'FLAG_DOCUMENT_5','FLAG_DOCUMENT_6','FLAG_DOCUMENT_7',
'FLAG_DOCUMENT_8','FLAG_DOCUMENT_9','FLAG_DOCUMENT_10',
'FLAG_DOCUMENT_11','FLAG_DOCUMENT_12','FLAG_DOCUMENT_13',
'FLAG_DOCUMENT_14','FLAG_DOCUMENT_15','FLAG_DOCUMENT_16',
'FLAG_DOCUMENT_17','FLAG_DOCUMENT_18','FLAG_DOCUMENT_19',
'FLAG_DOCUMENT_20','FLAG_DOCUMENT_21']
df= df.drop(dropcolum,axis=1)
 del test_df
gc.collect()
return df
def bureau_and_balance(num_rows = None, nan_as_category = True):
bureau = pd.read_csv('.. /input/bureau.csv', nrows = num_rows)
 bb = pd.read_csv('.. /input/bureau_balance.csv', nrows = num_rows)
 bb, bb_cat = one_hot_encoder(bb, nan_as_category)
 bureau, bureau_cat = one_hot_encoder(bureau, nan_as_category)
 bb_aggregations = {'MONTHS_BALANCE': ['min', 'max', 'size']}
for col in bb_cat:
bb_aggregations[col] = ['mean']
bb_agg = bb.groupby('SK_ID_BUREAU' ).agg(bb_aggregations)
 bb_agg.columns = pd.Index([e[0] + "_" + e[1].upper() for e in bb_agg.columns.tolist() ])
 bureau = bureau.join(bb_agg, how='left', on='SK_ID_BUREAU')
 bureau.drop(['SK_ID_BUREAU'], axis=1, inplace= True)
 del bb, bb_agg
gc.collect()
num_aggregations = {
'DAYS_CREDIT': [ 'mean', 'var'],
'DAYS_CREDIT_ENDDATE': [ 'mean'],
'DAYS_CREDIT_UPDATE': ['mean'],
'CREDIT_DAY_OVERDUE': ['mean'],
'AMT_CREDIT_MAX_OVERDUE': ['mean'],
'AMT_CREDIT_SUM': [ 'mean', 'sum'],
'AMT_CREDIT_SUM_DEBT': [ 'mean', 'sum'],
'AMT_CREDIT_SUM_OVERDUE': ['mean'],
'AMT_CREDIT_SUM_LIMIT': ['mean', 'sum'],
'AMT_ANNUITY': ['max', 'mean'],
'CNT_CREDIT_PROLONG': ['sum'],
'MONTHS_BALANCE_MIN': ['min'],
'MONTHS_BALANCE_MAX': ['max'],
'MONTHS_BALANCE_SIZE': ['mean', 'sum']
}
cat_aggregations = {}
for cat in bureau_cat: cat_aggregations[cat] = ['mean']
for cat in bb_cat: cat_aggregations[cat + "_MEAN"] = ['mean']
bureau_agg = bureau.groupby('SK_ID_CURR' ).agg({**num_aggregations, **cat_aggregations})
 bureau_agg.columns = pd.Index(['BURO_' + e[0] + "_" + e[1].upper() for e in bureau_agg.columns.tolist() ])
 active = bureau[bureau['CREDIT_ACTIVE_Active'] == 1]
active_agg = active.groupby('SK_ID_CURR' ).agg(num_aggregations)
 active_agg.columns = pd.Index(['ACTIVE_' + e[0] + "_" + e[1].upper() for e in active_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(active_agg, how='left', on='SK_ID_CURR')
 del active, active_agg
gc.collect()
closed = bureau[bureau['CREDIT_ACTIVE_Closed'] == 1]
closed_agg = closed.groupby('SK_ID_CURR' ).agg(num_aggregations)
 closed_agg.columns = pd.Index(['CLOSED_' + e[0] + "_" + e[1].upper() for e in closed_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(closed_agg, how='left', on='SK_ID_CURR')
 del closed, closed_agg, bureau
gc.collect()
return bureau_agg
def previous_applications(num_rows = None, nan_as_category = True):
prev = pd.read_csv('.. /input/previous_application.csv', nrows = num_rows)
 prev, cat_cols = one_hot_encoder(prev, nan_as_category= True)
 prev['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace= True)
 prev['DAYS_FIRST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_TERMINATION'].replace(365243, np.nan, inplace= True)
 prev['APP_CREDIT_PERC'] = prev['AMT_APPLICATION'] / prev['AMT_CREDIT']
num_aggregations = {
'AMT_ANNUITY': [ 'max', 'mean'],
'AMT_APPLICATION': [ 'max','mean'],
'AMT_CREDIT': [ 'max', 'mean'],
'APP_CREDIT_PERC': [ 'max', 'mean'],
'AMT_DOWN_PAYMENT': [ 'max', 'mean'],
'AMT_GOODS_PRICE': [ 'max', 'mean'],
'HOUR_APPR_PROCESS_START': [ 'max', 'mean'],
'RATE_DOWN_PAYMENT': [ 'max', 'mean'],
'DAYS_DECISION': [ 'max', 'mean'],
'CNT_PAYMENT': ['mean', 'sum'],
}
cat_aggregations = {}
for cat in cat_cols:
cat_aggregations[cat] = ['mean']
prev_agg = prev.groupby('SK_ID_CURR' ).agg({**num_aggregations, **cat_aggregations})
 prev_agg.columns = pd.Index(['PREV_' + e[0] + "_" + e[1].upper() for e in prev_agg.columns.tolist() ])
 approved = prev[prev['NAME_CONTRACT_STATUS_Approved'] == 1]
approved_agg = approved.groupby('SK_ID_CURR' ).agg(num_aggregations)
 approved_agg.columns = pd.Index(['APPROVED_' + e[0] + "_" + e[1].upper() for e in approved_agg.columns.tolist() ])
 prev_agg = prev_agg.join(approved_agg, how='left', on='SK_ID_CURR')
 refused = prev[prev['NAME_CONTRACT_STATUS_Refused'] == 1]
refused_agg = refused.groupby('SK_ID_CURR' ).agg(num_aggregations)
 refused_agg.columns = pd.Index(['REFUSED_' + e[0] + "_" + e[1].upper() for e in refused_agg.columns.tolist() ])
 prev_agg = prev_agg.join(refused_agg, how='left', on='SK_ID_CURR')
 del refused, refused_agg, approved, approved_agg, prev
gc.collect()
return prev_agg
def pos_cash(num_rows = None, nan_as_category = True):
pos = pd.read_csv('.. /input/POS_CASH_balance.csv', nrows = num_rows)
 pos, cat_cols = one_hot_encoder(pos, nan_as_category= True)
 aggregations = {
'MONTHS_BALANCE': ['max', 'mean', 'size'],
'SK_DPD': ['max', 'mean'],
'SK_DPD_DEF': ['max', 'mean']
}
for cat in cat_cols:
aggregations[cat] = ['mean']
pos_agg = pos.groupby('SK_ID_CURR' ).agg(aggregations)
 pos_agg.columns = pd.Index(['POS_' + e[0] + "_" + e[1].upper() for e in pos_agg.columns.tolist() ])
 pos_agg['POS_COUNT'] = pos.groupby('SK_ID_CURR' ).size()
del pos
gc.collect()
return pos_agg
def installments_payments(num_rows = None, nan_as_category = True):
ins = pd.read_csv('.. /input/installments_payments.csv', nrows = num_rows)
 ins, cat_cols = one_hot_encoder(ins, nan_as_category= True)
 ins['PAYMENT_PERC'] = ins['AMT_PAYMENT'] / ins['AMT_INSTALMENT']
ins['PAYMENT_DIFF'] = ins['AMT_INSTALMENT'] - ins['AMT_PAYMENT']
ins['DPD'] = ins['DAYS_ENTRY_PAYMENT'] - ins['DAYS_INSTALMENT']
ins['DBD'] = ins['DAYS_INSTALMENT'] - ins['DAYS_ENTRY_PAYMENT']
ins['DPD'] = ins['DPD'].apply(lambda x: x if x > 0 else 0)
 ins['DBD'] = ins['DBD'].apply(lambda x: x if x > 0 else 0)
 aggregations = {
'NUM_INSTALMENT_VERSION': ['nunique'],
'DPD': ['max', 'mean', 'sum'],
'DBD': ['max', 'mean', 'sum'],
'PAYMENT_PERC': [ 'mean', 'var'],
'PAYMENT_DIFF': [ 'mean', 'var'],
'AMT_INSTALMENT': ['max', 'mean', 'sum'],
'AMT_PAYMENT': ['min', 'max', 'mean', 'sum'],
'DAYS_ENTRY_PAYMENT': ['max', 'mean', 'sum']
}
for cat in cat_cols:
aggregations[cat] = ['mean']
ins_agg = ins.groupby('SK_ID_CURR' ).agg(aggregations)
 ins_agg.columns = pd.Index(['INSTAL_' + e[0] + "_" + e[1].upper() for e in ins_agg.columns.tolist() ])
 ins_agg['INSTAL_COUNT'] = ins.groupby('SK_ID_CURR' ).size()
del ins
gc.collect()
return ins_agg
def credit_card_balance(num_rows = None, nan_as_category = True):
cc = pd.read_csv('.. /input/credit_card_balance.csv', nrows = num_rows)
 cc, cat_cols = one_hot_encoder(cc, nan_as_category= True)
 cc.drop(['SK_ID_PREV'], axis= 1, inplace = True)
 cc_agg = cc.groupby('SK_ID_CURR' ).agg([ 'max', 'mean', 'sum', 'var'])
 cc_agg.columns = pd.Index(['CC_' + e[0] + "_" + e[1].upper() for e in cc_agg.columns.tolist() ])
 cc_agg['CC_COUNT'] = cc.groupby('SK_ID_CURR' ).size()
del cc
gc.collect()
return cc_agg
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 df_train[['Ticket', 'Survived']].groupby(['Survived'], as_index=True ).count()
df_train[['Ticket', 'Pclass']].groupby(['Pclass'], as_index=True ).count()<count_missing_values>
def kfold_lightgbm(train_df, train_target, test_df, num_folds, stratified=False, debug=False):
print("Starting LightGBM.Train shape: {}, test shape: {}".format(train_df.shape, test_df.shape))
if stratified:
folds = StratifiedKFold(n_splits=num_folds, shuffle=True, random_state=47)
 else:
folds = KFold(n_splits= num_folds, shuffle=True, random_state=47)
 oof_preds = np.zeros(train_df.shape[0])
 sub_preds = np.zeros(test_df.shape[0])
 feat_importance = pd.DataFrame()
scores = []
models = []
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(train_df, train_target)) :
train_x, train_y = train_df.iloc[train_idx], train_target.iloc[train_idx]
valid_x, valid_y = train_df.iloc[valid_idx], train_target.iloc[valid_idx]
clf = LGBMClassifier(
 nthread=4,
n_estimators=10000,
learning_rate=0.02,
num_leaves=32,
colsample_bytree=0.9497036,
subsample=0.8715623,
max_depth=8,
reg_alpha=0.04,
reg_lambda=0.073,
min_split_gain=0.0222415,
min_child_weight=40,
silent=-1,
verbose=-1, 
)
 clf.fit(train_x, train_y, eval_set=[(train_x, train_y),(valid_x, valid_y)],
eval_metric= 'auc', verbose= 1000, early_stopping_rounds= 300)
 oof_preds[valid_idx] = clf.predict_proba(valid_x, num_iteration=clf.best_iteration_)[:, 1]
sub_preds += clf.predict_proba(test_df, num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = test_df.columns.values
fold_importance_df["importance"] = clf.feature_importances_
fold_importance_df["shap_values"] = abs(shap.TreeExplainer(clf ).shap_values(valid_x)[:,:test_df.shape[1]] ).mean(axis=0 ).T
fold_importance_df["fold"] = n_fold + 1
feat_importance = pd.concat([feat_importance, fold_importance_df], axis=0)
 scores.append(roc_auc_score(valid_y, oof_preds[valid_idx]))
print('Fold %2d AUC : %.6f' %(n_fold + 1, scores[n_fold]))
models.append(clf)
 del clf, train_x, train_y, valid_x, valid_y, fold_importance_df
gc.collect()
score = roc_auc_score(train_target, oof_preds)
 print('Full AUC score %.6f' % score)
 print('Mean AUC score %.6f' % np.mean(scores))
if not debug:
pd.DataFrame(oof_preds ).to_csv("lgb{:03}_{:.5f}_train_oof.csv".format(test_df.shape[1], score), index=False)
 sub_df = pd.read_csv('.. /input/sample_submission.csv')
 sub_df['TARGET'] = sub_preds
sub_df.to_csv("lgb{:03}_{:.5f}.csv".format(test_df.shape[1], score), index= False)
 display_shapley_values(feat_importance)
 return feat_importance, models, scores
def display_importances(feat_importance):
best_features = feat_importance[["feature", "importance"]].groupby("feature")["importance"].agg(['mean', 'std'])\ 
.sort_values(by="mean", ascending=False ).head(40 ).reset_index()
best_features.columns = ["feature", "mean importance", "err"]
plt.figure(figsize=(8, 10))
sns.barplot(x="mean importance", y="feature", xerr=best_features['err'], data=best_features)
 plt.title('LightGBM Features(avg over folds)')
 plt.tight_layout()
plt.show()
def display_shapley_values(feat_importance):
best_features = feat_importance[["feature", "shap_values"]].groupby("feature")["shap_values"].agg(['mean', 'std'])\ 
.sort_values(by="mean", ascending=False ).head(40 ).reset_index()
best_features.columns = ["feature", "mean shapley values", "err"]
plt.figure(figsize=(8, 10))
sns.barplot(x="mean shapley values", y="feature", xerr=best_features['err'], data=best_features)
 plt.title('LightGBM shapley values(avg over folds)')
 plt.tight_layout()
plt.show()
for col in df_train.columns:
msg = 'column: {:>10}\t Percent of NaN value: {:.2f}%'.format(col, 100 *(df_train[col].isnull().sum() / df_train[col].shape[0]))
print(msg )<count_missing_values>
%%time
debug = False
num_rows = 10000 if debug else None
scores = {}
df = application_train_test(num_rows)
 with timer("Process bureau and bureau_balance"):
bureau = bureau_and_balance(num_rows)
 print("Bureau df shape:", bureau.shape)
 df = df.join(bureau, how='left', on='SK_ID_CURR')
 del bureau
gc.collect()
with timer("Process previous_applications"):
prev = previous_applications(num_rows)
 print("Previous applications df shape:", prev.shape)
 df = df.join(prev, how='left', on='SK_ID_CURR')
 del prev
gc.collect()
with timer("Process POS-CASH balance"):
pos = pos_cash(num_rows)
 print("Pos-cash balance df shape:", pos.shape)
 df = df.join(pos, how='left', on='SK_ID_CURR')
 del pos
gc.collect()
with timer("Process installments payments"):
ins = installments_payments(num_rows)
 print("Installments payments df shape:", ins.shape)
 df = df.join(ins, how='left', on='SK_ID_CURR')
 del ins
gc.collect()
with timer("Process credit card balance"):
cc = credit_card_balance(num_rows)
 print("Credit card balance df shape:", cc.shape)
 df = df.join(cc, how='left', on='SK_ID_CURR')
 del cc
gc.collect()
with timer("Save df"):
df.to_csv('merged_df.csv.gz', compression='gzip', index=False)
 with timer("Divide in training and test data"):
feats = [f for f in df.columns if f not in ['TARGET','SK_ID_CURR','SK_ID_BUREAU','SK_ID_PREV','index']]
train_df = df[df['TARGET'].notnull() ][feats]
train_target = df[df['TARGET'].notnull() ]['TARGET']
test_df = df[df['TARGET'].isnull() ][feats]
del df
gc.collect()
for col in df_test.columns:
msg = 'column: {:>10}\t Percent of NaN value: {:.2f}%'.format(col, 100 *(df_test[col].isnull().sum() / df_test[col].shape[0]))
print(msg )<count_missing_values>
%%time
feat_importance, models, scores = kfold_lightgbm(train_df, train_target, test_df, num_folds=5, stratified=False, debug=debug )
for col in df_test.columns:
msg = 'column: {:>10}\t Percent of NaN value: {:.2f}%'.format(col, 100 *(df_test[col].isnull().sum() / df_test[col].shape[0]))
print(msg )<feature_engineering>
def inv_logit(p): return np.exp(p)/(1 + np.exp(p))
base_value = shap_values[0,-1]
output = base_value + np.sum(shap_values[0,:-1])
 print('Log-odds output:', output, ' Logistic output:', inv_logit(output))
<feature_engineering><EOS> 
percentile = 0.15
best_features = feat_importance[["feature", "shap_values"]].groupby("feature")["shap_values"].agg(['mean'])\ 
.sort_values(by="mean", ascending=False ).reset_index()
best_features = best_features[:int(best_features.shape[0]*percentile)]["feature"].values
print("
with timer("Run LightGBM with kfold"):
train_df = train_df[best_features]
test_df = test_df[best_features]
feat_importance, models, scores = kfold_lightgbm(train_df, train_target, test_df, num_folds=5, stratified=False, debug=debug )