Kaggle competition: Give Me Some Credit
<SOS> metric: AUC Kaggle data source: give-me-some-credit<load_from_csv>
warnings.filterwarnings('ignore')
def previous_applications(num_rows = None, nan_as_category = True):
prev = pd.read_csv('.. /input/previous_application.csv', nrows = num_rows)
 prev, cat_cols = one_hot_encoder(prev, nan_as_category= True)
 prev['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace= True)
 prev['DAYS_FIRST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_TERMINATION'].replace(365243, np.nan, inplace= True)
 prev['APP_CREDIT_PERC'] = prev['AMT_APPLICATION'] / prev['AMT_CREDIT']
num_aggregations = {
'AMT_ANNUITY': ['min', 'max', 'mean'],
'AMT_APPLICATION': ['min', 'max', 'mean'],
'AMT_CREDIT': ['min', 'max', 'mean'],
'APP_CREDIT_PERC': ['min', 'max', 'mean', 'var'],
'AMT_DOWN_PAYMENT': ['min', 'max', 'mean'],
'AMT_GOODS_PRICE': ['min', 'max', 'mean'],
'HOUR_APPR_PROCESS_START': ['min', 'max', 'mean'],
'RATE_DOWN_PAYMENT': ['min', 'max', 'mean'],
'DAYS_DECISION': ['min', 'max', 'mean'],
'CNT_PAYMENT': ['mean', 'sum'],
}
cat_aggregations = {}
for cat in cat_cols:
cat_aggregations[cat] = ['mean']
prev_agg = prev.groupby('SK_ID_CURR' ).agg({**num_aggregations, **cat_aggregations})
 prev_agg.columns = pd.Index(['PREV_' + e[0] + "_" + e[1].upper() for e in prev_agg.columns.tolist() ])
 approved = prev[prev['NAME_CONTRACT_STATUS_Approved'] == 1]
approved_agg = approved.groupby('SK_ID_CURR' ).agg(num_aggregations)
 approved_agg.columns = pd.Index(['APR_' + e[0] + "_" + e[1].upper() for e in approved_agg.columns.tolist() ])
 prev_agg = prev_agg.join(approved_agg, how='left', on='SK_ID_CURR')
 refused = prev[prev['NAME_CONTRACT_STATUS_Refused'] == 1]
refused_agg = refused.groupby('SK_ID_CURR' ).agg(num_aggregations)
 refused_agg.columns = pd.Index(['REF_' + e[0] + "_" + e[1].upper() for e in refused_agg.columns.tolist() ])
 prev_agg = prev_agg.join(refused_agg, how='left', on='SK_ID_CURR')
 del refused, refused_agg, approved, approved_agg, prev
gc.collect()
return prev_agg<load_from_csv>
df_train = pd.read_csv('.. /input/GiveMeSomeCredit/cs-training.csv')
 df_test = pd.read_csv('.. /input/GiveMeSomeCredit/cs-test.csv')
 df_s = pd.read_csv('.. /input/GiveMeSomeCredit/sampleEntry.csv' )
def pos_cash(num_rows = None, nan_as_category = True):
pos = pd.read_csv('.. /input/POS_CASH_balance.csv', nrows = num_rows)
 pos, cat_cols = one_hot_encoder(pos, nan_as_category= True)
 aggregations = {
'MONTHS_BALANCE': ['max', 'mean', 'size'],
'SK_DPD': ['max', 'mean'],
'SK_DPD_DEF': ['max', 'mean']
}
for cat in cat_cols:
aggregations[cat] = ['mean']
pos_agg = pos.groupby('SK_ID_CURR' ).agg(aggregations)
 pos_agg.columns = pd.Index(['POS_' + e[0] + "_" + e[1].upper() for e in pos_agg.columns.tolist() ])
 pos_agg['POS_COUNT'] = pos.groupby('SK_ID_CURR' ).size()
del pos
gc.collect()
return pos_agg<load_from_csv>
df_train['Id'] = df_train['Unnamed: 0']
def bureau_and_balance(num_rows = None, nan_as_category = True):
bureau = pd.read_csv('.. /input/bureau.csv', nrows = num_rows)
 bb = pd.read_csv('.. /input/bureau_balance.csv', nrows = num_rows)
 bb, bb_cat = one_hot_encoder(bb, nan_as_category)
 bureau, bureau_cat = one_hot_encoder(bureau, nan_as_category)
 bb_aggregations = {'MONTHS_BALANCE': ['min', 'max', 'size']}
for col in bb_cat:
bb_aggregations[col] = ['mean']
bb_agg = bb.groupby('SK_ID_BUREAU' ).agg(bb_aggregations)
 bb_agg.columns = pd.Index([e[0] + "_" + e[1].upper() for e in bb_agg.columns.tolist() ])
 bureau = bureau.join(bb_agg, how='left', on='SK_ID_BUREAU')
 bureau.drop(columns= 'SK_ID_BUREAU', inplace= True)
 del bb, bb_agg
gc.collect()
num_aggregations = {
'DAYS_CREDIT': ['min', 'max', 'mean', 'var'],
'CREDIT_DAY_OVERDUE': ['max', 'mean'],
'DAYS_CREDIT_ENDDATE': ['min', 'max', 'mean'],
'AMT_CREDIT_MAX_OVERDUE': ['mean'],
'CNT_CREDIT_PROLONG': ['sum'],
'AMT_CREDIT_SUM': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_DEBT': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_OVERDUE': ['mean'],
'AMT_CREDIT_SUM_LIMIT': ['mean', 'sum'],
'DAYS_CREDIT_UPDATE': ['min', 'max', 'mean'],
'AMT_ANNUITY': ['max', 'mean'],
'MONTHS_BALANCE_MIN': ['min'],
'MONTHS_BALANCE_MAX': ['max'],
'MONTHS_BALANCE_SIZE': ['mean', 'sum']
}
cat_aggregations = {}
for cat in bureau_cat: cat_aggregations[cat] = ['mean']
for cat in bb_cat: cat_aggregations[cat + "_MEAN"] = ['mean']
bureau_agg = bureau.groupby('SK_ID_CURR' ).agg({**num_aggregations, **cat_aggregations})
 bureau_agg.columns = pd.Index(['BURO_' + e[0] + "_" + e[1].upper() for e in bureau_agg.columns.tolist() ])
 active = bureau[bureau['CREDIT_ACTIVE_Active'] == 1]
active_agg = active.groupby('SK_ID_CURR' ).agg(num_aggregations)
 active_agg.columns = pd.Index(['ACT_' + e[0] + "_" + e[1].upper() for e in active_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(active_agg, how='left', on='SK_ID_CURR')
 del active, active_agg
gc.collect()
closed = bureau[bureau['CREDIT_ACTIVE_Closed'] == 1]
closed_agg = closed.groupby('SK_ID_CURR' ).agg(num_aggregations)
 closed_agg.columns = pd.Index(['CLS_' + e[0] + "_" + e[1].upper() for e in closed_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(closed_agg, how='left', on='SK_ID_CURR')
 del closed, closed_agg, bureau
gc.collect()
return bureau_agg<load_from_csv>
df_train.drop('Unnamed: 0', axis=1, inplace=True )
def installments_payments(num_rows = None, nan_as_category = True):
ins = pd.read_csv('.. /input/installments_payments.csv', nrows = num_rows)
 ins, cat_cols = one_hot_encoder(ins, nan_as_category= True)
 ins['PAYMENT_PERC'] = ins['AMT_PAYMENT'] / ins['AMT_INSTALMENT']
ins['PAYMENT_DIFF'] = ins['AMT_INSTALMENT'] - ins['AMT_PAYMENT']
ins['DPD'] = ins['DAYS_ENTRY_PAYMENT'] - ins['DAYS_INSTALMENT']
ins['DBD'] = ins['DAYS_INSTALMENT'] - ins['DAYS_ENTRY_PAYMENT']
ins['DPD'] = ins['DPD'].apply(lambda x: x if x > 0 else 0)
 ins['DBD'] = ins['DBD'].apply(lambda x: x if x > 0 else 0)
 aggregations = {
'NUM_INSTALMENT_VERSION': ['nunique'],
'DPD': ['max', 'mean', 'sum'],
'DBD': ['max', 'mean', 'sum'],
'PAYMENT_PERC': ['max', 'mean', 'sum', 'var'],
'PAYMENT_DIFF': ['max', 'mean', 'sum', 'var'],
'AMT_INSTALMENT': ['max', 'mean', 'sum'],
'AMT_PAYMENT': ['min', 'max', 'mean', 'sum'],
'DAYS_ENTRY_PAYMENT': ['max', 'mean', 'sum']
}
for cat in cat_cols:
aggregations[cat] = ['mean']
ins_agg = ins.groupby('SK_ID_CURR' ).agg(aggregations)
 ins_agg.columns = pd.Index(['INS_' + e[0] + "_" + e[1].upper() for e in ins_agg.columns.tolist() ])
 ins_agg['INS_COUNT'] = ins.groupby('SK_ID_CURR' ).size()
del ins
gc.collect()
return ins_agg<load_from_csv>
df_test['Id'] = df_test['Unnamed: 0']
def credit_card_balance(num_rows = None, nan_as_category = True):
cc = pd.read_csv('.. /input/credit_card_balance.csv', nrows = num_rows)
 cc, cat_cols = one_hot_encoder(cc, nan_as_category= True)
 cc.drop(columns = ['SK_ID_PREV'], inplace = True)
 cc_agg = cc.groupby('SK_ID_CURR' ).agg(['min', 'max', 'mean', 'sum', 'var'])
 cc_agg.columns = pd.Index(['CC_' + e[0] + "_" + e[1].upper() for e in cc_agg.columns.tolist() ])
 cc_agg['CC_COUNT'] = cc.groupby('SK_ID_CURR' ).size()
del cc
gc.collect()
return cc_agg<train_model>
df_test.drop('Unnamed: 0', axis=1, inplace=True )
def kfold_lightgbm(df, num_folds, stratified = False):
train_df = df[df['TARGET'].notnull() ]
test_df = df[df['TARGET'].isnull() ]
print("Starting LightGBM.Train shape: {}, test shape: {}".format(train_df.shape, test_df.shape))
del df
gc.collect()
if stratified:
folds = StratifiedKFold(n_splits= num_folds, shuffle=True, random_state=1001)
 else:
folds = KFold(n_splits= num_folds, shuffle=True, random_state=1001)
 oof_preds = np.zeros(train_df.shape[0])
 sub_preds = np.zeros(test_df.shape[0])
 feature_importance_df = pd.DataFrame()
feats = [f for f in train_df.columns if f not in ['TARGET','SK_ID_CURR','SK_ID_BUREAU','SK_ID_PREV']]
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(train_df[feats], train_df['TARGET'])) :
train_x, train_y = train_df[feats].iloc[train_idx], train_df['TARGET'].iloc[train_idx]
valid_x, valid_y = train_df[feats].iloc[valid_idx], train_df['TARGET'].iloc[valid_idx]
clf = LGBMClassifier(
 nthread=4,
n_estimators=10000,
learning_rate=0.01,
num_leaves=40,
colsample_bytree=0.9497036,
subsample=0.8715623,
max_depth=8,
reg_alpha=0.041545473,
reg_lambda=0.0735294,
min_split_gain=0.0222415,
min_child_weight=39.3259775,
silent=-1,
verbose=-1,)
 clf.fit(train_x, train_y, eval_set=[(train_x, train_y),(valid_x, valid_y)],
eval_metric= 'auc', verbose= 100, early_stopping_rounds= 100)
 oof_preds[valid_idx] = clf.predict_proba(valid_x, num_iteration=clf.best_iteration_)[:, 1]
sub_preds += clf.predict_proba(test_df[feats], num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feats
fold_importance_df["importance"] = clf.feature_importances_
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 print('Fold %2d AUC : %.6f' %(n_fold + 1, roc_auc_score(valid_y, oof_preds[valid_idx])))
 del clf, train_x, train_y, valid_x, valid_y
gc.collect()
print('Full AUC score %.6f' % roc_auc_score(train_df['TARGET'], oof_preds))
test_df['TARGET'] = sub_preds
test_df[['SK_ID_CURR', 'TARGET']].to_csv(submission_file_name, index= False)
 display_importances(feature_importance_df)
 return feature_importance_df<merge>
df_train.isnull().sum()
def main(debug = False):
num_rows = 10000 if debug else None
df = application_train_test(num_rows)
 with timer("Process bureau and bureau_balance"):
bureau = bureau_and_balance(num_rows)
 print("Bureau df shape:", bureau.shape)
 df = df.join(bureau, how='left', on='SK_ID_CURR')
 del bureau
gc.collect()
with timer("Process previous_applications"):
prev = previous_applications(num_rows)
 print("Previous applications df shape:", prev.shape)
 df = df.join(prev, how='left', on='SK_ID_CURR')
 del prev
gc.collect()
with timer("Process POS-CASH balance"):
pos = pos_cash(num_rows)
 print("Pos-cash balance df shape:", pos.shape)
 df = df.join(pos, how='left', on='SK_ID_CURR')
 del pos
gc.collect()
with timer("Process installments payments"):
ins = installments_payments(num_rows)
 print("Installments payments df shape:", ins.shape)
 df = df.join(ins, how='left', on='SK_ID_CURR')
 del ins
gc.collect()
with timer("Process credit card balance"):
cc = credit_card_balance(num_rows)
 print("Credit card balance df shape:", cc.shape)
 df = df.join(cc, how='left', on='SK_ID_CURR')
 del cc
gc.collect()
with timer("Run LightGBM with kfold"):
feat_importance = kfold_lightgbm(df, num_folds= 5, stratified = False)
 if __name__ == "__main__":
submission_file_name = "submission_kernel26.csv"
with timer("Full model run"):
main(debug= False )<set_options>
df_train.nunique()
warnings.simplefilter(action='ignore', category=FutureWarning )<categorify>
df_test.isnull().sum()
def one_hot_encoder(df, nan_as_category = True):
original_columns = list(df.columns)
 categorical_columns = [col for col in df.columns if df[col].dtype == 'object']
df = pd.get_dummies(df, columns= categorical_columns, dummy_na= nan_as_category)
 new_columns = [c for c in df.columns if c not in original_columns]
return df, new_columns<load_from_csv>
df_train['MonthlyIncome'].fillna(df_train['MonthlyIncome'].mean() ,inplace=True )
def application_train_test(num_rows = None, nan_as_category = True):
df = pd.read_csv('.. /input/application_train.csv', nrows= num_rows)
 test_df = pd.read_csv('.. /input/application_test.csv', nrows= num_rows)
 print("Train samples: {}, test samples: {}".format(len(df), len(test_df)))
 df = df.append(test_df ).reset_index()
for bin_feature in ['CODE_GENDER', 'FLAG_OWN_CAR', 'FLAG_OWN_REALTY']:
df[bin_feature], uniques = pd.factorize(df[bin_feature])
 df, cat_cols = one_hot_encoder(df, nan_as_category)
 df['DAYS_EMPLOYED'].replace(365243, np.nan, inplace= True)
 df['DAYS_EMPLOYED_PERC'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']
df['INCOME_CREDIT_PERC'] = df['AMT_INCOME_TOTAL'] / df['AMT_CREDIT']
df['INCOME_PER_PERSON'] = df['AMT_INCOME_TOTAL'] / df['CNT_FAM_MEMBERS']
df['ANNUITY_INCOME_PERC'] = df['AMT_ANNUITY'] / df['AMT_INCOME_TOTAL']
del test_df
gc.collect()
return df<load_from_csv>
df_train['NumberOfDependents'].fillna(df_train['NumberOfDependents'].mode() [0], inplace=True )
def bureau_and_balance(num_rows = None, nan_as_category = True):
bureau = pd.read_csv('.. /input/bureau.csv', nrows = num_rows)
 bb = pd.read_csv('.. /input/bureau_balance.csv', nrows = num_rows)
 bb, bb_cat = one_hot_encoder(bb, nan_as_category)
 bureau, bureau_cat = one_hot_encoder(bureau, nan_as_category)
 bb_aggregations = {'MONTHS_BALANCE': ['min', 'max', 'size']}
for col in bb_cat:
bb_aggregations[col] = ['mean']
bb_agg = bb.groupby('SK_ID_BUREAU' ).agg(bb_aggregations)
 bb_agg.columns = pd.Index([e[0] + "_" + e[1].upper() for e in bb_agg.columns.tolist() ])
 bureau = bureau.join(bb_agg, how='left', on='SK_ID_BUREAU')
 bureau.drop(columns= 'SK_ID_BUREAU', inplace= True)
 del bb, bb_agg
gc.collect()
num_aggregations = {
'DAYS_CREDIT': ['min', 'max', 'mean', 'var'],
'CREDIT_DAY_OVERDUE': ['max', 'mean'],
'DAYS_CREDIT_ENDDATE': ['min', 'max', 'mean'],
'AMT_CREDIT_MAX_OVERDUE': ['mean'],
'CNT_CREDIT_PROLONG': ['sum'],
'AMT_CREDIT_SUM': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_DEBT': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_OVERDUE': ['mean'],
'AMT_CREDIT_SUM_LIMIT': ['mean', 'sum'],
'DAYS_CREDIT_UPDATE': ['min', 'max', 'mean'],
'AMT_ANNUITY': ['max', 'mean'],
'MONTHS_BALANCE_MIN': ['min'],
'MONTHS_BALANCE_MAX': ['max'],
'MONTHS_BALANCE_SIZE': ['mean', 'sum']
}
cat_aggregations = {}
for cat in bureau_cat: cat_aggregations[cat] = ['mean']
for cat in bb_cat: cat_aggregations[cat + "_MEAN"] = ['mean']
bureau_agg = bureau.groupby('SK_ID_CURR' ).agg({**num_aggregations, **cat_aggregations})
 bureau_agg.columns = pd.Index(['BURO_' + e[0] + "_" + e[1].upper() for e in bureau_agg.columns.tolist() ])
 active = bureau[bureau['CREDIT_ACTIVE_Active'] == 1]
active_agg = active.groupby('SK_ID_CURR' ).agg(num_aggregations)
 active_agg.columns = pd.Index(['ACT_' + e[0] + "_" + e[1].upper() for e in active_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(active_agg, how='left', on='SK_ID_CURR')
 del active, active_agg
gc.collect()
closed = bureau[bureau['CREDIT_ACTIVE_Closed'] == 1]
closed_agg = closed.groupby('SK_ID_CURR' ).agg(num_aggregations)
 closed_agg.columns = pd.Index(['CLS_' + e[0] + "_" + e[1].upper() for e in closed_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(closed_agg, how='left', on='SK_ID_CURR')
 del closed, closed_agg, bureau
gc.collect()
return bureau_agg 
<load_from_csv>
df_test['MonthlyIncome'].fillna(df_test['MonthlyIncome'].mean() ,inplace=True )
def previous_applications(num_rows = None, nan_as_category = True):
prev = pd.read_csv('.. /input/previous_application.csv', nrows = num_rows)
 prev, cat_cols = one_hot_encoder(prev, nan_as_category= True)
 prev['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace= True)
 prev['DAYS_FIRST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_TERMINATION'].replace(365243, np.nan, inplace= True)
 prev['APP_CREDIT_PERC'] = prev['AMT_APPLICATION'] / prev['AMT_CREDIT']
num_aggregations = {
'AMT_ANNUITY': ['min', 'max', 'mean'],
'AMT_APPLICATION': ['min', 'max', 'mean'],
'AMT_CREDIT': ['min', 'max', 'mean'],
'APP_CREDIT_PERC': ['min', 'max', 'mean', 'var'],
'AMT_DOWN_PAYMENT': ['min', 'max', 'mean'],
'AMT_GOODS_PRICE': ['min', 'max', 'mean'],
'HOUR_APPR_PROCESS_START': ['min', 'max', 'mean'],
'RATE_DOWN_PAYMENT': ['min', 'max', 'mean'],
'DAYS_DECISION': ['min', 'max', 'mean'],
'CNT_PAYMENT': ['mean', 'sum'],
}
cat_aggregations = {}
for cat in cat_cols:
cat_aggregations[cat] = ['mean']
prev_agg = prev.groupby('SK_ID_CURR' ).agg({**num_aggregations, **cat_aggregations})
 prev_agg.columns = pd.Index(['PREV_' + e[0] + "_" + e[1].upper() for e in prev_agg.columns.tolist() ])
 approved = prev[prev['NAME_CONTRACT_STATUS_Approved'] == 1]
approved_agg = approved.groupby('SK_ID_CURR' ).agg(num_aggregations)
 approved_agg.columns = pd.Index(['APR_' + e[0] + "_" + e[1].upper() for e in approved_agg.columns.tolist() ])
 prev_agg = prev_agg.join(approved_agg, how='left', on='SK_ID_CURR')
 refused = prev[prev['NAME_CONTRACT_STATUS_Refused'] == 1]
refused_agg = refused.groupby('SK_ID_CURR' ).agg(num_aggregations)
 refused_agg.columns = pd.Index(['REF_' + e[0] + "_" + e[1].upper() for e in refused_agg.columns.tolist() ])
 prev_agg = prev_agg.join(refused_agg, how='left', on='SK_ID_CURR')
 del refused, refused_agg, approved, approved_agg, prev
gc.collect()
return prev_agg<load_from_csv>
df_test['NumberOfDependents'].fillna(df_test['NumberOfDependents'].mode() [0], inplace=True )
def pos_cash(num_rows = None, nan_as_category = True):
pos = pd.read_csv('.. /input/POS_CASH_balance.csv', nrows = num_rows)
 pos, cat_cols = one_hot_encoder(pos, nan_as_category= True)
 aggregations = {
'MONTHS_BALANCE': ['max', 'mean', 'size'],
'SK_DPD': ['max', 'mean'],
'SK_DPD_DEF': ['max', 'mean']
}
for cat in cat_cols:
aggregations[cat] = ['mean']
pos_agg = pos.groupby('SK_ID_CURR' ).agg(aggregations)
 pos_agg.columns = pd.Index(['POS_' + e[0] + "_" + e[1].upper() for e in pos_agg.columns.tolist() ])
 pos_agg['POS_COUNT'] = pos.groupby('SK_ID_CURR' ).size()
del pos
gc.collect()
return pos_agg<load_from_csv>
df_train.isnull().sum()
def bureau_and_balance(num_rows = None, nan_as_category = True):
bureau = pd.read_csv('.. /input/bureau.csv', nrows = num_rows)
 bb = pd.read_csv('.. /input/bureau_balance.csv', nrows = num_rows)
 bb, bb_cat = one_hot_encoder(bb, nan_as_category)
 bureau, bureau_cat = one_hot_encoder(bureau, nan_as_category)
 bb_aggregations = {'MONTHS_BALANCE': ['min', 'max', 'size']}
for col in bb_cat:
bb_aggregations[col] = ['mean']
bb_agg = bb.groupby('SK_ID_BUREAU' ).agg(bb_aggregations)
 bb_agg.columns = pd.Index([e[0] + "_" + e[1].upper() for e in bb_agg.columns.tolist() ])
 bureau = bureau.join(bb_agg, how='left', on='SK_ID_BUREAU')
 bureau.drop(columns= 'SK_ID_BUREAU', inplace= True)
 del bb, bb_agg
gc.collect()
num_aggregations = {
'DAYS_CREDIT': ['min', 'max', 'mean', 'var'],
'CREDIT_DAY_OVERDUE': ['max', 'mean'],
'DAYS_CREDIT_ENDDATE': ['min', 'max', 'mean'],
'AMT_CREDIT_MAX_OVERDUE': ['mean'],
'CNT_CREDIT_PROLONG': ['sum'],
'AMT_CREDIT_SUM': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_DEBT': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_OVERDUE': ['mean'],
'AMT_CREDIT_SUM_LIMIT': ['mean', 'sum'],
'DAYS_CREDIT_UPDATE': ['min', 'max', 'mean'],
'AMT_ANNUITY': ['max', 'mean'],
'MONTHS_BALANCE_MIN': ['min'],
'MONTHS_BALANCE_MAX': ['max'],
'MONTHS_BALANCE_SIZE': ['mean', 'sum']
}
cat_aggregations = {}
for cat in bureau_cat: cat_aggregations[cat] = ['mean']
for cat in bb_cat: cat_aggregations[cat + "_MEAN"] = ['mean']
bureau_agg = bureau.groupby('SK_ID_CURR' ).agg({**num_aggregations, **cat_aggregations})
 bureau_agg.columns = pd.Index(['BURO_' + e[0] + "_" + e[1].upper() for e in bureau_agg.columns.tolist() ])
 active = bureau[bureau['CREDIT_ACTIVE_Active'] == 1]
active_agg = active.groupby('SK_ID_CURR' ).agg(num_aggregations)
 active_agg.columns = pd.Index(['ACT_' + e[0] + "_" + e[1].upper() for e in active_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(active_agg, how='left', on='SK_ID_CURR')
 del active, active_agg
gc.collect()
closed = bureau[bureau['CREDIT_ACTIVE_Closed'] == 1]
closed_agg = closed.groupby('SK_ID_CURR' ).agg(num_aggregations)
 closed_agg.columns = pd.Index(['CLS_' + e[0] + "_" + e[1].upper() for e in closed_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(closed_agg, how='left', on='SK_ID_CURR')
 del closed, closed_agg, bureau
gc.collect()
return bureau_agg<load_from_csv>
df_test.isnull().sum()
def installments_payments(num_rows = None, nan_as_category = True):
ins = pd.read_csv('.. /input/installments_payments.csv', nrows = num_rows)
 ins, cat_cols = one_hot_encoder(ins, nan_as_category= True)
 ins['PAYMENT_PERC'] = ins['AMT_PAYMENT'] / ins['AMT_INSTALMENT']
ins['PAYMENT_DIFF'] = ins['AMT_INSTALMENT'] - ins['AMT_PAYMENT']
ins['DPD'] = ins['DAYS_ENTRY_PAYMENT'] - ins['DAYS_INSTALMENT']
ins['DBD'] = ins['DAYS_INSTALMENT'] - ins['DAYS_ENTRY_PAYMENT']
ins['DPD'] = ins['DPD'].apply(lambda x: x if x > 0 else 0)
 ins['DBD'] = ins['DBD'].apply(lambda x: x if x > 0 else 0)
 aggregations = {
'NUM_INSTALMENT_VERSION': ['nunique'],
'DPD': ['max', 'mean', 'sum'],
'DBD': ['max', 'mean', 'sum'],
'PAYMENT_PERC': ['max', 'mean', 'sum', 'var'],
'PAYMENT_DIFF': ['max', 'mean', 'sum', 'var'],
'AMT_INSTALMENT': ['max', 'mean', 'sum'],
'AMT_PAYMENT': ['min', 'max', 'mean', 'sum'],
'DAYS_ENTRY_PAYMENT': ['max', 'mean', 'sum']
}
for cat in cat_cols:
aggregations[cat] = ['mean']
ins_agg = ins.groupby('SK_ID_CURR' ).agg(aggregations)
 ins_agg.columns = pd.Index(['INS_' + e[0] + "_" + e[1].upper() for e in ins_agg.columns.tolist() ])
 ins_agg['INS_COUNT'] = ins.groupby('SK_ID_CURR' ).size()
del ins
gc.collect()
return ins_agg<load_from_csv>
Id = df_test['Id']
def credit_card_balance(num_rows = None, nan_as_category = True):
cc = pd.read_csv('.. /input/credit_card_balance.csv', nrows = num_rows)
 cc, cat_cols = one_hot_encoder(cc, nan_as_category= True)
 cc.drop(columns = ['SK_ID_PREV'], inplace = True)
 cc_agg = cc.groupby('SK_ID_CURR' ).agg(['min', 'max', 'mean', 'sum', 'var'])
 cc_agg.columns = pd.Index(['CC_' + e[0] + "_" + e[1].upper() for e in cc_agg.columns.tolist() ])
 cc_agg['CC_COUNT'] = cc.groupby('SK_ID_CURR' ).size()
del cc
gc.collect()
return cc_agg<train_model>
df_train.drop('Id', axis=1, inplace=True )
def kfold_lightgbm(df, num_folds, stratified = False):
train_df = df[df['TARGET'].notnull() ]
test_df = df[df['TARGET'].isnull() ]
print("Starting LightGBM.Train shape: {}, test shape: {}".format(train_df.shape, test_df.shape))
del df
gc.collect()
if stratified:
folds = StratifiedKFold(n_splits= num_folds, shuffle=True, random_state=1001)
 else:
folds = KFold(n_splits= num_folds, shuffle=True, random_state=1001)
 oof_preds = np.zeros(train_df.shape[0])
 sub_preds = np.zeros(test_df.shape[0])
 feature_importance_df = pd.DataFrame()
feats = [f for f in train_df.columns if f not in ['TARGET','SK_ID_CURR','SK_ID_BUREAU','SK_ID_PREV']]
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(train_df[feats], train_df['TARGET'])) :
train_x, train_y = train_df[feats].iloc[train_idx], train_df['TARGET'].iloc[train_idx]
valid_x, valid_y = train_df[feats].iloc[valid_idx], train_df['TARGET'].iloc[valid_idx]
clf = LGBMClassifier(
 nthread=4,
n_estimators=10000,
learning_rate=0.02,
num_leaves=34,
colsample_bytree=0.9497036,
subsample=0.8715623,
max_depth=8,
reg_alpha=0.041545473,
reg_lambda=0.0735294,
min_split_gain=0.0222415,
min_child_weight=39.3259775,
silent=-1,
verbose=-1,)
 clf.fit(train_x, train_y, eval_set=[(train_x, train_y),(valid_x, valid_y)],
eval_metric= 'auc', verbose= 100, early_stopping_rounds= 100)
 oof_preds[valid_idx] = clf.predict_proba(valid_x, num_iteration=clf.best_iteration_)[:, 1]
sub_preds += clf.predict_proba(test_df[feats], num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feats
fold_importance_df["importance"] = clf.feature_importances_
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 print('Fold %2d AUC : %.6f' %(n_fold + 1, roc_auc_score(valid_y, oof_preds[valid_idx])))
 del clf, train_x, train_y, valid_x, valid_y
gc.collect()
print('Full AUC score %.6f' % roc_auc_score(train_df['TARGET'], oof_preds))
test_df['TARGET'] = sub_preds
test_df[['SK_ID_CURR', 'TARGET']].to_csv(submission_file_name, index= False)
 display_importances(feature_importance_df)
 return feature_importance_df<merge>
df_test.drop('Id', axis=1, inplace=True )
def main(debug = False):
num_rows = 10000 if debug else None
df = application_train_test(num_rows)
 with timer("Process bureau and bureau_balance"):
bureau = bureau_and_balance(num_rows)
 print("Bureau df shape:", bureau.shape)
 df = df.join(bureau, how='left', on='SK_ID_CURR')
 del bureau
gc.collect()
with timer("Process previous_applications"):
prev = previous_applications(num_rows)
 print("Previous applications df shape:", prev.shape)
 df = df.join(prev, how='left', on='SK_ID_CURR')
 del prev
gc.collect()
with timer("Process POS-CASH balance"):
pos = pos_cash(num_rows)
 print("Pos-cash balance df shape:", pos.shape)
 df = df.join(pos, how='left', on='SK_ID_CURR')
 del pos
gc.collect()
with timer("Process installments payments"):
ins = installments_payments(num_rows)
 print("Installments payments df shape:", ins.shape)
 df = df.join(ins, how='left', on='SK_ID_CURR')
 del ins
gc.collect()
with timer("Process credit card balance"):
cc = credit_card_balance(num_rows)
 print("Credit card balance df shape:", cc.shape)
 df = df.join(cc, how='left', on='SK_ID_CURR')
 del cc
gc.collect()
with timer("Run LightGBM with kfold"):
feat_importance = kfold_lightgbm(df, num_folds= 5, stratified = True)
 if __name__ == "__main__":
submission_file_name = "submission_kernel26.csv"
with timer("Full model run"):
main(debug= False )<set_options>
X = df_train.drop('SeriousDlqin2yrs',axis=1)
 y = df_train['SeriousDlqin2yrs']
warnings.simplefilter(action='ignore', category=FutureWarning )<categorify>
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, classification_report
from sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score
from sklearn.model_selection import GridSearchCV, cross_val_score, RandomizedSearchCV
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from xgboost import XGBClassifier
def one_hot_encoder(df, nan_as_category = True):
original_columns = list(df.columns)
 categorical_columns = [col for col in df.columns if df[col].dtype == 'object']
df = pd.get_dummies(df, columns= categorical_columns, dummy_na= nan_as_category)
 new_columns = [c for c in df.columns if c not in original_columns]
return df, new_columns<load_from_csv>
X_train, X_test, y_train, y_test = train_test_split(X.values, y.values, test_size=0.2, random_state=116214 )
def application_train_test(num_rows = None, nan_as_category = True):
df = pd.read_csv('.. /input/application_train.csv', nrows= num_rows)
 test_df = pd.read_csv('.. /input/application_test.csv', nrows= num_rows)
 print("Train samples: {}, test samples: {}".format(len(df), len(test_df)))
 df = df.append(test_df ).reset_index()
for bin_feature in ['CODE_GENDER', 'FLAG_OWN_CAR', 'FLAG_OWN_REALTY']:
df[bin_feature], uniques = pd.factorize(df[bin_feature])
 df, cat_cols = one_hot_encoder(df, nan_as_category)
 df['DAYS_EMPLOYED'].replace(365243, np.nan, inplace= True)
 df['DAYS_EMPLOYED_PERC'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']
df['INCOME_CREDIT_PERC'] = df['AMT_INCOME_TOTAL'] / df['AMT_CREDIT']
df['INCOME_PER_PERSON'] = df['AMT_INCOME_TOTAL'] / df['CNT_FAM_MEMBERS']
df['ANNUITY_INCOME_PERC'] = df['AMT_ANNUITY'] / df['AMT_INCOME_TOTAL']
del test_df
gc.collect()
return df<load_from_csv>
RF = RandomForestClassifier()
def bureau_and_balance(num_rows = None, nan_as_category = True):
bureau = pd.read_csv('.. /input/bureau.csv', nrows = num_rows)
 bb = pd.read_csv('.. /input/bureau_balance.csv', nrows = num_rows)
 bb, bb_cat = one_hot_encoder(bb, nan_as_category)
 bureau, bureau_cat = one_hot_encoder(bureau, nan_as_category)
 bb_aggregations = {'MONTHS_BALANCE': ['min', 'max', 'size']}
for col in bb_cat:
bb_aggregations[col] = ['mean']
bb_agg = bb.groupby('SK_ID_BUREAU' ).agg(bb_aggregations)
 bb_agg.columns = pd.Index([e[0] + "_" + e[1].upper() for e in bb_agg.columns.tolist() ])
 bureau = bureau.join(bb_agg, how='left', on='SK_ID_BUREAU')
 bureau.drop(columns= 'SK_ID_BUREAU', inplace= True)
 del bb, bb_agg
gc.collect()
num_aggregations = {
'DAYS_CREDIT': ['min', 'max', 'mean', 'var'],
'CREDIT_DAY_OVERDUE': ['max', 'mean'],
'DAYS_CREDIT_ENDDATE': ['min', 'max', 'mean'],
'AMT_CREDIT_MAX_OVERDUE': ['mean'],
'CNT_CREDIT_PROLONG': ['sum'],
'AMT_CREDIT_SUM': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_DEBT': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_OVERDUE': ['mean'],
'AMT_CREDIT_SUM_LIMIT': ['mean', 'sum'],
'DAYS_CREDIT_UPDATE': ['min', 'max', 'mean'],
'AMT_ANNUITY': ['max', 'mean'],
'MONTHS_BALANCE_MIN': ['min'],
'MONTHS_BALANCE_MAX': ['max'],
'MONTHS_BALANCE_SIZE': ['mean', 'sum']
}
cat_aggregations = {}
for cat in bureau_cat: cat_aggregations[cat] = ['mean']
for cat in bb_cat: cat_aggregations[cat + "_MEAN"] = ['mean']
bureau_agg = bureau.groupby('SK_ID_CURR' ).agg({**num_aggregations, **cat_aggregations})
 bureau_agg.columns = pd.Index(['BURO_' + e[0] + "_" + e[1].upper() for e in bureau_agg.columns.tolist() ])
 active = bureau[bureau['CREDIT_ACTIVE_Active'] == 1]
active_agg = active.groupby('SK_ID_CURR' ).agg(num_aggregations)
 active_agg.columns = pd.Index(['ACT_' + e[0] + "_" + e[1].upper() for e in active_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(active_agg, how='left', on='SK_ID_CURR')
 del active, active_agg
gc.collect()
closed = bureau[bureau['CREDIT_ACTIVE_Closed'] == 1]
closed_agg = closed.groupby('SK_ID_CURR' ).agg(num_aggregations)
 closed_agg.columns = pd.Index(['CLS_' + e[0] + "_" + e[1].upper() for e in closed_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(closed_agg, how='left', on='SK_ID_CURR')
 del closed, closed_agg, bureau
gc.collect()
return bureau_agg 
<load_from_csv>
param_grid = {
"n_estimators" : [9, 18, 27, 36, 100, 150],
"max_depth" : [2,3,5,7,9],
"min_samples_leaf" : [2, 4, 6, 8]}
def previous_applications(num_rows = None, nan_as_category = True):
prev = pd.read_csv('.. /input/previous_application.csv', nrows = num_rows)
 prev, cat_cols = one_hot_encoder(prev, nan_as_category= True)
 prev['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace= True)
 prev['DAYS_FIRST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_TERMINATION'].replace(365243, np.nan, inplace= True)
 prev['APP_CREDIT_PERC'] = prev['AMT_APPLICATION'] / prev['AMT_CREDIT']
num_aggregations = {
'AMT_ANNUITY': ['min', 'max', 'mean'],
'AMT_APPLICATION': ['min', 'max', 'mean'],
'AMT_CREDIT': ['min', 'max', 'mean'],
'APP_CREDIT_PERC': ['min', 'max', 'mean', 'var'],
'AMT_DOWN_PAYMENT': ['min', 'max', 'mean'],
'AMT_GOODS_PRICE': ['min', 'max', 'mean'],
'HOUR_APPR_PROCESS_START': ['min', 'max', 'mean'],
'RATE_DOWN_PAYMENT': ['min', 'max', 'mean'],
'DAYS_DECISION': ['min', 'max', 'mean'],
'CNT_PAYMENT': ['mean', 'sum'],
}
cat_aggregations = {}
for cat in cat_cols:
cat_aggregations[cat] = ['mean']
prev_agg = prev.groupby('SK_ID_CURR' ).agg({**num_aggregations, **cat_aggregations})
 prev_agg.columns = pd.Index(['PREV_' + e[0] + "_" + e[1].upper() for e in prev_agg.columns.tolist() ])
 approved = prev[prev['NAME_CONTRACT_STATUS_Approved'] == 1]
approved_agg = approved.groupby('SK_ID_CURR' ).agg(num_aggregations)
 approved_agg.columns = pd.Index(['APR_' + e[0] + "_" + e[1].upper() for e in approved_agg.columns.tolist() ])
 prev_agg = prev_agg.join(approved_agg, how='left', on='SK_ID_CURR')
 refused = prev[prev['NAME_CONTRACT_STATUS_Refused'] == 1]
refused_agg = refused.groupby('SK_ID_CURR' ).agg(num_aggregations)
 refused_agg.columns = pd.Index(['REF_' + e[0] + "_" + e[1].upper() for e in refused_agg.columns.tolist() ])
 prev_agg = prev_agg.join(refused_agg, how='left', on='SK_ID_CURR')
 del refused, refused_agg, approved, approved_agg, prev
gc.collect()
return prev_agg<load_from_csv>
RF_random = RandomizedSearchCV(RF, param_distributions=param_grid, cv=5 )
def pos_cash(num_rows = None, nan_as_category = True):
pos = pd.read_csv('.. /input/POS_CASH_balance.csv', nrows = num_rows)
 pos, cat_cols = one_hot_encoder(pos, nan_as_category= True)
 aggregations = {
'MONTHS_BALANCE': ['max', 'mean', 'size'],
'SK_DPD': ['max', 'mean'],
'SK_DPD_DEF': ['max', 'mean']
}
for cat in cat_cols:
aggregations[cat] = ['mean']
pos_agg = pos.groupby('SK_ID_CURR' ).agg(aggregations)
 pos_agg.columns = pd.Index(['POS_' + e[0] + "_" + e[1].upper() for e in pos_agg.columns.tolist() ])
 pos_agg['POS_COUNT'] = pos.groupby('SK_ID_CURR' ).size()
del pos
gc.collect()
return pos_agg<load_from_csv>
RF_random.fit(X_train, y_train )
def bureau_and_balance(num_rows = None, nan_as_category = True):
bureau = pd.read_csv('.. /input/bureau.csv', nrows = num_rows)
 bb = pd.read_csv('.. /input/bureau_balance.csv', nrows = num_rows)
 bb, bb_cat = one_hot_encoder(bb, nan_as_category)
 bureau, bureau_cat = one_hot_encoder(bureau, nan_as_category)
 bb_aggregations = {'MONTHS_BALANCE': ['min', 'max', 'size']}
for col in bb_cat:
bb_aggregations[col] = ['mean']
bb_agg = bb.groupby('SK_ID_BUREAU' ).agg(bb_aggregations)
 bb_agg.columns = pd.Index([e[0] + "_" + e[1].upper() for e in bb_agg.columns.tolist() ])
 bureau = bureau.join(bb_agg, how='left', on='SK_ID_BUREAU')
 bureau.drop(columns= 'SK_ID_BUREAU', inplace= True)
 del bb, bb_agg
gc.collect()
num_aggregations = {
'DAYS_CREDIT': ['min', 'max', 'mean', 'var'],
'CREDIT_DAY_OVERDUE': ['max', 'mean'],
'DAYS_CREDIT_ENDDATE': ['min', 'max', 'mean'],
'AMT_CREDIT_MAX_OVERDUE': ['mean'],
'CNT_CREDIT_PROLONG': ['sum'],
'AMT_CREDIT_SUM': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_DEBT': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_OVERDUE': ['mean'],
'AMT_CREDIT_SUM_LIMIT': ['mean', 'sum'],
'DAYS_CREDIT_UPDATE': ['min', 'max', 'mean'],
'AMT_ANNUITY': ['max', 'mean'],
'MONTHS_BALANCE_MIN': ['min'],
'MONTHS_BALANCE_MAX': ['max'],
'MONTHS_BALANCE_SIZE': ['mean', 'sum']
}
cat_aggregations = {}
for cat in bureau_cat: cat_aggregations[cat] = ['mean']
for cat in bb_cat: cat_aggregations[cat + "_MEAN"] = ['mean']
bureau_agg = bureau.groupby('SK_ID_CURR' ).agg({**num_aggregations, **cat_aggregations})
 bureau_agg.columns = pd.Index(['BURO_' + e[0] + "_" + e[1].upper() for e in bureau_agg.columns.tolist() ])
 active = bureau[bureau['CREDIT_ACTIVE_Active'] == 1]
active_agg = active.groupby('SK_ID_CURR' ).agg(num_aggregations)
 active_agg.columns = pd.Index(['ACT_' + e[0] + "_" + e[1].upper() for e in active_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(active_agg, how='left', on='SK_ID_CURR')
 del active, active_agg
gc.collect()
closed = bureau[bureau['CREDIT_ACTIVE_Closed'] == 1]
closed_agg = closed.groupby('SK_ID_CURR' ).agg(num_aggregations)
 closed_agg.columns = pd.Index(['CLS_' + e[0] + "_" + e[1].upper() for e in closed_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(closed_agg, how='left', on='SK_ID_CURR')
 del closed, closed_agg, bureau
gc.collect()
return bureau_agg<load_from_csv>
best_est_RF = RF_random.best_estimator_
def installments_payments(num_rows = None, nan_as_category = True):
ins = pd.read_csv('.. /input/installments_payments.csv', nrows = num_rows)
 ins, cat_cols = one_hot_encoder(ins, nan_as_category= True)
 ins['PAYMENT_PERC'] = ins['AMT_PAYMENT'] / ins['AMT_INSTALMENT']
ins['PAYMENT_DIFF'] = ins['AMT_INSTALMENT'] - ins['AMT_PAYMENT']
ins['DPD'] = ins['DAYS_ENTRY_PAYMENT'] - ins['DAYS_INSTALMENT']
ins['DBD'] = ins['DAYS_INSTALMENT'] - ins['DAYS_ENTRY_PAYMENT']
ins['DPD'] = ins['DPD'].apply(lambda x: x if x > 0 else 0)
 ins['DBD'] = ins['DBD'].apply(lambda x: x if x > 0 else 0)
 aggregations = {
'NUM_INSTALMENT_VERSION': ['nunique'],
'DPD': ['max', 'mean', 'sum'],
'DBD': ['max', 'mean', 'sum'],
'PAYMENT_PERC': ['max', 'mean', 'sum', 'var'],
'PAYMENT_DIFF': ['max', 'mean', 'sum', 'var'],
'AMT_INSTALMENT': ['max', 'mean', 'sum'],
'AMT_PAYMENT': ['min', 'max', 'mean', 'sum'],
'DAYS_ENTRY_PAYMENT': ['max', 'mean', 'sum']
}
for cat in cat_cols:
aggregations[cat] = ['mean']
ins_agg = ins.groupby('SK_ID_CURR' ).agg(aggregations)
 ins_agg.columns = pd.Index(['INS_' + e[0] + "_" + e[1].upper() for e in ins_agg.columns.tolist() ])
 ins_agg['INS_COUNT'] = ins.groupby('SK_ID_CURR' ).size()
del ins
gc.collect()
return ins_agg<load_from_csv>
print('Accuracy of classifier on training set: {:.2f}'.format(RF_random.score(X_train, y_train)* 100))
print('Accuracy of classifier on test set: {:.2f}'.format(RF_random.score(X_test, y_test)* 100))
def credit_card_balance(num_rows = None, nan_as_category = True):
cc = pd.read_csv('.. /input/credit_card_balance.csv', nrows = num_rows)
 cc, cat_cols = one_hot_encoder(cc, nan_as_category= True)
 cc.drop(columns = ['SK_ID_PREV'], inplace = True)
 cc_agg = cc.groupby('SK_ID_CURR' ).agg(['min', 'max', 'mean', 'sum', 'var'])
 cc_agg.columns = pd.Index(['CC_' + e[0] + "_" + e[1].upper() for e in cc_agg.columns.tolist() ])
 cc_agg['CC_COUNT'] = cc.groupby('SK_ID_CURR' ).size()
del cc
gc.collect()
return cc_agg<train_model>
y_pred = best_est_RF.predict_proba(X_train)
 y_pred = y_pred[:,1]
def kfold_lightgbm(df, num_folds, stratified = False):
train_df = df[df['TARGET'].notnull() ]
test_df = df[df['TARGET'].isnull() ]
print("Starting LightGBM.Train shape: {}, test shape: {}".format(train_df.shape, test_df.shape))
del df
gc.collect()
if stratified:
folds = StratifiedKFold(n_splits= num_folds, shuffle=True, random_state=1001)
 else:
folds = KFold(n_splits= num_folds, shuffle=True, random_state=1001)
 oof_preds = np.zeros(train_df.shape[0])
 sub_preds = np.zeros(test_df.shape[0])
 feature_importance_df = pd.DataFrame()
feats = [f for f in train_df.columns if f not in ['TARGET','SK_ID_CURR','SK_ID_BUREAU','SK_ID_PREV']]
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(train_df[feats], train_df['TARGET'])) :
train_x, train_y = train_df[feats].iloc[train_idx], train_df['TARGET'].iloc[train_idx]
valid_x, valid_y = train_df[feats].iloc[valid_idx], train_df['TARGET'].iloc[valid_idx]
clf = LGBMClassifier(
 nthread=4,
n_estimators=10000,
learning_rate=0.02,
num_leaves=34,
colsample_bytree=0.9497036,
subsample=0.8715623,
max_depth=8,
reg_alpha=0.041545473,
reg_lambda=0.0735294,
min_split_gain=0.0222415,
min_child_weight=39.3259775,
silent=-1,
verbose=-1,)
 clf.fit(train_x, train_y, eval_set=[(train_x, train_y),(valid_x, valid_y)],
eval_metric= 'auc', verbose= 100, early_stopping_rounds= 100)
 oof_preds[valid_idx] = clf.predict_proba(valid_x, num_iteration=clf.best_iteration_)[:, 1]
sub_preds += clf.predict_proba(test_df[feats], num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feats
fold_importance_df["importance"] = clf.feature_importances_
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 print('Fold %2d AUC : %.6f' %(n_fold + 1, roc_auc_score(valid_y, oof_preds[valid_idx])))
 del clf, train_x, train_y, valid_x, valid_y
gc.collect()
print('Full AUC score %.6f' % roc_auc_score(train_df['TARGET'], oof_preds))
test_df['TARGET'] = sub_preds
test_df[['SK_ID_CURR', 'TARGET']].to_csv(submission_file_name, index= False)
 display_importances(feature_importance_df)
 return feature_importance_df<merge>
from sklearn.metrics import roc_curve, auc
def main(debug = False):
num_rows = 10000 if debug else None
df = application_train_test(num_rows)
 with timer("Process bureau and bureau_balance"):
bureau = bureau_and_balance(num_rows)
 print("Bureau df shape:", bureau.shape)
 df = df.join(bureau, how='left', on='SK_ID_CURR')
 del bureau
gc.collect()
with timer("Process previous_applications"):
prev = previous_applications(num_rows)
 print("Previous applications df shape:", prev.shape)
 df = df.join(prev, how='left', on='SK_ID_CURR')
 del prev
gc.collect()
with timer("Process POS-CASH balance"):
pos = pos_cash(num_rows)
 print("Pos-cash balance df shape:", pos.shape)
 df = df.join(pos, how='left', on='SK_ID_CURR')
 del pos
gc.collect()
with timer("Process installments payments"):
ins = installments_payments(num_rows)
 print("Installments payments df shape:", ins.shape)
 df = df.join(ins, how='left', on='SK_ID_CURR')
 del ins
gc.collect()
with timer("Process credit card balance"):
cc = credit_card_balance(num_rows)
 print("Credit card balance df shape:", cc.shape)
 df = df.join(cc, how='left', on='SK_ID_CURR')
 del cc
gc.collect()
with timer("Run LightGBM with kfold"):
feat_importance = kfold_lightgbm(df, num_folds= 5, stratified = False)
 if __name__ == "__main__":
submission_file_name = "submission_kernel26.csv"
with timer("Full model run"):
main(debug= False )<load_from_csv>
df_test.drop('SeriousDlqin2yrs', axis=1, inplace=True)
 y_pred=best_est_RF.predict_proba(df_test)
 y_pred= y_pred[:,1]
PATH = ".. /input"
list_of_files = os.listdir(PATH)
 application_train = pd.read_csv(PATH+"/application_train.csv")
 application_test = pd.read_csv(PATH+"/application_test.csv")
 bureau = pd.read_csv(PATH+"/bureau.csv")
 bureau_balance = pd.read_csv(PATH+"/bureau_balance.csv")
 credit_card_balance = pd.read_csv(PATH+"/credit_card_balance.csv")
 installments_payments = pd.read_csv(PATH+"/installments_payments.csv")
 previous_application = pd.read_csv(PATH+"/previous_application.csv")
 POS_CASH_balance = pd.read_csv(PATH+"/POS_CASH_balance.csv" )<concatenate>
df_s["Probability"]=y_pred
df_s.head()
total_IDS = np.concatenate(( application_test["SK_ID_CURR"].values, application_train["SK_ID_CURR"].values))
print(len(np.unique(np.array(total_IDS)))== len(total_IDS))<define_variables>
df_s.to_csv("submission_RF.csv",index=False )
POS_CASH_balance_IDS = POS_CASH_balance["SK_ID_CURR"].values
bureau_IDS = bureau["SK_ID_CURR"].values
credit_card_balance_IDS = credit_card_balance["SK_ID_CURR"].values
installments_payments_IDS = installments_payments["SK_ID_CURR"].values
previous_application_IDS = previous_application["SK_ID_CURR"].values
tot = len(total_IDS)
 print(tot)
 print(len(np.intersect1d(POS_CASH_balance_IDS, total_IDS)) /tot*100,
len(np.intersect1d(bureau_IDS, total_IDS)) /tot*100,
len(np.intersect1d(credit_card_balance_IDS, total_IDS)) /tot*100,
len(np.intersect1d(installments_payments_IDS, total_IDS)) /tot*100,
len(np.intersect1d(previous_application_IDS, total_IDS)) /tot*100 )<feature_engineering>
XGB = XGBClassifier(n_jobs=-1)
 param_grid = {
'n_estimators' :[100,150,200,250,300],
"learning_rate" : [0.001,0.01,0.0001,0.05, 0.10 ],
"gamma" : [ 0.0, 0.1, 0.2 , 0.3 ],
"colsample_bytree" : [0.5,0.7],
'max_depth': [3,4,6,8]
}
prev = previous_application["SK_ID_PREV"].values
POS_CASH_balance_IDS_prev = POS_CASH_balance["SK_ID_PREV"].values
credit_card_balance_IDS_prev = credit_card_balance["SK_ID_PREV"].values
installments_payments_IDS_prev = installments_payments["SK_ID_PREV"].values
prev_num = len(prev)
 print(prev_num)
 print(len(np.intersect1d(POS_CASH_balance_IDS_prev, prev)) /prev_num*100,
len(np.intersect1d(credit_card_balance_IDS_prev, prev)) /prev_num*100,
len(np.intersect1d(installments_payments_IDS_prev, prev)) /prev_num*100)
<count_unique_values>
XGB_random = RandomizedSearchCV(XGB, param_distributions=param_grid, cv=5 )
bureau_br = np.unique(bureau["SK_ID_BUREAU"].values)
 print(len(np.intersect1d(np.unique(bureau_balance["SK_ID_BUREAU"].values), bureau_br)) /len(bureau_br)*100 )<count_unique_values>
XGB_random.fit(X_train,y_train )
breau_total = np.unique(np.intersect1d(bureau_IDS, total_IDS))
bureau_filtered = bureau.loc[bureau["SK_ID_CURR"].isin(breau_total)]
b = np.intersect1d(np.unique(bureau_filtered["SK_ID_BUREAU"].values), np.unique(bureau_balance["SK_ID_BUREAU"].values))
bureau_filtered = bureau_filtered.loc[bureau_filtered["SK_ID_BUREAU"].isin(b)]
len(bureau_filtered["SK_ID_CURR"].values)
 bureau_filtered<count_unique_values>
best_est_XGB = XGB_random.best_estimator_
print(len(np.unique(bureau_filtered["SK_ID_CURR"].values)) /tot*100 )<prepare_x_and_y>
print('Accuracy of classifier on training set: {:.2f}'.format(XGB_random.score(X_train, y_train)* 100))
print('Accuracy of classifier on test set: {:.2f}'.format(XGB_random.score(X_test, y_test)* 100))
train = application_train.drop(["TARGET"], axis = 1)
 train_target = application_train["TARGET"]
test= application_test.copy()
tr = len(application_train)
 print(all(i ==True for i in train.columns==test.columns)) 
<concatenate>
from imblearn.over_sampling import SMOTE
df = pd.concat([train, test])
 del train, test, application_train, application_test
gc.collect()
def categorical_features(data):
features = [i for i in list(data.columns)if data[i].dtype == 'object']
return features
categorical = categorical_features(df)
 numerical = [i for i in df.columns if i not in categorical]
numerical.remove("SK_ID_CURR")
 IDs = df["SK_ID_CURR"] 
<categorify>
smote = SMOTE(random_state=0)
 s_X,s_y=smote.fit_sample(X_train,y_train )
for feature in categorical:
df[feature].fillna("unidentified")
 print(f'Transforming {feature}...')
 encoder = LabelEncoder()
encoder.fit(df[feature].astype(str))
df[feature] = encoder.transform(df[feature].astype(str))
df.head()<feature_engineering>
RF_random.fit(s_X,s_y )
for feats in df.columns:
df[feats] = df[feats].fillna(-1)
 df.head()<count_unique_values>
best_est_RF1 = RF_random.best_estimator_
POS_CASH_balance_G1 = POS_CASH_balance.loc[POS_CASH_balance["SK_ID_CURR"].isin(total_IDS)]
print(len(np.unique(POS_CASH_balance_G1["SK_ID_CURR"].values)))
 POS_CASH_balance_G1.head()<merge>
print('Accuracy of classifier on training set: {:.2f}'.format(RF_random.score(s_X,s_y)* 100))
print('Accuracy of classifier on test set: {:.2f}'.format(RF_random.score(s_X,s_y)* 100))
np.unique(POS_CASH_balance_G1["NAME_CONTRACT_STATUS"].values)
 POS_CASH_balance_G1_num =(POS_CASH_balance_G1.groupby("SK_ID_CURR", as_index=False ).mean())
 nb = POS_CASH_balance_G1[["SK_ID_CURR", "NAME_CONTRACT_STATUS"]].groupby("SK_ID_CURR", as_index = False ).count()
nb["num_in_POS_CASH"] = nb["NAME_CONTRACT_STATUS"]
df = df.merge(POS_CASH_balance_G1_num.drop("SK_ID_PREV", axis = 1), on='SK_ID_CURR', how='left' ).fillna(-1)
 df = df.merge(nb.drop("NAME_CONTRACT_STATUS", axis = 1), on='SK_ID_CURR', how='left' ).fillna(-1)
 del nb, POS_CASH_balance_G1_num, POS_CASH_balance_G1
gc.collect()
df.head() 
<count_unique_values>
y_pred_RF1 = best_est_RF1.predict_proba(X_train)
 y_pred_RF1 = y_pred_RF1[:,1]
bureau_G1 = bureau.drop(["SK_ID_BUREAU"], axis = 1 ).loc[bureau["SK_ID_CURR"].isin(total_IDS)]
print(len(np.unique(bureau_G1["SK_ID_CURR"].values)))
 bureau_G1.head()<merge>
prediction_RF = best_est_RF1.predict_proba(df_test)
 prediction_RF = prediction_RF[:,1]
bureau_G1_num =(bureau_G1.groupby("SK_ID_CURR", as_index=False ).mean())
 nb = bureau_G1[["SK_ID_CURR", "CREDIT_ACTIVE"]].groupby("SK_ID_CURR", as_index = False ).count()
nb["num_in_bureau"] = nb["CREDIT_ACTIVE"]
df = df.merge(bureau_G1_num, on='SK_ID_CURR', how='left' ).fillna(-1)
 df = df.merge(nb.drop("CREDIT_ACTIVE", axis=1), on='SK_ID_CURR', how='left' ).fillna(-1)
 del nb, bureau_G1_num, bureau_G1
gc.collect()
df.head()<count_unique_values>
df_s["Probability"]=prediction_RF
df_s.head()
<merge><EOS> 
df_s.to_csv("submission_RF_S.csv",index=False )