Kaggle competition: Give Me Some Credit
<SOS> metric: AUC Kaggle data source: give-me-some-credit<count_unique_values>
warnings.filterwarnings('ignore' )
installments_payments_G1 = installments_payments.drop(["SK_ID_PREV"], axis = 1 ).loc[installments_payments["SK_ID_CURR"].isin(total_IDS)]
print(len(np.unique(installments_payments_G1["SK_ID_CURR"].values)))
 installments_payments_G1.head()<merge>
sampleEntry = pd.read_csv('.. /input/credit/sampleEntry.csv')
 train = pd.read_csv('.. /input/credit/cs-training.csv')
 test = pd.read_csv('.. /input/credit/cs-test.csv' )
installments_payments_G1_num =(installments_payments_G1.groupby("SK_ID_CURR", as_index=False ).mean())
 nb = installments_payments_G1[["SK_ID_CURR", "NUM_INSTALMENT_VERSION"]].groupby("SK_ID_CURR", as_index = False ).count()
nb["num_in_install_pay"] = nb["NUM_INSTALMENT_VERSION"]
df = df.merge(installments_payments_G1_num, on='SK_ID_CURR', how='left' ).fillna(-1)
 df = df.merge(nb.drop("NUM_INSTALMENT_VERSION", axis=1), on='SK_ID_CURR', how='left' ).fillna(-1)
 del nb, installments_payments_G1_num, installments_payments_G1
gc.collect()
df.head()<count_unique_values>
print(train.isnull().sum() )
previous_application_G1 = previous_application.drop(["SK_ID_PREV"], axis = 1 ).loc[previous_application["SK_ID_CURR"].isin(total_IDS)]
print(len(np.unique(previous_application_G1["SK_ID_CURR"].values)))
 previous_application_G1.head()<merge>
print(test.isnull().sum() )
previous_application_G1_num =(previous_application_G1.groupby("SK_ID_CURR", as_index=False ).mean())
 nb = previous_application_G1[["SK_ID_CURR", "NAME_CONTRACT_TYPE"]].groupby("SK_ID_CURR", as_index = False ).count()
nb["num_in_previous_app"] = nb["NAME_CONTRACT_TYPE"]
df = df.merge(previous_application_G1_num, on='SK_ID_CURR', how='left' ).fillna(-1)
 df = df.merge(nb.drop("NAME_CONTRACT_TYPE", axis=1), on='SK_ID_CURR', how='left' ).fillna(-1)
 del nb, previous_application_G1_num, previous_application_G1
gc.collect()
df.head()<prepare_x_and_y>
train.nunique()
train_X = df[:tr].drop("SK_ID_CURR", axis = 1)
 test_X = df[tr:].drop("SK_ID_CURR", axis = 1)
 train_X["TARGET"] = train_target
y = train_target
folds = KFold(n_splits=5, shuffle=True, random_state=42)
 oof_preds = np.zeros(train_X.shape[0])
 sub_preds = np.zeros(test_X.shape[0])
 feats = [f for f in train_X.columns if f not in ['SK_ID_CURR','TARGET']]<split>
train['MonthlyIncome'].fillna(train['MonthlyIncome'].mean() ,inplace=True )
for n_fold,(trn_idx, val_idx)in enumerate(folds.split(train_X)) :
trn_x, trn_y = train_X[feats].iloc[trn_idx], train_X.iloc[trn_idx]['TARGET']
val_x, val_y = train_X[feats].iloc[val_idx], train_X.iloc[val_idx]['TARGET']
clf = LGBMClassifier(
 n_estimators=10000,
learning_rate=0.01,
num_leaves=30,
colsample_bytree=.8,
subsample=.9,
max_depth=7,
reg_alpha=.1,
reg_lambda=.1,
min_split_gain=.01,
min_child_weight=100,
silent=-1,
verbose=-1, 
)
 clf.fit(trn_x, trn_y,
eval_set= [(trn_x, trn_y),(val_x, val_y)],
eval_metric='auc', verbose=100, early_stopping_rounds=100 
)
 oof_preds[val_idx] = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)[:, 1]
sub_preds += clf.predict_proba(test_X[feats], num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits
print('Fold %2d AUC : %.6f' %(n_fold + 1, roc_auc_score(val_y, oof_preds[val_idx])))
 del trn_x, trn_y, val_x, val_y
gc.collect()<save_to_csv>
train['NumberOfDependents'].fillna(train['NumberOfDependents'].mode() [0], inplace=True )
submission = pd.read_csv(".. /input/sample_submission.csv")
 submission['TARGET'] = sub_preds
submission.to_csv("baseline2.csv", index=False)
 submission.head()<load_from_csv>
test['MonthlyIncome'].fillna(test['MonthlyIncome'].mean() ,inplace=True )
df_test = pd.read_csv(".. /input/application_test.csv")
 df_train = pd.read_csv(".. /input/application_train.csv" )<train_model>
test['NumberOfDependents'].fillna(test['NumberOfDependents'].mode() [0], inplace=True )
def kfold_lightgbm(df, num_folds=5, stratified = False, debug= False):
train_df = df[df['TARGET'].notnull() ]
test_df = df[df['TARGET'].isnull() ]
print("Starting LightGBM.Train shape: {}, test shape: {}".format(train_df.shape, test_df.shape))
del df
gc.collect()
if stratified:
folds = StratifiedKFold(n_splits= num_folds, shuffle=True, random_state=1001)
 else:
folds = KFold(n_splits= num_folds, shuffle=True, random_state=1001)
 oof_preds = np.zeros(train_df.shape[0])
 sub_preds = np.zeros(test_df.shape[0])
 feature_importance_df = pd.DataFrame()
feats = [f for f in train_df.columns if f not in ['TARGET','SK_ID_CURR','SK_ID_BUREAU','SK_ID_PREV','index']]
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(train_df[feats], train_df['TARGET'])) :
train_x, train_y = train_df[feats].iloc[train_idx], train_df['TARGET'].iloc[train_idx]
valid_x, valid_y = train_df[feats].iloc[valid_idx], train_df['TARGET'].iloc[valid_idx]
clf = LGBMClassifier(
 nthread=4,
n_estimators=1000,
learning_rate=0.02,
num_leaves=34,
colsample_bytree=0.9497036,
subsample=0.8715623,
max_depth=8,
reg_alpha=0.041545473,
reg_lambda=0.0735294,
min_split_gain=0.0222415,
min_child_weight=39.3259775,
silent=-1,
verbose=-1,)
 clf.fit(train_x, train_y, eval_set=[(train_x, train_y),(valid_x, valid_y)],
eval_metric= 'auc', verbose= 100, early_stopping_rounds= 200)
 oof_preds[valid_idx] = clf.predict_proba(valid_x, num_iteration=clf.best_iteration_)[:, 1]
sub_preds += clf.predict_proba(test_df[feats], num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feats
fold_importance_df["importance"] = clf.feature_importances_
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 print('Fold %2d AUC : %.6f' %(n_fold + 1, roc_auc_score(valid_y, oof_preds[valid_idx])))
 del clf, train_x, train_y, valid_x, valid_y
gc.collect()
print('Full AUC score %.6f' % roc_auc_score(train_df['TARGET'], oof_preds))
test_df['TARGET'] = sub_preds
display_importances(feature_importance_df)
 return test_df, feature_importance_df
def display_importances(feature_importance_df_):
cols = feature_importance_df_[["feature", "importance"]].groupby("feature" ).mean().sort_values(by="importance", ascending=False)[:40].index
best_features = feature_importance_df_.loc[feature_importance_df_.feature.isin(cols)]
plt.figure(figsize=(8, 10))
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False))
plt.title('LightGBM Features(avg over folds)')
 plt.tight_layout()
plt.savefig('lgbm_importances01.png' )<categorify>
print(train.isnull().sum() )
def installments_payments(df):
aggs = ['sum', 'mean', 'max', 'min']
feat_cols = ['NUM_INSTALMENT_NUMBER', 'AMT_INSTALMENT', 'AMT_PAYMENT']
agg_dict = {k:aggs for k in feat_cols}
df_agg = df.groupby('SK_ID_CURR' ).agg(agg_dict)
 df_agg.columns = [col + '_' + agg.upper() + '_INSTALL' for col in feat_cols for agg in agg_dict[col]]
return df_agg
def pos_cash_balance(df):
df['COMPLETED_CONTRACTS'] =(df['NAME_CONTRACT_STATUS'] == 'Active' ).astype(int)
 aggs = ['sum', 'mean']
feat_cols = ['CNT_INSTALMENT', 'CNT_INSTALMENT_FUTURE', 'SK_DPD', 'COMPLETED_CONTRACTS']
agg_dict = {k:aggs for k in feat_cols}
agg_dict['COMPLETED_CONTRACTS'] = ['mean', 'size']
df_agg = df.groupby('SK_ID_CURR' ).agg(agg_dict)
 df_agg.columns = [col + '_' + agg.upper() + '_POS' for col in feat_cols for agg in agg_dict[col]]
return df_agg
def credit_card_balance(df):
df['PCT_CREDIT_LIMIT'] = df['AMT_BALANCE']/df['AMT_CREDIT_LIMIT_ACTUAL']
aggs = ['sum', 'mean', 'max']
feat_cols = ['PCT_CREDIT_LIMIT', 'AMT_CREDIT_LIMIT_ACTUAL', 'AMT_DRAWINGS_ATM_CURRENT',
'AMT_DRAWINGS_CURRENT', 'AMT_DRAWINGS_OTHER_CURRENT', 'AMT_DRAWINGS_POS_CURRENT',
'AMT_INST_MIN_REGULARITY', 'AMT_PAYMENT_TOTAL_CURRENT', 'AMT_RECEIVABLE_PRINCIPAL',
'AMT_RECIVABLE', 'AMT_TOTAL_RECEIVABLE', 'CNT_DRAWINGS_ATM_CURRENT',
'CNT_INSTALMENT_MATURE_CUM', 'SK_DPD']
agg_dict = {k:aggs for k in feat_cols}
agg_dict['PCT_CREDIT_LIMIT'] = ['mean', 'max']
df_agg = df.groupby('SK_ID_CURR' ).agg(agg_dict)
 df_agg.columns = [col + '_' + agg.upper() + '_CREDIT' for col in feat_cols for agg in agg_dict[col]]
return df_agg
def bureau(df):
df['CREDIT_ACTIVE_NUM'] =(df.CREDIT_ACTIVE == 'ACTIVE' ).astype(int)
 aggs = ['sum', 'mean', 'max']
feat_cols = ['CREDIT_DAY_OVERDUE', 'CNT_CREDIT_PROLONG', 'AMT_CREDIT_MAX_OVERDUE',
'AMT_CREDIT_SUM',
'AMT_CREDIT_SUM_LIMIT', 'AMT_CREDIT_SUM_OVERDUE', 'AMT_ANNUITY',
'CREDIT_ACTIVE_NUM', 'AMT_CREDIT_SUM_DEBT']
agg_dict = {k:aggs for k in feat_cols}
agg_dict['CREDIT_ACTIVE_NUM'] = ['sum', 'mean', 'size']
df_agg = df.groupby('SK_ID_CURR' ).agg(agg_dict)
 df_agg.columns = [col + '_' + agg.upper() + '_BUREAU' for col in feat_cols for agg in agg_dict[col]]
return df_agg
def application_train(df):
feat_num_cols = ['CNT_CHILDREN', 'AMT_INCOME_TOTAL', 'AMT_CREDIT', 'AMT_ANNUITY',
'AMT_GOODS_PRICE', 'REGION_POPULATION_RELATIVE', 'DAYS_BIRTH',
'DAYS_EMPLOYED', 'DAYS_REGISTRATION', 'OWN_CAR_AGE', 'CNT_FAM_MEMBERS',
'EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']
feat_binary_cols = ['CODE_GENDER', 'FLAG_OWN_CAR', 'FLAG_OWN_REALTY']
for col in feat_binary_cols:
df[col] = df[col].factorize() [0]
return df[['SK_ID_CURR', 'TARGET'] + feat_num_cols + feat_binary_cols].set_index('SK_ID_CURR')
 def previous_application(df):
df['CONTRACTS_REFUSED'] =(df['NAME_CONTRACT_STATUS'] == 'REFUSED' ).astype(int)
 aggs = ['sum', 'mean', 'max']
feat_cols = ['AMT_ANNUITY', 'AMT_APPLICATION', 'AMT_CREDIT', 'AMT_DOWN_PAYMENT',
'AMT_GOODS_PRICE', 'RATE_DOWN_PAYMENT', 'RATE_INTEREST_PRIMARY',
'RATE_INTEREST_PRIVILEGED', 'CNT_PAYMENT', 'CONTRACTS_REFUSED']
agg_dict = {k:aggs for k in feat_cols}
df_agg = df.groupby('SK_ID_CURR' ).agg(agg_dict)
 df_agg.columns = [col + '_' + agg.upper() + '_PREV' for col in feat_cols for agg in aggs]
return df_agg
def crossval_predict(df_train, df_test, fit_predictor,
previous_app_features=True,
bureau_features=True,
credit_features=True,
pos_features=True,
install_features=True):
hold = []
hold.append(application_train(pd.concat([df_train, df_test])))
 del(df_train, df_test)
 gc.collect()
if previous_app_features:
df_prev = pd.read_csv('.. /input/previous_application.csv')
 hold.append(previous_application(df_prev))
del(df_prev)
 gc.collect()
if bureau_features:
df_bureau = pd.read_csv('.. /input/bureau.csv')
 hold.append(bureau(df_bureau))
del(df_bureau)
 gc.collect()
if credit_features:
df_credit = pd.read_csv('.. /input/credit_card_balance.csv')
 hold.append(credit_card_balance(df_credit))
del(df_credit)
 gc.collect()
if pos_features:
df_pos = pd.read_csv('.. /input/POS_CASH_balance.csv')
 hold.append(pos_cash_balance(df_pos))
del(df_pos)
 gc.collect()
if install_features:
df_install = pd.read_csv('.. /input/installments_payments.csv')
 hold.append(installments_payments(df_install))
del(df_install)
 gc.collect()
df_test, feat_importance = fit_predictor(pd.concat(hold, axis=1))
del(hold)
 gc.collect()
return df_test[['TARGET']].reset_index() 
<save_to_csv>
print(test.isnull().sum() )
%%time
output = crossval_predict(df_train,
df_test,
kfold_lightgbm)
 output.to_csv('extratrees_simple.csv', index=False )<set_options>
X = train.drop('SeriousDlqin2yrs',1)
 y = train['SeriousDlqin2yrs']
gc.enable()<load_from_csv>
from sklearn.model_selection import train_test_split
buro_bal = pd.read_csv('.. /input/bureau_balance.csv')
 print('Buro bal shape : ', buro_bal.shape )<categorify>
from sklearn.model_selection import train_test_split
print('transform to dummies')
 buro_bal = pd.concat([buro_bal, pd.get_dummies(buro_bal.STATUS, prefix='buro_bal_status')], axis=1 ).drop('STATUS', axis=1)
<groupby>
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.20, random_state=568)
 print(X_train.shape)
 print(X_val.shape)
 print(y_train.shape)
 print(y_val.shape )
print('Counting buros')
 buro_counts = buro_bal[['SK_ID_BUREAU', 'MONTHS_BALANCE']].groupby('SK_ID_BUREAU' ).count()
buro_bal['buro_count'] = buro_bal['SK_ID_BUREAU'].map(buro_counts['MONTHS_BALANCE'] )<groupby>
from xgboost import XGBClassifier
print('averaging buro bal')
 avg_buro_bal = buro_bal.groupby('SK_ID_BUREAU' ).mean() 
<drop_column>
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
avg_buro_bal.columns = ['avg_buro_' + f_ for f_ in avg_buro_bal.columns]
del buro_bal
gc.collect()<load_from_csv>
xgb = XGBClassifier(n_jobs=-1)
 param_grid = {
'n_estimators' :[100,150,200,250,300],
"learning_rate" : [0.001,0.01,0.0001,0.05, 0.10 ],
"gamma" : [ 0.0, 0.1, 0.2 , 0.3 ],
"colsample_bytree" : [0.5,0.7],
'max_depth': [3,4,6,8]
}
print('Read Bureau')
 buro = pd.read_csv('.. /input/bureau.csv' )<categorify>
xgb_randomgrid = RandomizedSearchCV(xgb, param_distributions=param_grid, cv=5 )
print('Go to dummies')
 buro_credit_active_dum = pd.get_dummies(buro.CREDIT_ACTIVE, prefix='ca_')
 buro_credit_currency_dum = pd.get_dummies(buro.CREDIT_CURRENCY, prefix='cu_')
 buro_credit_type_dum = pd.get_dummies(buro.CREDIT_TYPE, prefix='ty_')
<concatenate>
%%time
xgb_randomgrid.fit(X_train,y_train )
buro_full = pd.concat([buro, buro_credit_active_dum, buro_credit_currency_dum, buro_credit_type_dum], axis=1 )<drop_column>
best_est = xgb_randomgrid.best_estimator_
del buro_credit_active_dum, buro_credit_currency_dum, buro_credit_type_dum
gc.collect()<merge>
y_pred = best_est.predict_proba(X_train)
 y_pred = y_pred[:,1]
print('Merge with buro avg')
 buro_full = buro_full.merge(right=avg_buro_bal.reset_index() , how='left', on='SK_ID_BUREAU', suffixes=('', '_bur_bal'))<groupby>
X_test = test.drop('SeriousDlqin2yrs',1)
 y_test=best_est.predict_proba(X_test)
 y_test= y_test[:,1]
print('Counting buro per SK_ID_CURR')
 nb_bureau_per_curr = buro_full[['SK_ID_CURR', 'SK_ID_BUREAU']].groupby('SK_ID_CURR' ).count()
buro_full['SK_ID_BUREAU'] = buro_full['SK_ID_CURR'].map(nb_bureau_per_curr['SK_ID_BUREAU'] )<groupby>
sampleEntry["Probability"]=y_test
sampleEntry.head()
print('Averaging bureau')
 avg_buro = buro_full.groupby('SK_ID_CURR' ).mean()
print(avg_buro.head() )<set_options>
sampleEntry.to_csv("submission.csv",index=False )
del buro, buro_full
gc.collect()<load_from_csv>
from imblearn.over_sampling import SMOTE
print('Read prev')
 prev = pd.read_csv('.. /input/previous_application.csv')
<define_variables>
smote = SMOTE(random_state=0)
 os_data_X,os_data_y=smote.fit_sample(X_train,y_train )
prev_cat_features = [
f_ for f_ in prev.columns if prev[f_].dtype == 'object'
] 
<categorify>
rfc = RandomForestClassifier(n_jobs=-1, max_features='sqrt')
 param_grid = {
"n_estimators" : [9, 18, 27, 36, 100, 150],
"max_depth" : [2,3,5,7,9],
"min_samples_leaf" : [2, 4]}
print('Go to dummies')
 prev_dum = pd.DataFrame()
for f_ in prev_cat_features:
prev_dum = pd.concat([prev_dum, pd.get_dummies(prev[f_], prefix=f_ ).astype(np.uint8)], axis=1)
 prev = pd.concat([prev, prev_dum], axis=1 )<set_options>
rfc_randomgrid = RandomizedSearchCV(rfc, param_distributions=param_grid, cv=5 )
del prev_dum
gc.collect()<feature_engineering>
rfc_randomgrid.fit(os_data_X,os_data_y )
print('Counting number of Prevs')
 nb_prev_per_curr = prev[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
prev['SK_ID_PREV'] = prev['SK_ID_CURR'].map(nb_prev_per_curr['SK_ID_PREV'])
 print('Averaging prev')
 avg_prev = prev.groupby('SK_ID_CURR' ).mean()
print(avg_prev.head())
 del prev
gc.collect()
print('Reading POS_CASH')
 pos = pd.read_csv('.. /input/POS_CASH_balance.csv')
 print('Go to dummies')
 pos = pd.concat([pos, pd.get_dummies(pos['NAME_CONTRACT_STATUS'])], axis=1)
 print('Compute nb of prevs per curr')
 nb_prevs = pos[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
pos['SK_ID_PREV'] = pos['SK_ID_CURR'].map(nb_prevs['SK_ID_PREV'])
 print('Go to averages')
 avg_pos = pos.groupby('SK_ID_CURR' ).mean()
del pos, nb_prevs
gc.collect()
print('Reading CC balance')
 cc_bal = pd.read_csv('.. /input/credit_card_balance.csv')
 print('Go to dummies')
 cc_bal = pd.concat([cc_bal, pd.get_dummies(cc_bal['NAME_CONTRACT_STATUS'], prefix='cc_bal_status_')], axis=1)
 nb_prevs = cc_bal[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
cc_bal['SK_ID_PREV'] = cc_bal['SK_ID_CURR'].map(nb_prevs['SK_ID_PREV'])
 print('Compute average')
 avg_cc_bal = cc_bal.groupby('SK_ID_CURR' ).mean()
avg_cc_bal.columns = ['cc_bal_' + f_ for f_ in avg_cc_bal.columns]
del cc_bal, nb_prevs
gc.collect()
print('Reading Installments')
 inst = pd.read_csv('.. /input/installments_payments.csv')
 nb_prevs = inst[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
inst['SK_ID_PREV'] = inst['SK_ID_CURR'].map(nb_prevs['SK_ID_PREV'])
 avg_inst = inst.groupby('SK_ID_CURR' ).mean()
avg_inst.columns = ['inst_' + f_ for f_ in avg_inst.columns]
print('Read data and test')
 data = pd.read_csv('.. /input/application_train.csv')
 test = pd.read_csv('.. /input/application_test.csv')
 print('Shapes : ', data.shape, test.shape)
 y = data['TARGET']
del data['TARGET']
categorical_feats = [
f for f in data.columns if data[f].dtype == 'object'
]
categorical_feats
for f_ in categorical_feats:
data[f_], indexer = pd.factorize(data[f_])
 test[f_] = indexer.get_indexer(test[f_] )<merge>
best_est1 = rfc_randomgrid.best_estimator_
data = data.merge(right=avg_buro.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_buro.reset_index() , how='left', on='SK_ID_CURR')
 data = data.merge(right=avg_prev.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_prev.reset_index() , how='left', on='SK_ID_CURR')
 data = data.merge(right=avg_pos.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_pos.reset_index() , how='left', on='SK_ID_CURR')
 data = data.merge(right=avg_cc_bal.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_cc_bal.reset_index() , how='left', on='SK_ID_CURR')
 data = data.merge(right=avg_inst.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_inst.reset_index() , how='left', on='SK_ID_CURR')
 del avg_buro, avg_prev
gc.collect()<create_dataframe>
y_pred1 = best_est1.predict_proba(X_train)
 y_pred1 = y_pred1[:,1]
gc.enable()
folds = KFold(n_splits=6, shuffle=True, random_state=546789)
 oof_preds = np.zeros(data.shape[0])
 sub_preds = np.zeros(test.shape[0])
 feature_importance_df = pd.DataFrame()
feats = [f for f in data.columns if f not in ['SK_ID_CURR']]<split>
y_test1=best_est1.predict_proba(X_test)
 y_test1= y_test1[:,1]
for n_fold,(trn_idx, val_idx)in enumerate(folds.split(data)) :
trn_x, trn_y = data[feats].iloc[trn_idx], y.iloc[trn_idx]
val_x, val_y = data[feats].iloc[val_idx], y.iloc[val_idx]
clf = LGBMClassifier(
 n_estimators=10000,
learning_rate=0.03,
num_leaves = 22,
colsample_bytree=0.8,
subsample=0.8,
max_depth=6,
reg_alpha=0.1,
reg_lambda=0.1,
min_split_gain=0.01,
min_child_weight=100,
silent=-1,
verbose=-1, 
)
 clf.fit(trn_x, trn_y,
eval_set= [(trn_x, trn_y),(val_x, val_y)],
eval_metric='auc', verbose=100, early_stopping_rounds=300 
)
 oof_preds[val_idx] = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)[:, 1]
sub_preds += clf.predict_proba(test[feats], num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feats
fold_importance_df["importance"] = clf.feature_importances_
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 print('Fold %2d AUC : %.6f' %(n_fold + 1, roc_auc_score(val_y, oof_preds[val_idx])))
 del clf, trn_x, trn_y, val_x, val_y
gc.collect()<save_to_csv>
sampleEntry["Probability"]=y_test1
sampleEntry.head()
print('Full AUC score %.6f' % roc_auc_score(y, oof_preds))
test['TARGET'] = sub_preds
test[['SK_ID_CURR', 'TARGET']].to_csv('first_submission.csv', index=False )<import_modules>
sampleEntry.to_csv("submission1.csv",index=False )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix,f1_score
import gc 
<load_from_csv>
from sklearn.neighbors import KNeighborsClassifier
application_test=pd.read_csv('.. /input/application_test.csv')
 application_train=pd.read_csv('.. /input/application_train.csv')
 bureau=pd.read_csv('.. /input/bureau.csv')
 bureau_balance=pd.read_csv('.. /input/bureau_balance.csv')
 credit_card_balance=pd.read_csv('.. /input/credit_card_balance.csv')
 installments_payments=pd.read_csv('.. /input/installments_payments.csv')
 POS_CASH_balance=pd.read_csv('.. /input/POS_CASH_balance.csv')
 previous_application=pd.read_csv('.. /input/previous_application.csv' )<sort_values>
from sklearn.neighbors import KNeighborsClassifier
def check_missing_data(df):
total = df.isnull().sum().sort_values(ascending = False)
 percent =(( df.isnull().sum() /df.isnull().count())*100 ).sort_values(ascending = False)
 return pd.concat([total, percent], axis=1, keys=['Total', 'Percent'] )<filter>
from sklearn.neighbors import KNeighborsClassifier
def categorical_features(df):
cat_features=df.columns[df.dtypes=='object']
return list(cat_features )<categorify>
knn = KNeighborsClassifier(n_neighbors=3 )
def onehot_encoding(df,cat_features_name):
df=pd.get_dummies(df,columns=cat_features_name)
 return df<feature_engineering>
knn.fit(X_train,y_train )
categorical_features(bureau )<count_values>
y_pred2 = knn.predict_proba(X_train)
 y_pred2 = y_pred2[:,1]
bureau.CREDIT_ACTIVE.value_counts()<count_values>
md_KNN = KNeighborsClassifier()
neighbors = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]
param_grid = dict(n_neighbors=neighbors )
bureau.CREDIT_CURRENCY.value_counts()<count_values>
KNN_GridSearch = GridSearchCV(md_KNN,param_grid=param_grid,cv=10 )
bureau.CREDIT_TYPE.value_counts()<data_type_conversions>
KNN_GridSearch.fit(X_train,y_train )
bureau.AMT_CREDIT_SUM.fillna(value=bureau.AMT_CREDIT_SUM.median() ,inplace=True )<feature_engineering>
best_est2=KNN_GridSearch.best_estimator_
bureau['DAYS_CREDIT_ENDDATE']=np.where(bureau.DAYS_CREDIT_ENDDATE.isnull() ,bureau.DAYS_ENDDATE_FACT,bureau.DAYS_CREDIT_ENDDATE )<data_type_conversions>
y_pred3 = best_est2.predict_proba(X_train)
 y_pred3 = y_pred3[:,1]
bureau.DAYS_CREDIT_ENDDATE.fillna(value=0.0,inplace=True )<drop_column>
y_test2=best_est2.predict_proba(X_test)
 y_test2= y_test2[:,1]
bureau.drop('DAYS_ENDDATE_FACT',axis=1,inplace=True )<set_options>
sampleEntry["Probability"]=y_test2
sampleEntry.head()
<set_options><EOS> 
sampleEntry.to_csv("submission2.csv",index=False )