Kaggle competition: Home Credit Default Risk
all_model = [RandomForestClassifier() ,ExtraTreeClassifier() , LogisticRegression() ,RidgeClassifier() ,
DecisionTreeClassifier() , KNeighborsClassifier() , PassiveAggressiveClassifier() ,
]<choose_model_class>
df_application = pd.read_csv('.. /input/application_train.csv')
 df_application_test = pd.read_csv('.. /input/application_test.csv')
 df_application.head()
params = {'loss_function':'Logloss',
'eval_metric':'F1',
'iterations' : 1000,
'learning_rate': 0.01,
'verbose': 1000,
'random_seed': random_state
}
cbc = CatBoostClassifier(**params )<prepare_x_and_y>
df_application['Source'] = 'Train'
df_application_test['Source'] = 'Test'
df = pd.concat(( df_application,df_application_test),axis = 0,sort = False)
 cat_cols = [col for col in df.columns if(df[col].dtype == object)&(col != 'Source')]
le = preprocessing.LabelEncoder()
for col in cat_cols:
df[col] = le.fit_transform(df[col].fillna("Missing"))
df.head()
df_train = df[df['Source'] == "Train"].drop('Source', axis =1)
 df_test = df[df['Source'] == "Test"].drop('Source', axis =1)
 del df
data_dmatrix = xgb.DMatrix(data=X,label=Y )<train_on_grid>
df_bureau = pd.read_csv(".. /input/bureau.csv")
 df_bureau_balance = pd.read_csv(".. /input/bureau_balance.csv")
 df_bureau_balance["MONTHS_BALANCE"]= np.abs(df_bureau_balance["MONTHS_BALANCE"])
 df_bureau_balance["Period"] = np.where(( df_bureau_balance["MONTHS_BALANCE"] < 7),"short",np.where(( df_bureau_balance["MONTHS_BALANCE"] < 13),"medium","long"))
df_bureau_balance["Period_status"] = df_bureau_balance["Period"].astype(str)+ "_" + df_bureau_balance["STATUS"]
df_bureau_balance.head(5)
params = {"objective":"binary:logistic",'colsample_bytree': 0.3,'learning_rate': 0.1,
'max_depth': 10, 'alpha': 10} 
<compute_train_metric>
df_bureau_balance = df_bureau_balance.groupby(["SK_ID_BUREAU","Period_status"])\ 
.agg({"MONTHS_BALANCE" : ["count","min","max","mean"]})\ 
.reset_index()
df_bureau_balance.columns = [''.join(col ).strip() for col in df_bureau_balance.columns.values]
df_bureau_balance.head()
for model in all_model:
scores = cross_val_score(model, X, Y, cv=get_kfold() , scoring='f1')
 print("Model : " , model , " has f1 score : " , scores.mean() )<train_model>
df_bureau_balance = df_bureau_balance.pivot_table(index = 'SK_ID_BUREAU',columns = 'Period_status',values = ['MONTHS_BALANCEcount','MONTHS_BALANCEmin','MONTHS_BALANCEmax','MONTHS_BALANCEmean'] ).reset_index()
df_bureau_balance.columns = [''.join(col ).strip() for col in df_bureau_balance.columns.values]
df_bureau = pd.merge(df_bureau,df_bureau_balance, how="left", on = "SK_ID_BUREAU")
 df_bureau=pd.get_dummies(df_bureau,prefix =['CREDIT_ACTIVE','CREDIT_CURRENCY','CREDIT_TYPE'] ,columns = ['CREDIT_ACTIVE','CREDIT_CURRENCY','CREDIT_TYPE'],dummy_na = True)
 df_bureau.head(5 )
<import_modules>
df_bureau_features = df_bureau.groupby(['SK_ID_CURR'])\ 
.agg({'SK_ID_BUREAU' : ['nunique'],
'DAYS_CREDIT' : ['min','max','mean','std'],
'CREDIT_DAY_OVERDUE' :['min','max','mean','std'],
'DAYS_CREDIT_ENDDATE':['min','max','mean','std'],
'DAYS_ENDDATE_FACT': ['min','max','mean','std'],
'AMT_CREDIT_MAX_OVERDUE' : ['mean','min','max'],
'CNT_CREDIT_PROLONG' : ['mean','min','max'],
'AMT_CREDIT_SUM' : ['min','max','mean','std'],
'AMT_CREDIT_SUM_DEBT' : ['sum','min','max'],
'AMT_CREDIT_SUM_LIMIT': ['sum','min','max'],
'AMT_CREDIT_SUM_OVERDUE':['sum','min','max'],
'DAYS_CREDIT_UPDATE' : ['sum','min','max'],
'AMT_ANNUITY' : ['sum','min','max','mean']
})
 df_bureau_features = df_bureau_features.reset_index()
df_bureau_features.columns = [''.join(col ).strip() for col in df_bureau_features.columns.values]
df_bureau_features_ohe = df_bureau[['SK_ID_CURR'] + ohe_cols].groupby(['SK_ID_CURR'] ).mean().reset_index()
df_bureau_features = pd.merge(df_bureau_features,df_bureau_features_ohe,how = 'left',on = 'SK_ID_CURR')
 del df_bureau_features_ohe
df_train = pd.merge(df_train,df_bureau_features,how = 'left',on = 'SK_ID_CURR')
 df_test = pd.merge(df_test,df_bureau_features,how = 'left',on = 'SK_ID_CURR' )
from sklearn.model_selection import RepeatedStratifiedKFold
from sklearn.model_selection import GridSearchCV<choose_model_class>
df_previous_application = pd.read_csv(".. /input/previous_application.csv")
 cat_cols = [col for col in df_previous_application.columns if(df_previous_application[col].dtype == object)&(( col != 'SK_ID_CURR')|(col != 'SK_ID_PREV')) ]
df_previous_application = pd.get_dummies(df_previous_application,prefix = cat_cols,columns = cat_cols)
 df_previous_application.head()
model = LogisticRegression()
solvers = ['newton-cg', 'lbfgs', 'liblinear']
penalty = ['l2']
c_values = [100, 10, 1.0, 0.1, 0.01]<train_on_grid>
df_POS_CASH_balance = pd.read_csv(".. /input/POS_CASH_balance.csv")
 df_POS_CASH_balance = pd.get_dummies(df_POS_CASH_balance, columns= ["NAME_CONTRACT_STATUS"])
 df_POS_CASH_balance_current = df_POS_CASH_balance.drop('SK_ID_PREV',axis = 1 ).groupby('SK_ID_CURR' ).mean().reset_index()
df_POS_CASH_balance_previous = df_POS_CASH_balance.drop('SK_ID_CURR',axis = 1 ).groupby('SK_ID_PREV' ).mean().reset_index()
del df_POS_CASH_balance
df_POS_CASH_balance_current.head()
grid = dict(solver=solvers,penalty=penalty,C=c_values)
 cv = RepeatedStratifiedKFold(n_splits=5, n_repeats=3, random_state=1)
 grid_search = GridSearchCV(estimator=model, param_grid=grid, n_jobs=-1, cv=cv, scoring='f1',error_score=0, verbose = 3)
 grid_result = grid_search.fit(X, Y)
<find_best_params>
df_installments_payments = pd.read_csv(".. /input/installments_payments.csv")
 df_installments_payments_current = df_installments_payments.drop('SK_ID_PREV',axis= 1 ).groupby('SK_ID_CURR' ).mean().reset_index()
df_installments_payments_previous = df_installments_payments.drop('SK_ID_CURR',axis= 1 ).groupby('SK_ID_PREV' ).mean().reset_index()
del df_installments_payments
df_installments_payments_current.head()
print("Best: %f using %s" %(grid_result.best_score_, grid_result.best_params_))
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
print("%f(%f)with: %r" %(mean, stdev, param))<choose_model_class>
df_credit_card_balance = pd.read_csv(".. /input/credit_card_balance.csv")
 df_credit_card_balance = pd.get_dummies(df_credit_card_balance, columns= ['NAME_CONTRACT_STATUS'])
 df_credit_card_balance_current = df_credit_card_balance.drop('SK_ID_PREV',axis = 1 ).groupby('SK_ID_CURR' ).mean().reset_index()
df_credit_card_balance_previous = df_credit_card_balance.drop('SK_ID_CURR',axis = 1 ).groupby('SK_ID_PREV' ).mean().reset_index()
del df_credit_card_balance
df_credit_card_balance_current.head()
model = LogisticRegression(C= 1.0, penalty= 'l2', solver = 'newton-cg' )<compute_train_metric>
df_train = df_train.merge(df_POS_CASH_balance_current, on = 'SK_ID_CURR',how = 'left',suffixes=['','_POS_bal_curr'])\ 
.merge(df_installments_payments_current, on = 'SK_ID_CURR',how = 'left', suffixes = ['','_installments_curr'])\ 
.merge(df_credit_card_balance_current, on = 'SK_ID_CURR',how = 'left',suffixes=['','_credit_card_bal_curr'])
 df_test = df_test.merge(df_POS_CASH_balance_current, on = 'SK_ID_CURR',how = 'left',suffixes=['','_POS_bal_curr'])\ 
.merge(df_installments_payments_current, on = 'SK_ID_CURR',how = 'left',suffixes = ['','_installments_curr'])\ 
.merge(df_credit_card_balance_current, on = 'SK_ID_CURR',how = 'left',suffixes=['','_credit_card_bal_curr'])\
df_train.head()
scores = cross_val_score(model, X, Y, cv=get_kfold() , scoring='f1')
 print(scores.mean() )<train_model>
df_previous_application = df_previous_application.merge(df_POS_CASH_balance_previous, on = 'SK_ID_PREV',how = 'left',suffixes=['','_POS_bal_past'])\ 
.merge(df_installments_payments_previous, on = 'SK_ID_PREV',how = 'left',suffixes = ['','_installments_past'])\ 
.merge(df_credit_card_balance_previous, on = 'SK_ID_PREV',how = 'left',suffixes=['','_credit_card_bal_past'])
 df_previous_application = df_previous_application.drop("SK_ID_PREV",axis= 1 ).groupby(['SK_ID_CURR'] ).mean().reset_index()
df_train = df_train.merge(df_previous_application, on = 'SK_ID_CURR',how = 'left',suffixes=('','_past_appl'))
df_test = df_test.merge(df_previous_application, on = 'SK_ID_CURR',how = 'left',suffixes=('','_past_appl'))
df_previous_application.head()
del df_previous_application
model = model.fit(X,Y )<drop_column>
df1 = df_train.sample(frac = 1)
 msk = np.random.rand(len(df1))
eval_set = df1[msk >= 0.95]
train = df1[msk < 0.95]
X_train = train.drop(['TARGET','SK_ID_CURR'],axis = 1)
 Y_train = train['TARGET']
X_eval = eval_set.drop(['TARGET','SK_ID_CURR'],axis = 1)
 Y_eval = eval_set['TARGET']
def xgb_(X_train = X_train, Y_train = Y_train,
params = {
"objective" :['multi:softmax']
,"max_depth" :[2]
,'eta' :[0.1]
},
fit_params = {
'eval_metric' :['mlogloss']
,'eval_set' :[(X_eval,Y_eval)]},
X_eval = X_eval, Y_eval = Y_eval):
print("
print()
model_xgb = xgb.XGBClassifier()
clf = GridSearchCV(model_xgb, param_grid = params,
fit_params = fit_params,cv =3, scoring ="roc_auc")
 clf.fit(X_train, Y_train)
 print("
Best parameters set found on development set:")
 print()
print(clf.best_params_)
 print()
print("Grid scores on development set:")
 print()
means = clf.cv_results_['mean_test_score']
stds = clf.cv_results_['std_test_score']
for mean, std, params in zip(means, stds, clf.cv_results_['params']):
print("%0.3f(+/-%0.03f)for %r"
%(mean, std * 2, params))
print()
print("Detailed classification report:")
 print()
print("The model is trained on the full development set.")
 print("The scores are computed on the full evaluation set.")
 print()
y_true, y_pred = Y_eval, clf.predict(X_eval)
 print(classification_report(y_true, y_pred))
print()
print(confusion_matrix(y_true, y_pred))
print()
model_xgb = clf.best_estimator_
xgb.plot_importance(model_xgb,max_num_features = 20)
 Y_dev_pred = model_xgb.predict_proba(X_eval)[:,1]
score = roc_auc_score(Y_eval,Y_dev_pred)
 results = model_xgb.evals_result()
metrics = fit_params['eval_metric']
epochs = len(results['validation_0'][metrics[0]])
 x_axis = range(0, epochs)
 for metric in metrics:
fig, ax = pyplot.subplots()
ax.plot(x_axis, results['validation_0'][metric], label='Train')
 ax.plot(x_axis, results['validation_1'][metric], label='Validation/Hold out set')
 ax.legend()
pyplot.ylabel('%s' %(metric))
pyplot.title('XGBoost %s' %(metric))
pyplot.show()
return model_xgb,score,clf,results,model_xgb.feature_importances_
test = test.drop(['isChurned'], axis = True )<predict_on_test>
xgb_params = {
'learning_rate' :[0.02]
,'reg_lambda' :[16]
,"max_depth" :[9]
,'silent' :[False]
,'n_estimators' :[1000]
,'colsample_bytree' :[0.5]
,'nthread' :[-1]
,'subsample' :[0.5]
,'objective' :["binary:logistic"]
,'scale_pos_weight' :[2]
}
fit_params = {
'eval_metric' :['auc']
,'eval_set' :[(X_train,Y_train),(X_eval,Y_eval)]
,'early_stopping_rounds' : 30
}
model_xgb,score_xgb,xgb_gridsearch,results,feats = xgb_(X_train = X_train,X_eval = X_eval,params = xgb_params,fit_params = fit_params)
prediction = model.predict(test )<data_type_conversions>
X_test = df_test.drop(['TARGET','SK_ID_CURR'],axis = 1)
 Y_test = model_xgb.predict_proba(X_test)[:,1]
<count_values><EOS> 
df_application_test["TARGET"] = Y_test
df_submit = df_application_test[["SK_ID_CURR","TARGET"]]
df_submit.to_csv('submission_appl_bureau.csv', index=False )