Kaggle competition: Don't Overfit! II
<SOS> metric: AUC Kaggle data source: dont-overfit!-ii<train_model>
pd.set_option('max_columns', None)
 %matplotlib inline
py.init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore')
print("End Bureau................ " )<train_model>
%%time
train = pd.read_csv(os.path.join(location, 'train.csv'))
test = pd.read_csv(os.path.join(location, 'test.csv'))
sample_submission = pd.read_csv(os.path.join(location, 'sample_submission.csv'))
print("Start previous_application................ " )<load_from_csv>
def missing_data(data):
total = data.isnull().sum()
percent =(total/data.isnull().count() *100)
 miss_column = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 types = []
for col in data.columns:
dtype = str(data[col].dtype)
 types.append(dtype)
 miss_column['Types'] = types
return(np.transpose(miss_column))
prev = pd.read_csv('.. /input/previous_application.csv', nrows = num_rows )<categorify>
missing_data(train )
prev, cat_cols = one_hot_encoder(prev, nan_as_category= True)
 prev['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace= True)
 prev['DAYS_FIRST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_TERMINATION'].replace(365243, np.nan, inplace= True )<feature_engineering>
missing_data(test )
prev['APP_CREDIT_PERC'] = prev['AMT_APPLICATION'] / prev['AMT_CREDIT']<define_variables>
label = train['target']
train = train.drop(['id', 'target'], axis=1)
 test = test.drop(['id'], axis=1 )
num_aggregations = {
'AMT_ANNUITY': ['min', 'max', 'mean'],
'AMT_APPLICATION': ['min', 'max', 'mean'],
'AMT_CREDIT': ['min', 'max', 'mean'],
'APP_CREDIT_PERC': ['min', 'max', 'mean', 'var'],
'AMT_DOWN_PAYMENT': ['min', 'max', 'mean'],
'AMT_GOODS_PRICE': ['min', 'max', 'mean'],
'HOUR_APPR_PROCESS_START': ['min', 'max', 'mean'],
'RATE_DOWN_PAYMENT': ['min', 'max', 'mean'],
'DAYS_DECISION': ['min', 'max', 'mean'],
'CNT_PAYMENT': ['mean', 'sum'],
}<groupby>
label_count = label.value_counts().reset_index().rename(columns = {'index' : 'Labels'})
 label_count
cat_aggregations = {}
for cat in cat_cols:
cat_aggregations[cat] = ['mean']
prev_agg = prev.groupby('SK_ID_CURR' ).agg({**num_aggregations, **cat_aggregations})
 prev_agg.columns = pd.Index(['PREV_' + e[0] + "_" + e[1].upper() for e in prev_agg.columns.tolist() ] )<merge>
missing_data(train )
approved = prev[prev['NAME_CONTRACT_STATUS_Approved'] == 1]
approved_agg = approved.groupby('SK_ID_CURR' ).agg(num_aggregations)
 approved_agg.columns = pd.Index(['APR_' + e[0] + "_" + e[1].upper() for e in approved_agg.columns.tolist() ])
 prev_agg = prev_agg.join(approved_agg, how='left')
 refused = prev[prev['NAME_CONTRACT_STATUS_Refused'] == 1]
refused_agg = refused.groupby('SK_ID_CURR' ).agg(num_aggregations)
 refused_agg.columns = pd.Index(['REF_' + e[0] + "_" + e[1].upper() for e in refused_agg.columns.tolist() ])
 prev_agg = prev_agg.join(refused_agg, how='left')
 del refused, refused_agg, approved, approved_agg, prev
gc.collect() 
<train_model>
folds = StratifiedKFold(n_splits=10, shuffle=True, random_state=4422)
 forcast_rfc2 = np.zeros(len(xtest))
validation_pred_rfc2 = np.zeros(len(xtrain))
scores_rfc2 = []
valid_rfc2 = np.zeros(len(label))
for fold_,(xtrn_indx, xvad_indx)in enumerate(folds.split(xtrain, label)) :
print('Fold {}, started at {}'.format(fold_, time.ctime()))
x_train, x_valid = xtrain[xtrn_indx], xtrain[xvad_indx]
y_train, y_valid = label[xtrn_indx], label[xvad_indx]
clf_rfc2 = RandomForestClassifier(n_estimators=1000, n_jobs=-1, random_state=0)
 clf_rfc2.fit(x_train, y_train)
 pred_valid = clf_rfc2.predict(x_valid ).reshape(-1,)
 score = accuracy_score(y_valid, pred_valid)
 Pred_Real = clf_rfc2.predict_proba(xtest)[:, 1]
validation_pred_rfc2[xvad_indx] +=(pred_valid ).reshape(-1,)
 scores_rfc2.append(accuracy_score(y_valid, pred_valid))
forcast_rfc2 += Pred_Real
valid_rfc2[xvad_indx] +=(y_valid)
 print(c_report(valid_rfc2[xvad_indx], validation_pred_rfc2[xvad_indx]))
print(c_matrix(valid_rfc2[xvad_indx], validation_pred_rfc2[xvad_indx]))
print('accuracy is: {}, std: {}.'.format(np.mean(scores_rfc2), np.std(scores_rfc2)) )
print("End previous_application................ " )<train_model>
forcast_dtc2 = np.zeros(len(xtest))
validation_pred_dtc2 = np.zeros(len(xtrain))
scores_dtc2 = []
valid_dtc2 = np.zeros(len(label))
for fold_,(xtrn_indx, xvad_indx)in enumerate(folds.split(xtrain, label)) :
print('Fold {}, started at {}'.format(fold_, time.ctime()))
x_train, x_valid = xtrain[xtrn_indx], xtrain[xvad_indx]
y_train, y_valid = label[xtrn_indx], label[xvad_indx]
clf_dtc2 = DecisionTreeClassifier()
clf_dtc2.fit(x_train, y_train)
 pred_valid = clf_dtc2.predict(x_valid ).reshape(-1,)
 score = accuracy_score(y_valid, pred_valid)
 Pred_Real = clf_dtc2.predict_proba(xtest)[:, 1]
validation_pred_dtc2[xvad_indx] +=(pred_valid ).reshape(-1,)
 scores_dtc2.append(accuracy_score(y_valid, pred_valid))
forcast_dtc2 += Pred_Real
valid_dtc2[xvad_indx] +=(y_valid)
 print(c_report(valid_dtc2[xvad_indx], validation_pred_dtc2[xvad_indx]))
print(c_matrix(valid_dtc2[xvad_indx], validation_pred_dtc2[xvad_indx]))
print('accuracy is: {}, std: {}.'.format(np.mean(scores_dtc2), np.std(scores_dtc2)) )
print("Start POS_CASH_balance................ " )<load_from_csv>
forcast_svm2 = np.zeros(len(xtest))
validation_pred_svm2 = np.zeros(len(xtrain))
scores_svm2 = []
valid_svm2 = np.zeros(len(label))
for fold_,(xtrn_indx, xvad_indx)in enumerate(folds.split(xtrain, label)) :
print('Fold {}, started at {}'.format(fold_, time.ctime()))
x_train, x_valid = xtrain[xtrn_indx], xtrain[xvad_indx]
y_train, y_valid = label[xtrn_indx], label[xvad_indx]
clf_svm2 = svm.SVC(kernel='linear', gamma=1)
 clf_svm2.fit(x_train, y_train)
 pred_valid = clf_svm2.predict(x_valid ).reshape(-1,)
 score = accuracy_score(y_valid, pred_valid)
 Pred_Real = clf_svm2.predict(xtest)
 validation_pred_svm2[xvad_indx] +=(pred_valid ).reshape(-1,)
 scores_svm2.append(accuracy_score(y_valid, pred_valid))
forcast_svm2 += Pred_Real
valid_svm2[xvad_indx] +=(y_valid)
 print(c_report(valid_svm2[xvad_indx], validation_pred_svm2[xvad_indx]))
print(c_matrix(valid_svm2[xvad_indx], validation_pred_svm2[xvad_indx]))
print('accuracy is: {}, std: {}.'.format(np.mean(scores_svm2), np.std(scores_svm2)) )
pos = pd.read_csv('.. /input/POS_CASH_balance.csv', nrows = num_rows )<categorify>
forcast_lr2 = np.zeros(len(xtest))
validation_pred_lr2 = np.zeros(len(xtrain))
scores_lr2 = []
valid_lr2 = np.zeros(len(label))
for fold_,(xtrn_indx, xvad_indx)in enumerate(folds.split(xtrain, label)) :
print('Fold {}, started at {}'.format(fold_, time.ctime()))
x_train, x_valid = xtrain[xtrn_indx], xtrain[xvad_indx]
y_train, y_valid = label[xtrn_indx], label[xvad_indx]
clf_lr2 = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 clf_lr2.fit(x_train, y_train)
 pred_valid = clf_lr2.predict(x_valid ).reshape(-1,)
 score = accuracy_score(y_valid, pred_valid)
 Pred_Real = clf_lr2.predict_proba(xtest)[:, 1]
validation_pred_lr2[xvad_indx] +=(pred_valid ).reshape(-1,)
 scores_lr2.append(accuracy_score(y_valid, pred_valid))
forcast_lr2 += Pred_Real
valid_lr2[xvad_indx] +=(y_valid)
 print(c_report(valid_lr2[xvad_indx], validation_pred_lr2[xvad_indx]))
print(c_matrix(valid_lr2[xvad_indx], validation_pred_lr2[xvad_indx]))
print('accuracy is: {}, std: {}.'.format(np.mean(scores_lr2), np.std(scores_lr2)) )
pos, cat_cols = one_hot_encoder(pos, nan_as_category= True)
 aggregations = {
'MONTHS_BALANCE': ['max', 'mean', 'size'],
'SK_DPD': ['max', 'mean'],
'SK_DPD_DEF': ['max', 'mean']
}<groupby>
perm = PermutationImportance(clf_lr2, random_state=1 ).fit(xtrain, label)
 eli5.show_weights(perm, top=50 )
for cat in cat_cols:
aggregations[cat] = ['mean']
pos_agg = pos.groupby('SK_ID_CURR' ).agg(aggregations)
 pos_agg.columns = pd.Index(['POS_' + e[0] + "_" + e[1].upper() for e in pos_agg.columns.tolist() ])
 pos_agg['POS_COUNT'] = pos.groupby('SK_ID_CURR' ).size()
del pos
gc.collect()<train_model>
top_features = [i[1:] for i in eli5.formatters.as_dataframe.explain_weights_df(clf_lr2 ).feature if 'BIAS' not in i]
xtrain_lr_elif = train[top_features]
xtest_lr_elif = test[top_features]
scaler = StandardScaler()
xtrain_lr_elif = scaler.fit_transform(xtrain_lr_elif)
 xtest_lr_elif = scaler.transform(xtest_lr_elif)
 forcast_lr4 = np.zeros(len(xtest_lr_elif))
validation_pred_lr4 = np.zeros(len(xtrain_lr_elif))
scores_lr4 = []
valid_lr4 = np.zeros(len(label))
for fold_,(xtrn_indx, xvad_indx)in enumerate(folds.split(xtrain_lr_elif, label)) :
print('Fold {}, started at {}'.format(fold_, time.ctime()))
x_train, x_valid = xtrain_lr_elif[xtrn_indx], xtrain_lr_elif[xvad_indx]
y_train, y_valid = label[xtrn_indx], label[xvad_indx]
clf_lr4 = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 clf_lr4.fit(x_train, y_train)
 pred_valid = clf_lr4.predict(x_valid ).reshape(-1,)
 score = accuracy_score(y_valid, pred_valid)
 Pred_Real = clf_lr4.predict_proba(xtest_lr_elif)[:, 1]
validation_pred_lr4[xvad_indx] +=(pred_valid ).reshape(-1,)
 scores_lr4.append(accuracy_score(y_valid, pred_valid))
forcast_lr4 += Pred_Real
valid_lr4[xvad_indx] +=(y_valid)
 print(c_report(valid_lr4[xvad_indx], validation_pred_lr4[xvad_indx]))
print(c_matrix(valid_lr4[xvad_indx], validation_pred_lr4[xvad_indx]))
print('accuracy is: {}, std: {}.'.format(np.mean(scores_lr4), np.std(scores_lr4)) )
print("Start POS_CASH_balance................ " )<load_from_csv>
perm = PermutationImportance(clf_lr2, random_state=1 ).fit(xtrain, label)
 eli5.show_weights(perm, top=50 )
ins = pd.read_csv('.. /input/installments_payments.csv', nrows = num_rows)
 ins, cat_cols = one_hot_encoder(ins, nan_as_category= True )<feature_engineering>
top_features = [i[1:] for i in eli5.formatters.as_dataframe.explain_weights_df(perm ).feature if 'BIAS' not in i]
xtrain_lr_perm = train[top_features]
xtest_lr_perm = test[top_features]
scaler = StandardScaler()
xtrain_lr_perm = scaler.fit_transform(xtrain_lr_perm)
 xtest_lr_perm = scaler.transform(xtest_lr_perm)
 forcast_lr3 = np.zeros(len(xtest_lr_perm))
validation_pred_lr3 = np.zeros(len(xtrain_lr_perm))
scores_lr3 = []
valid_lr3 = np.zeros(len(label))
for fold_,(xtrn_indx, xvad_indx)in enumerate(folds.split(xtrain_lr_perm, label)) :
print('Fold {}, started at {}'.format(fold_, time.ctime()))
x_train, x_valid = xtrain_lr_perm[xtrn_indx], xtrain_lr_perm[xvad_indx]
y_train, y_valid = label[xtrn_indx], label[xvad_indx]
clf_lr3 = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 clf_lr3.fit(x_train, y_train)
 pred_valid = clf_lr3.predict(x_valid ).reshape(-1,)
 score = accuracy_score(y_valid, pred_valid)
 Pred_Real = clf_lr3.predict_proba(xtest_lr_perm)[:, 1]
validation_pred_lr3[xvad_indx] +=(pred_valid ).reshape(-1,)
 scores_lr3.append(accuracy_score(y_valid, pred_valid))
forcast_lr3 += Pred_Real
valid_lr3[xvad_indx] +=(y_valid)
 print(c_report(valid_lr3[xvad_indx], validation_pred_lr3[xvad_indx]))
print(c_matrix(valid_lr3[xvad_indx], validation_pred_lr3[xvad_indx]))
print('accuracy is: {}, std: {}.'.format(np.mean(scores_lr3), np.std(scores_lr3)) )
ins['PAYMENT_PERC'] = ins['AMT_PAYMENT'] / ins['AMT_INSTALMENT']
ins['PAYMENT_DIFF'] = ins['AMT_INSTALMENT'] - ins['AMT_PAYMENT']
ins['DPD'] = ins['DAYS_ENTRY_PAYMENT'] - ins['DAYS_INSTALMENT']
ins['DBD'] = ins['DAYS_INSTALMENT'] - ins['DAYS_ENTRY_PAYMENT']
ins['DPD'] = ins['DPD'].apply(lambda x: x if x > 0 else 0)
 ins['DBD'] = ins['DBD'].apply(lambda x: x if x > 0 else 0 )<groupby>
forcast_bc = np.zeros(len(xtest))
validation_pred_bc = np.zeros(len(xtrain))
scores_bc = []
valid_bc = np.zeros(len(label))
for fold_,(xtrn_indx, xvad_indx)in enumerate(folds.split(xtrain, label)) :
print('Fold {}, started at {}'.format(fold_, time.ctime()))
x_train, x_valid = xtrain[xtrn_indx], xtrain[xvad_indx]
y_train, y_valid = label[xtrn_indx], label[xvad_indx]
clf_bc = BaggingClassifier()
clf_bc.fit(x_train, y_train)
 pred_valid = clf_bc.predict(x_valid ).reshape(-1,)
 score = accuracy_score(y_valid, pred_valid)
 Pred_Real = clf_bc.predict_proba(xtest)[:, 1]
validation_pred_bc[xvad_indx] +=(pred_valid ).reshape(-1,)
 scores_bc.append(accuracy_score(y_valid, pred_valid))
forcast_bc += Pred_Real
valid_bc[xvad_indx] +=(y_valid)
 print(c_report(valid_bc[xvad_indx], validation_pred_bc[xvad_indx]))
print(c_matrix(valid_bc[xvad_indx], validation_pred_bc[xvad_indx]))
print('accuracy is: {}, std: {}.'.format(np.mean(scores_bc), np.std(scores_bc)) )
aggregations = {
'NUM_INSTALMENT_VERSION': ['nunique'],
'DPD': ['max', 'mean', 'sum'],
'DBD': ['max', 'mean', 'sum'],
'PAYMENT_PERC': ['max', 'mean', 'sum', 'var'],
'PAYMENT_DIFF': ['max', 'mean', 'sum', 'var'],
'AMT_INSTALMENT': ['max', 'mean', 'sum'],
'AMT_PAYMENT': ['min', 'max', 'mean', 'sum'],
'DAYS_ENTRY_PAYMENT': ['max', 'mean', 'sum']
}
for cat in cat_cols:
aggregations[cat] = ['mean']
ins_agg = ins.groupby('SK_ID_CURR' ).agg(aggregations)
 ins_agg.columns = pd.Index(['INS_' + e[0] + "_" + e[1].upper() for e in ins_agg.columns.tolist() ])
 ins_agg['INS_COUNT'] = ins.groupby('SK_ID_CURR' ).size()
del ins
gc.collect()<train_model>
forcast_adac = np.zeros(len(xtest))
validation_pred_adac = np.zeros(len(xtrain))
scores_adac = []
valid_adac = np.zeros(len(label))
for fold_,(xtrn_indx, xvad_indx)in enumerate(folds.split(xtrain, label)) :
print('Fold {}, started at {}'.format(fold_, time.ctime()))
x_train, x_valid = xtrain[xtrn_indx], xtrain[xvad_indx]
y_train, y_valid = label[xtrn_indx], label[xvad_indx]
clf_adac = AdaBoostClassifier()
clf_adac.fit(x_train, y_train)
 pred_valid = clf_adac.predict(x_valid ).reshape(-1,)
 score = accuracy_score(y_valid, pred_valid)
 Pred_Real = clf_adac.predict_proba(xtest)[:, 1]
validation_pred_adac[xvad_indx] +=(pred_valid ).reshape(-1,)
 scores_adac.append(accuracy_score(y_valid, pred_valid))
forcast_adac += Pred_Real
valid_adac[xvad_indx] +=(y_valid)
 print(c_report(valid_adac[xvad_indx], validation_pred_adac[xvad_indx]))
print(c_matrix(valid_adac[xvad_indx], validation_pred_adac[xvad_indx]))
print('accuracy is: {}, std: {}.'.format(np.mean(scores_adac), np.std(scores_adac)) )
print("End POS_CASH_balance................ " )<train_model>
forcast_gbc = np.zeros(len(xtest))
validation_pred_gbc = np.zeros(len(xtrain))
scores_gbc = []
valid_gbc = np.zeros(len(label))
for fold_,(xtrn_indx, xvad_indx)in enumerate(folds.split(xtrain, label)) :
print('Fold {}, started at {}'.format(fold_, time.ctime()))
x_train, x_valid = xtrain[xtrn_indx], xtrain[xvad_indx]
y_train, y_valid = label[xtrn_indx], label[xvad_indx]
clf_gbc = GradientBoostingClassifier()
clf_gbc.fit(x_train, y_train)
 pred_valid = clf_gbc.predict(x_valid ).reshape(-1,)
 score = accuracy_score(y_valid, pred_valid)
 Pred_Real = clf_gbc.predict_proba(xtest)[:, 1]
validation_pred_gbc[xvad_indx] +=(pred_valid ).reshape(-1,)
 scores_gbc.append(accuracy_score(y_valid, pred_valid))
forcast_gbc += Pred_Real
valid_gbc[xvad_indx] +=(y_valid)
 print(c_report(valid_gbc[xvad_indx], validation_pred_gbc[xvad_indx]))
print(c_matrix(valid_gbc[xvad_indx], validation_pred_gbc[xvad_indx]))
print('accuracy is: {}, std: {}.'.format(np.mean(scores_gbc), np.std(scores_gbc)) )
print("Start credit_card_balance................ " )<load_from_csv>
cat_params = {'learning_rate': 0.02,
'depth': 5,
'l2_leaf_reg': 10,
'bootstrap_type': 'Bernoulli',
'od_type': 'Iter',
'od_wait': 50,
'random_seed': 11,
'allow_writing_files': False}
forcast_catc = np.zeros(len(xtest))
validation_pred_catc = np.zeros(len(xtrain))
scores_catc = []
valid_catc = np.zeros(len(label))
for fold_,(xtrn_indx, xvad_indx)in enumerate(folds.split(xtrain, label)) :
x_train, x_valid = xtrain[xtrn_indx], xtrain[xvad_indx]
y_train, y_valid = label[xtrn_indx], label[xvad_indx]
clf_catc = CatBoostClassifier(iterations=400, **cat_params)
 clf_catc.fit(x_train, y_train)
 pred_valid = clf_catc.predict(x_valid ).reshape(-1,)
 score = accuracy_score(y_valid, pred_valid)
 Pred_Real = clf_catc.predict_proba(xtest)[:, 1]
validation_pred_catc[xvad_indx] +=(pred_valid ).reshape(-1,)
 scores_catc.append(accuracy_score(y_valid, pred_valid))
forcast_catc += Pred_Real
valid_catc[xvad_indx] +=(y_valid)
 print(c_report(valid_catc[xvad_indx], validation_pred_catc[xvad_indx]))
print(c_matrix(valid_catc[xvad_indx], validation_pred_catc[xvad_indx]))
print('accuracy is: {}, std: {}.'.format(np.mean(scores_catc), np.std(scores_catc)) )
cc = pd.read_csv('.. /input/credit_card_balance.csv', nrows = num_rows )<categorify>
sample_submission['target'] = forcast_lr4
sample_submission.to_csv('Forcasting.csv', index=False)
 sample_submission.head(20 )