Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<split>
import os
import numpy as np
import pandas as pd
from sklearn import preprocessing
import xgboost as xgb
import gc
X_train, X_test, y_train, y_test = train_test_split(train.iloc[:, 1:].values, train.iloc[:, 0].values, test_size=0.1 )<train_model>
%%time
train_transaction = pd.read_csv('.. /input/train_transaction.csv', index_col='TransactionID')
 test_transaction = pd.read_csv('.. /input/test_transaction.csv', index_col='TransactionID')
 train_identity = pd.read_csv('.. /input/train_identity.csv', index_col='TransactionID')
 test_identity = pd.read_csv('.. /input/test_identity.csv', index_col='TransactionID')
 sample_submission = pd.read_csv('.. /input/sample_submission.csv', index_col='TransactionID')
 train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)
 print(train.shape)
 print(test.shape)
 y_train = train['isFraud'].copy()
del train_transaction, train_identity, test_transaction, test_identity
X_train = train.drop('isFraud', axis=1)
 X_test = test.copy()
del train, test
many_null_cols = [col for col in X_train.columns if X_train[col].isnull().sum() / X_train.shape[0] > 0.96]
many_null_cols_X_test = [col for col in X_test.columns if X_test[col].isnull().sum() / X_test.shape[0] > 0.96]
big_top_value_cols = [col for col in X_train.columns if X_train[col].value_counts(dropna=False, normalize=True ).values[0] > 0.96]
big_top_value_cols_X_test = [col for col in X_test.columns if X_test[col].value_counts(dropna=False, normalize=True ).values[0] > 0.96]
cols_to_drop = list(set(many_null_cols + many_null_cols_X_test + big_top_value_cols + big_top_value_cols_X_test))
len(cols_to_drop)
 print(cols_to_drop)
 X_train = X_train.drop(cols_to_drop, axis=1)
 X_test = X_test.drop(cols_to_drop, axis=1)
 X_train.drop('TransactionDT', axis=1, inplace=True)
 X_test.drop('TransactionDT', axis=1, inplace=True)
 print(X_train.shape)
 print(X_test.shape)
 for f in X_train.columns:
if X_train[f].dtype=='object' or X_test[f].dtype=='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(X_train[f].values)+ list(X_test[f].values))
X_train[f] = lbl.transform(list(X_train[f].values))
X_test[f] = lbl.transform(list(X_test[f].values))
X_train = X_train.fillna(-999)
 X_test = X_test.fillna(-999 )
X_train = X_train.reshape(X_train.shape[0], 28, 28, 1)
 X_test = X_test.reshape(X_test.shape[0], 28, 28, 1 )<categorify>
print("Before OverSampling, counts of label '1': {}".format(sum(y_train==1)))
 print("Before OverSampling, counts of label '0': {}
".format(sum(y_train==0)))
 sm = KMeansSMOTE(random_state=99, sampling_strategy = 0.15, k_neighbors = 10,cluster_balance_threshold = 0.02, n_jobs=4)
 X_train_new, y_train_new = sm.fit_sample(X_train, y_train.ravel())
 X_train_new = pd.DataFrame(X_train_new)
 X_train_new.columns = X_train.columns
y_train_new = pd.DataFrame(y_train_new)
 print('After OverSampling, the shape of X_train_new: {}'.format(X_train_new.shape))
print('After OverSampling, the shape of y_train_new: {}
'.format(y_train_new.shape))
y_train = np_utils.to_categorical(y_train, 10)
 y_test = np_utils.to_categorical(y_test, 10)
 print('Data y origin ', y_train[0])
 print('Data y one-hot encoding ',y_train[0] )<choose_model_class>
%%time
EPOCHS = 4
y_preds = np.zeros(sample_submission.shape[0])
 kf = StratifiedKFold(n_splits=EPOCHS, random_state= 99, shuffle=True)
 y_oof_new = np.zeros(X_train_new.shape[0])
 gc.collect()
for tr_idx, val_idx in kf.split(X_train_new, y_train_new):
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=17,
learning_rate=0.03,
subsample=0.9,
colsample_bytree=0.9,
tree_method='gpu_hist',
missing=-999 
)
 X_tr, X_vl = X_train_new.iloc[tr_idx, :], X_train_new.iloc[val_idx, :]
y_tr, y_vl = y_train_new.iloc[tr_idx], y_train_new.iloc[val_idx]
clf.fit(X_tr, y_tr)
 y_pred_train = clf.predict_proba(X_vl)[:,1]
print('ROC AUC {}'.format(roc_auc_score(y_vl, y_pred_train)))
 y_oof_new[val_idx] = y_pred_train
y_preds+= clf.predict_proba(X_test)[:,1] / EPOCHS
del clf
gc.collect()
print('ROC AUC oof_new {}'.format(roc_auc_score(y_train_new, y_oof_new)))
 del X_train_new
gc.collect()
sample_submission1a = sample_submission.copy()
sample_submission1a['isFraud'] = y_preds
sample_submission1a.to_csv('xgboost_oversample.csv')
 sample_submission1a['isFraud'].describe()
model = Sequential()
model.add(Conv2D(32,(3, 3), activation='sigmoid', input_shape=(28,28,1)))
 model.add(Conv2D(32,(3, 3), activation='sigmoid'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Flatten())
 model.add(Dense(128, activation='sigmoid'))
model.add(Dense(10, activation='softmax'))<choose_model_class>
print("Before OverSampling, counts of label '1': {}".format(sum(y_train==1)))
 print("Before OverSampling, counts of label '0': {}
".format(sum(y_train==0)))
 sm = SMOTE(random_state=99, sampling_strategy = 0.15)
 X_train_new, y_train_new = sm.fit_sample(X_train, y_train.ravel())
 X_train_new = pd.DataFrame(X_train_new)
 X_train_new.columns = X_train.columns
y_train_new = pd.DataFrame(y_train_new)
 print('After OverSampling, the shape of X_train_new: {}'.format(X_train_new.shape))
print('After OverSampling, the shape of y_train_new: {}
'.format(y_train_new.shape))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'] )<train_model>
%%time
EPOCHS = 4
y_preds = np.zeros(sample_submission.shape[0])
 kf = StratifiedKFold(n_splits=EPOCHS, random_state= 99, shuffle=True)
 y_oof_new = np.zeros(X_train_new.shape[0])
 gc.collect()
for tr_idx, val_idx in kf.split(X_train_new, y_train_new):
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=17,
learning_rate=0.02,
subsample=0.9,
colsample_bytree=0.9,
tree_method='gpu_hist',
missing=-999 
)
 X_tr, X_vl = X_train_new.iloc[tr_idx, :], X_train_new.iloc[val_idx, :]
y_tr, y_vl = y_train_new.iloc[tr_idx], y_train_new.iloc[val_idx]
clf.fit(X_tr, y_tr)
 y_pred_train = clf.predict_proba(X_vl)[:,1]
print('ROC AUC {}'.format(roc_auc_score(y_vl, y_pred_train)))
 y_oof_new[val_idx] = y_pred_train
y_preds+= clf.predict_proba(X_test)[:,1] / EPOCHS
del clf
gc.collect()
print('ROC AUC oof_new {}'.format(roc_auc_score(y_train_new, y_oof_new)))
 del X_train_new
gc.collect()
sample_submission1 = sample_submission.copy()
sample_submission1['isFraud'] = y_preds
sample_submission1.to_csv('xgboost_oversample.csv')
 sample_submission1['isFraud'].describe()
datagen = ImageDataGenerator(
 rotation_range= 10,
zoom_range = 0.1,
width_shift_range = 0.1,
height_shift_range = 0.1 
)
 datagen.fit(X_train )<define_variables>
print("Before OverSampling, counts of label '1': {}".format(sum(y_train==1)))
 print("Before OverSampling, counts of label '0': {}
".format(sum(y_train==0)))
 sm = RandomOverSampler(random_state=99, sampling_strategy = 0.12)
 X_train_new, y_train_new = sm.fit_sample(X_train, y_train.ravel())
 X_train_new = pd.DataFrame(X_train_new)
 X_train_new.columns = X_train.columns
y_train_new = pd.DataFrame(y_train_new)
 print('After OverSampling, the shape of X_train_new: {}'.format(X_train_new.shape))
print('After OverSampling, the shape of y_train_new: {}
'.format(y_train_new.shape))
train_generator = datagen.flow(X_train, y_train, batch_size = 32)
 validation_generator = datagen.flow(X_test, y_test, batch_size = 32 )<train_model>
%%time
EPOCHS = 4
y_preds = np.zeros(sample_submission.shape[0])
 kf = StratifiedKFold(n_splits=EPOCHS, random_state= 99, shuffle=True)
 y_oof_new = np.zeros(X_train_new.shape[0])
 gc.collect()
for tr_idx, val_idx in kf.split(X_train_new, y_train_new):
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=11,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.9,
tree_method='gpu_hist',
missing=-999,
min_child_weight=1 
)
 X_tr, X_vl = X_train_new.iloc[tr_idx, :], X_train_new.iloc[val_idx, :]
y_tr, y_vl = y_train_new.iloc[tr_idx], y_train_new.iloc[val_idx]
clf.fit(X_tr, y_tr)
 y_pred_train = clf.predict_proba(X_vl)[:,1]
print('ROC AUC {}'.format(roc_auc_score(y_vl, y_pred_train)))
 y_oof_new[val_idx] = y_pred_train
y_preds+= clf.predict_proba(X_test)[:,1] / EPOCHS
del clf
gc.collect()
print('ROC AUC oof_new {}'.format(roc_auc_score(y_train_new, y_oof_new)))
 del X_train_new
gc.collect()
sample_submission2 = sample_submission.copy()
sample_submission2['isFraud'] = y_preds
sample_submission2.to_csv('xgboost_oversample_random.csv')
 sample_submission2['isFraud'].describe()
%%time
H = model.fit_generator(train_generator,
steps_per_epoch = X_train.shape[0] // 32,
epochs=50,
validation_data = validation_generator,
validation_steps = X_test.shape[0] // 32,
verbose=1 )<compute_test_metric>
print("Before OverSampling, counts of label '1': {}".format(sum(y_train==1)))
 print("Before OverSampling, counts of label '0': {}
".format(sum(y_train==0)))
 sm = RandomOverSampler(random_state=999, sampling_strategy = 0.13)
 X_train_new, y_train_new = sm.fit_sample(X_train, y_train.ravel())
 X_train_new = pd.DataFrame(X_train_new)
 X_train_new.columns = X_train.columns
y_train_new = pd.DataFrame(y_train_new)
 print('After OverSampling, the shape of X_train_new: {}'.format(X_train_new.shape))
print('After OverSampling, the shape of y_train_new: {}
'.format(y_train_new.shape))
score = model.evaluate(X_test, y_test, verbose=0)
 print(score )<predict_on_test>
%%time
EPOCHS = 4
y_preds = np.zeros(sample_submission.shape[0])
 kf = StratifiedKFold(n_splits=EPOCHS, random_state= 99, shuffle=True)
 y_oof_new = np.zeros(X_train_new.shape[0])
 gc.collect()
for tr_idx, val_idx in kf.split(X_train_new, y_train_new):
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=17,
learning_rate=0.035,
subsample=0.9,
colsample_bytree=0.9,
tree_method='gpu_hist',
missing=-999 
)
 X_tr, X_vl = X_train_new.iloc[tr_idx, :], X_train_new.iloc[val_idx, :]
y_tr, y_vl = y_train_new.iloc[tr_idx], y_train_new.iloc[val_idx]
clf.fit(X_tr, y_tr)
 y_pred_train = clf.predict_proba(X_vl)[:,1]
print('ROC AUC {}'.format(roc_auc_score(y_vl, y_pred_train)))
 y_oof_new[val_idx] = y_pred_train
y_preds+= clf.predict_proba(X_test)[:,1] / EPOCHS
del clf
gc.collect()
print('ROC AUC oof_new {}'.format(roc_auc_score(y_train_new, y_oof_new)))
 del X_train_new
gc.collect()
sample_submission3 = sample_submission.copy()
sample_submission3['isFraud'] = y_preds
sample_submission3.to_csv('xgboost_oversample_random.csv')
 sample_submission3['isFraud'].describe()
plt.imshow(X_test[0].reshape(28,28), cmap='gray')
 y_predict = model.predict(X_test[0].reshape(1,28,28,1))
print('Value: ', np.argmax(y_predict))<predict_on_test>
%%time
EPOCHS = 4
TIMES = 24
frac_inc = np.arange(0,TIMES,1)/400 +0.1
print(frac_inc)
 y_preds = np.zeros(sample_submission.shape[0])
 for _ in range(TIMES):
kf = StratifiedKFold(n_splits=EPOCHS, random_state= _, shuffle=True)
 X_train_new = X_train[y_train == 0].sample(frac = frac_inc[_] ).append(X_train[y_train == 1])
 y_train_new = y_train[X_train_new.index].reset_index().drop(columns = 'TransactionID')
 X_train_new = X_train_new.reset_index().drop(columns = 'TransactionID')
 y_oof_new = np.zeros(X_train_new.shape[0])
 gc.collect()
for tr_idx, val_idx in kf.split(X_train_new, y_train_new):
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=10,
learning_rate=0.04,
subsample=0.8,
colsample_bytree=0.9,
tree_method='gpu_hist',
missing=-999,
min_child_weight=2 
)
 X_tr, X_vl = X_train_new.iloc[tr_idx, :], X_train_new.iloc[val_idx, :]
y_tr, y_vl = y_train_new.iloc[tr_idx], y_train_new.iloc[val_idx]
clf.fit(X_tr, y_tr)
 y_pred_train = clf.predict_proba(X_vl)[:,1]
y_oof_new[val_idx] = y_pred_train
y_preds+= clf.predict_proba(X_test)[:,1] / EPOCHS
print('ROC AUC oof_new {}'.format(roc_auc_score(y_train_new, y_oof_new)))
 sample_submission4 = sample_submission.copy()
sample_submission4['isFraud'] = y_preds/TIMES
sample_submission4.to_csv('xgboost_undersample.csv')
 sample_submission4['isFraud'].describe()
%%time
y_pred = model.predict(sub )<prepare_output>
sample_submission_blend = sample_submission.copy()
sample_submission_blend['isFraud'] =(sample_submission1a['isFraud']+sample_submission1['isFraud'] + sample_submission2['isFraud']*0.5+ sample_submission3['isFraud']*0.5+ sample_submission4['isFraud'])/4
sample_submission_blend.to_csv('xgboost_under_over_blend.csv')
 sample_submission_blend2 = sample_submission.copy()
sample_submission_blend2['isFraud'] =(sample_submission1a['isFraud']*0.5+sample_submission1['isFraud']*0.5 + sample_submission2['isFraud']*0.25+ sample_submission3['isFraud']*0.25+ sample_submission4['isFraud'])/2.5
sample_submission_blend2.to_csv('xgboost_under_over_blend2.csv')
 sample_submission_blend_equal = sample_submission.copy()
sample_submission_blend_equal['isFraud'] =(sample_submission1a['isFraud']+sample_submission1['isFraud'] + sample_submission2['isFraud']+ sample_submission3['isFraud']+ sample_submission4['isFraud'])/5
sample_submission_blend_equal.to_csv('xgboost_under_over_blend_equal.csv' )