Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<import_modules>
warnings.simplefilter('ignore')
 sns.set()
%matplotlib inline
from keras.layers import Activation,Dropout,Dense,Conv2D,AveragePooling2D,MaxPooling2D,Flatten,ZeroPadding2D,BatchNormalization
from tensorflow.keras import optimizers
from tensorflow.keras.models import Sequential
from tensorflow.keras.losses import categorical_crossentropy
from keras.callbacks import EarlyStopping 
<choose_model_class>
files = ['.. /input/test_identity.csv',
'.. /input/test_transaction.csv',
'.. /input/train_identity.csv',
'.. /input/train_transaction.csv',
'.. /input/sample_submission.csv']
class LeNet5(Sequential):
def __init__(self, input_shape=(rows, cols, channels),activation='tanh',pooling='avg',dropout = 0, name="Base"):
super().__init__(name='LeNet5_'+name)
 self.add(Conv2D(6, kernel_size=(5, 5), strides=(1, 1), activation=activation, input_shape=input_shape, padding="same"))
if pooling == 'avg':
self.add(AveragePooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
else:
self.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
self.add(Conv2D(16, kernel_size=(5, 5), strides=(1, 1), activation=activation, padding='valid'))
if pooling == 'avg':
self.add(AveragePooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
else:
self.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
if dropout > 0:
self.add(( Dropout(rate = dropout)))
 self.add(Flatten())
 self.add(Dense(120, activation=activation))
self.add(Dense(84, activation=activation))
self.add(Dense(10, activation='softmax'))
self.compile(optimizer='adam',
loss=categorical_crossentropy,
metrics=['accuracy'])
 class LeNet5BN(Sequential):
def __init__(self, input_shape=(rows, cols, channels), name="ImprovedBN"):
super().__init__(name='LeNet5_'+name)
 self.add(Conv2D(6, kernel_size=(5, 5), strides=(1, 1), input_shape=input_shape, padding="same"))
self.add(BatchNormalization())
 self.add(Activation('relu'))
self.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
self.add(Conv2D(16, kernel_size=(5, 5), strides=(1, 1), padding='valid'))
self.add(BatchNormalization())
 self.add(Activation('relu'))
self.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
self.add(Flatten())
 self.add(Dense(120))
self.add(BatchNormalization())
 self.add(Activation('relu'))
self.add(Dense(84))
self.add(BatchNormalization())
 self.add(Activation('relu'))
self.add(Dense(10, activation='softmax'))
self.compile(optimizer='adam',
loss=categorical_crossentropy,
metrics=['accuracy'] )<train_model>
%%time
def load_data(file):
return pd.read_csv(file)
 with multiprocessing.Pool() as pool:
test_id, test_tr, train_id, train_tr, sub = pool.map(load_data, files )
early_stop = EarlyStopping(patience=5, monitor='val_accuracy', restore_best_weights=True)
 model_LeNet5_base.fit(X_train, y=Y_train,
epochs=25, batch_size=128,
validation_data=(X_val, Y_val),
callbacks=[early_stop] )<compute_test_metric>
train = pd.merge(train_tr, train_id, on='TransactionID', how='left')
 test = pd.merge(test_tr, test_id, on='TransactionID', how='left')
 del test_id, test_tr, train_id, train_tr
gc.collect()
preds1 = model_LeNet5_base.evaluate(x = X_val, y = Y_val)
 print("Loss = " + str(preds1[0]))
print("Val Accuracy = " + str(preds1[1]))<choose_model_class>
useful_features = ['TransactionAmt', 'ProductCD', 'card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'addr1', 'addr2', 'dist1',
'P_emaildomain', 'R_emaildomain', 'C1', 'C2', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11', 'C12', 'C13',
'C14', 'D1', 'D2', 'D3', 'D4', 'D5', 'D6', 'D8', 'D9', 'D10', 'D11', 'D12', 'D13', 'D14', 'D15', 'M2', 'M3',
'M4', 'M5', 'M6', 'M7', 'M8', 'M9', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V17',
'V19', 'V20', 'V29', 'V30', 'V33', 'V34', 'V35', 'V36', 'V37', 'V38', 'V40', 'V44', 'V45', 'V46', 'V47', 'V48',
'V49', 'V51', 'V52', 'V53', 'V54', 'V56', 'V58', 'V59', 'V60', 'V61', 'V62', 'V63', 'V64', 'V69', 'V70', 'V71',
'V72', 'V73', 'V74', 'V75', 'V76', 'V78', 'V80', 'V81', 'V82', 'V83', 'V84', 'V85', 'V87', 'V90', 'V91', 'V92',
'V93', 'V94', 'V95', 'V96', 'V97', 'V99', 'V100', 'V126', 'V127', 'V128', 'V130', 'V131', 'V138', 'V139', 'V140',
'V143', 'V145', 'V146', 'V147', 'V149', 'V150', 'V151', 'V152', 'V154', 'V156', 'V158', 'V159', 'V160', 'V161',
'V162', 'V163', 'V164', 'V165', 'V166', 'V167', 'V169', 'V170', 'V171', 'V172', 'V173', 'V175', 'V176', 'V177',
'V178', 'V180', 'V182', 'V184', 'V187', 'V188', 'V189', 'V195', 'V197', 'V200', 'V201', 'V202', 'V203', 'V204',
'V205', 'V206', 'V207', 'V208', 'V209', 'V210', 'V212', 'V213', 'V214', 'V215', 'V216', 'V217', 'V219', 'V220',
'V221', 'V222', 'V223', 'V224', 'V225', 'V226', 'V227', 'V228', 'V229', 'V231', 'V233', 'V234', 'V238', 'V239',
'V242', 'V243', 'V244', 'V245', 'V246', 'V247', 'V249', 'V251', 'V253', 'V256', 'V257', 'V258', 'V259', 'V261',
'V262', 'V263', 'V264', 'V265', 'V266', 'V267', 'V268', 'V270', 'V271', 'V272', 'V273', 'V274', 'V275', 'V276',
'V277', 'V278', 'V279', 'V280', 'V282', 'V283', 'V285', 'V287', 'V288', 'V289', 'V291', 'V292', 'V294', 'V303',
'V304', 'V306', 'V307', 'V308', 'V310', 'V312', 'V313', 'V314', 'V315', 'V317', 'V322', 'V323', 'V324', 'V326',
'V329', 'V331', 'V332', 'V333', 'V335', 'V336', 'V338', 'id_01', 'id_02', 'id_03', 'id_05', 'id_06', 'id_09',
'id_11', 'id_12', 'id_13', 'id_14', 'id_15', 'id_17', 'id_19', 'id_20', 'id_30', 'id_31', 'id_32', 'id_33',
'id_36', 'id_37', 'id_38', 'DeviceType', 'DeviceInfo']
model_LeNet5 = LeNet5BN(name="Improved_BN")
 model_LeNet5.summary()<train_model>
cols_to_drop = [col for col in train.columns if col not in useful_features]
cols_to_drop.remove('isFraud')
 cols_to_drop.remove('TransactionID')
 cols_to_drop.remove('TransactionDT' )
early_stop = EarlyStopping(patience=5, monitor='val_accuracy', restore_best_weights=True)
 model_LeNet5.fit(X_train, y=Y_train,
epochs=25, batch_size=128,
validation_data=(X_val, Y_val),
callbacks=[early_stop] )<compute_test_metric>
print('{} features are going to be dropped for being useless'.format(len(cols_to_drop)))
 train = train.drop(cols_to_drop, axis=1)
 test = test.drop(cols_to_drop, axis=1 )
preds2 = model_LeNet5.evaluate(x = X_val, y = Y_val)
 print("Loss = " + str(preds2[0]))
print("Val Accuracy = " + str(preds2[1]))<split>
train['TransactionAmt_decimal'] =(( train['TransactionAmt'] - train['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 test['TransactionAmt_decimal'] =(( test['TransactionAmt'] - test['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 train['card1_count_full'] = train['card1'].map(pd.concat([train['card1'], test['card1']], ignore_index=True ).value_counts(dropna=False))
test['card1_count_full'] = test['card1'].map(pd.concat([train['card1'], test['card1']], ignore_index=True ).value_counts(dropna=False))
train['Transaction_day_of_week'] = np.floor(( train['TransactionDT'] /(3600 * 24)- 1)% 7)
 test['Transaction_day_of_week'] = np.floor(( test['TransactionDT'] /(3600 * 24)- 1)% 7)
 train['Transaction_hour'] = np.floor(train['TransactionDT'] / 3600)% 24
test['Transaction_hour'] = np.floor(test['TransactionDT'] / 3600)% 24
for feature in ['id_02__id_20', 'id_02__D8', 'D11__DeviceInfo', 'DeviceInfo__P_emaildomain', 'P_emaildomain__C2',
'card2__dist1', 'card1__card5', 'card2__id_20', 'card5__P_emaildomain', 'addr1__card1']:
f1, f2 = feature.split('__')
 train[feature] = train[f1].astype(str)+ '_' + train[f2].astype(str)
 test[feature] = test[f1].astype(str)+ '_' + test[f2].astype(str)
 le = LabelEncoder()
le.fit(list(train[feature].astype(str ).values)+ list(test[feature].astype(str ).values))
train[feature] = le.transform(list(train[feature].astype(str ).values))
test[feature] = le.transform(list(test[feature].astype(str ).values))
for feature in ['id_34', 'id_36']:
if feature in useful_features:
train[feature + '_count_full'] = train[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
test[feature + '_count_full'] = test[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
for feature in ['id_01', 'id_31', 'id_33', 'id_35', 'id_36']:
if feature in useful_features:
train[feature + '_count_dist'] = train[feature].map(train[feature].value_counts(dropna=False))
test[feature + '_count_dist'] = test[feature].map(test[feature].value_counts(dropna=False))
conf_mat, class_report, accuracy, faults = analyze_model(model_LeNet5, X_val, Y_val )<feature_engineering>
for col in tqdm_notebook(train.columns):
if train[col].dtype == 'object':
le = LabelEncoder()
le.fit(list(train[col].astype(str ).values)+ list(test[col].astype(str ).values))
train[col] = le.transform(list(train[col].astype(str ).values))
test[col] = le.transform(list(test[col].astype(str ).values))
show_faults(faults )<set_options>
X = train.sort_values('TransactionDT' ).drop(['isFraud', 'TransactionDT', 'TransactionID'], axis=1)
 y = train.sort_values('TransactionDT')['isFraud']
test = test.sort_values('TransactionDT' ).drop(['TransactionDT', 'TransactionID'], axis=1 )
datagen = ImageDataGenerator(rotation_range=20,
width_shift_range=0.1,
shear_range=0.2,
zoom_range=0.1,
data_format="channels_last")
<define_variables>
del train
gc.collect()
X_train_aug = X_train
Y_train_aug = Y_train
print("Original train size: " , X_train_aug.shape)
 target_size = 32200
batch_size = 200
digits_gen = datagen.flow(X_train, Y_train, batch_size=batch_size)
 for i in range(target_size//batch_size):
batch = digits_gen.next()
X_train_aug = np.append(X_train_aug, batch[0], axis=0)
 Y_train_aug = np.append(Y_train_aug, batch[1], axis=0)
 print("Augmented train size: " , X_train_aug.shape)
<choose_model_class>
params = {'num_leaves': 491,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.3797454081646243,
'bagging_fraction': 0.4181193142567742,
'min_data_in_leaf': 106,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.006883242363721497,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
'random_state': 47
}
model_LeNet5_aug = LeNet5BN(name="Augmented_BN" )<train_model>
folds = TimeSeriesSplit(n_splits=5)
 aucs = list()
feature_importances = pd.DataFrame()
feature_importances['feature'] = X.columns
training_start_time = time()
for fold,(trn_idx, test_idx)in enumerate(folds.split(X, y)) :
start_time = time()
print('Training on fold {}'.format(fold + 1))
trn_data = lgb.Dataset(X.iloc[trn_idx], label=y.iloc[trn_idx])
 val_data = lgb.Dataset(X.iloc[test_idx], label=y.iloc[test_idx])
 clf = lgb.train(params, trn_data, 10000, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds=500)
 feature_importances['fold_{}'.format(fold + 1)] = clf.feature_importance()
aucs.append(clf.best_score['valid_1']['auc'])
 print('Fold {} finished in {}'.format(fold + 1, str(datetime.timedelta(seconds=time() - start_time))))
print('-' * 30)
 print('Training has finished.')
 print('Total training time is {}'.format(str(datetime.timedelta(seconds=time() - training_start_time))))
print('Mean AUC:', np.mean(aucs))
print('-' * 30 )
early_stop = EarlyStopping(patience=5, monitor='loss', restore_best_weights=True)
 batch_size = 128
model_LeNet5_aug.fit(X_train_aug, Y_train_aug, batch_size=batch_size,
epochs=100,
validation_data=(X_val, Y_val),
callbacks=[early_stop])
<compute_test_metric>
feature_importances['average'] = feature_importances[['fold_{}'.format(fold + 1)for fold in range(folds.n_splits)]].mean(axis=1)
 feature_importances.to_csv('feature_importances.csv')
 plt.figure(figsize=(16, 16))
sns.barplot(data=feature_importances.sort_values(by='average', ascending=False ).head(50), x='average', y='feature');
plt.title('50 TOP feature importance over {} folds average'.format(folds.n_splits)) ;
preds3 = model_LeNet5_aug.evaluate(x = X_val, y = Y_val)
 print("Loss = " + str(preds3[0]))
print("Val Accuracy = " + str(preds3[1]))<split>
best_iter = clf.best_iteration
conf_mat, class_report, accuracy, faults = analyze_model(model_LeNet5_aug, X_val, Y_val )<predict_on_test>
clf = lgb.LGBMClassifier(**params, num_boost_round=best_iter)
 clf.fit(X, y )
y_pred = model_LeNet5_aug.predict(X_train_aug,batch_size=None, verbose=0)
 mask = np.argmax(Y_train_aug, axis=1)== np.argmax(y_pred, axis=1)
 x_hit = X_train_aug[mask]
y_hit = Y_train_aug[mask]
x_miss = X_train_aug[~mask]
y_miss = Y_train_aug[~mask]
num_misses = y_miss.shape[0]
sample_idx = sample(range(y_hit.shape[0]), y_miss.shape[0])
 x_hit_reduced = x_hit[sample_idx]
y_hit_reduced = y_hit[sample_idx]
X_train2 = np.append(x_miss,x_hit_reduced, axis=0)
 Y_train2 = np.append(y_miss,y_hit_reduced, axis=0)
 idx_shuffle = list(range(X_train2.shape[0]))
np.random.shuffle(idx_shuffle)
 X_train2 = X_train2[idx_shuffle]
Y_train2 = Y_train2[idx_shuffle] 
<choose_model_class>
sub['isFraud'] = clf.predict_proba(test)[:, 1]
<train_model><EOS> 
sub.to_csv('ieee_cis_fraud_detection_v2.csv', index=False )