Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<train_model>
warnings.filterwarnings('ignore' )
early_stop = EarlyStopping(
 patience=6, restore_best_weights=True,monitor = 'val_acc')
 model.fit(aug.flow(x_train, y_train,batch_size = 64),steps_per_epoch=len(x_train)/ 64, epochs=100,validation_data =(x_val,y_val),callbacks=[early_stop],use_multiprocessing = True, verbose = 0 )<prepare_output>
def seed_everything(seed=0):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed )
y_pred = np.argmax(model.predict(x_test),axis = 1)
 submission = pd.DataFrame({"imageId": np.arange(1, len(y_pred)+1),"Label":y_pred})
<save_to_csv>
def make_predictions(tr_df, tt_df, features_columns, target, lgb_params, NFOLDS=2):
folds = KFold(n_splits=NFOLDS, shuffle=True, random_state=SEED)
 X,y = tr_df[features_columns], tr_df[target]
P,P_y = tt_df[features_columns], tt_df[target]
tt_df = tt_df[['TransactionID',target]]
predictions = np.zeros(len(tt_df))
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X, y)) :
print('Fold:',fold_)
 tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]
vl_x, vl_y = X.iloc[val_idx,:], y[val_idx]
print(len(tr_x),len(vl_x))
tr_data = lgb.Dataset(tr_x, label=tr_y)
 if LOCAL_TEST:
vl_data = lgb.Dataset(P, label=P_y)
 else:
vl_data = lgb.Dataset(vl_x, label=vl_y)
 estimator = lgb.train(
 lgb_params,
tr_data,
valid_sets = [tr_data, vl_data],
verbose_eval = 200, 
)
 pp_p = estimator.predict(P)
 predictions += pp_p/NFOLDS
if LOCAL_TEST:
feature_imp = pd.DataFrame(sorted(zip(estimator.feature_importance() ,X.columns)) , columns=['Value','Feature'])
 print(feature_imp)
 del tr_x, tr_y, vl_x, vl_y, tr_data, vl_data
gc.collect()
tt_df['prediction'] = predictions
return tt_df
submission.to_csv("lenet_sub.csv",index=False )<import_modules>
SEED = 42
seed_everything(SEED)
 LOCAL_TEST = False
TARGET = 'isFraud'
START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d' )
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
import itertools
from keras.models import Sequential, load_model
from keras.layers import Flatten, Dense, MaxPool2D, Conv2D, Dropout, BatchNormalization
from keras.optimizers import RMSprop
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint
from keras.utils.np_utils import to_categorical
import tensorflow as tf<load_from_csv>
print('Load Data')
 train_df = pd.read_pickle('.. /input/ieee-data-minification/train_transaction.pkl')
 if LOCAL_TEST:
train_df['DT_M'] = train_df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 train_df['DT_M'] =(train_df['DT_M'].dt.year-2017)*12 + train_df['DT_M'].dt.month
test_df = train_df[train_df['DT_M']==train_df['DT_M'].max() ].reset_index(drop=True)
 train_df = train_df[train_df['DT_M']<(train_df['DT_M'].max() -1)].reset_index(drop=True)
 train_identity = pd.read_pickle('.. /input/ieee-data-minification/train_identity.pkl')
 test_identity = train_identity[train_identity['TransactionID'].isin(
 test_df['TransactionID'])].reset_index(drop=True)
 train_identity = train_identity[train_identity['TransactionID'].isin(
 train_df['TransactionID'])].reset_index(drop=True)
 del train_df['DT_M'], test_df['DT_M']
else:
test_df = pd.read_pickle('.. /input/ieee-data-minification/test_transaction.pkl')
 train_identity = pd.read_pickle('.. /input/ieee-data-minification/train_identity.pkl')
 test_identity = pd.read_pickle('.. /input/ieee-data-minification/test_identity.pkl')
 base_columns = list(train_df)+ list(train_identity)
 print('Shape control:', train_df.shape, test_df.shape )
train =pd.read_csv('.. /input/digit-recognizer/train.csv')
 test =pd.read_csv('.. /input/digit-recognizer/test.csv' )<prepare_x_and_y>
for df in [train_df, test_df]:
df['DT'] = df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 df['DT_M'] =(df['DT'].dt.year-2017)*12 + df['DT'].dt.month
df['DT_W'] =(df['DT'].dt.year-2017)*52 + df['DT'].dt.weekofyear
df['DT_D'] =(df['DT'].dt.year-2017)*365 + df['DT'].dt.dayofyear
df['DT_hour'] = df['DT'].dt.hour
df['DT_day_week'] = df['DT'].dt.dayofweek
df['DT_day'] = df['DT'].dt.day
df['D9'] = np.where(df['D9'].isna() ,0,1 )
y_train = train['label']
X_train = train.drop('label', axis=1 )<count_missing_values>
i_cols = ['card1']
for col in i_cols:
valid_card = pd.concat([train_df[[col]], test_df[[col]]])
 valid_card = valid_card[col].value_counts()
valid_card = valid_card[valid_card>2]
valid_card = list(valid_card.index)
 train_df[col] = np.where(train_df[col].isin(test_df[col]), train_df[col], np.nan)
 test_df[col] = np.where(test_df[col].isin(train_df[col]), test_df[col], np.nan)
 train_df[col] = np.where(train_df[col].isin(valid_card), train_df[col], np.nan)
 test_df[col] = np.where(test_df[col].isin(valid_card), test_df[col], np.nan )
train.isnull().sum()<count_missing_values>
i_cols = ['M1','M2','M3','M5','M6','M7','M8','M9']
for df in [train_df, test_df]:
df['M_sum'] = df[i_cols].sum(axis=1 ).astype(np.int8)
 df['M_na'] = df[i_cols].isna().sum(axis=1 ).astype(np.int8 )
train.isnull().sum()<categorify>
for col in ['ProductCD','M4']:
temp_dict = train_df.groupby([col])[TARGET].agg(['mean'] ).reset_index().rename(
 columns={'mean': col+'_target_mean'})
 temp_dict.index = temp_dict[col].values
temp_dict = temp_dict[col+'_target_mean'].to_dict()
train_df[col+'_target_mean'] = train_df[col].map(temp_dict)
 test_df[col+'_target_mean'] = test_df[col].map(temp_dict )
X_train = X_train / 255.0
test = test / 255.0
X_train = X_train.values.reshape(-1, 28,28,1)
 test = test.values.reshape(-1, 28,28,1)
 y_train = to_categorical(y_train, num_classes=10 )<train_model>
train_df['uid'] = train_df['card1'].astype(str)+'_'+train_df['card2'].astype(str)
 test_df['uid'] = test_df['card1'].astype(str)+'_'+test_df['card2'].astype(str)
 train_df['uid2'] = train_df['uid'].astype(str)+'_'+train_df['card3'].astype(str)+'_'+train_df['card5'].astype(str)
 test_df['uid2'] = test_df['uid'].astype(str)+'_'+test_df['card3'].astype(str)+'_'+test_df['card5'].astype(str)
 train_df['uid3'] = train_df['uid2'].astype(str)+'_'+train_df['addr1'].astype(str)+'_'+train_df['addr2'].astype(str)
 test_df['uid3'] = test_df['uid2'].astype(str)+'_'+test_df['addr1'].astype(str)+'_'+test_df['addr2'].astype(str)
 train_df['TransactionAmt_check'] = np.where(train_df['TransactionAmt'].isin(test_df['TransactionAmt']), 1, 0)
 test_df['TransactionAmt_check'] = np.where(test_df['TransactionAmt'].isin(train_df['TransactionAmt']), 1, 0)
 i_cols = ['card1','card2','card3','card5','uid','uid2','uid3']
for col in i_cols:
for agg_type in ['mean','std']:
new_col_name = col+'_TransactionAmt_'+agg_type
temp_df = pd.concat([train_df[[col, 'TransactionAmt']], test_df[[col,'TransactionAmt']]])
 temp_df = temp_df.groupby([col])['TransactionAmt'].agg([agg_type] ).reset_index().rename(
 columns={agg_type: new_col_name})
 temp_df.index = list(temp_df[col])
 temp_df = temp_df[new_col_name].to_dict()
train_df[new_col_name] = train_df[col].map(temp_df)
 test_df[new_col_name] = test_df[col].map(temp_df)
 train_df['TransactionAmt'] = np.log1p(train_df['TransactionAmt'])
 test_df['TransactionAmt'] = np.log1p(test_df['TransactionAmt'] )
X_train, X_test, y_train, y_test = train_test_split(X_train, y_train)
 print("X_train shape: ", X_train.shape,"
X_test shape: ", X_test.shape,
"
y_train shape: ", y_train.shape,"
y_test shape: ", y_test.shape )<init_hyperparams>
p = 'P_emaildomain'
r = 'R_emaildomain'
uknown = 'email_not_provided'
for df in [train_df, test_df]:
df[p] = df[p].fillna(uknown)
 df[r] = df[r].fillna(uknown)
 df['email_check'] = np.where(( df[p]==df[r])&(df[p]!=uknown),1,0)
 df[p+'_prefix'] = df[p].apply(lambda x: x.split('.')[0])
 df[r+'_prefix'] = df[r].apply(lambda x: x.split('.')[0])
IMG_SIZE = 28
batch_size= 86
AUTOTUNE = tf.data.experimental.AUTOTUNE
num_classes = 10<import_modules>
for df in [train_identity, test_identity]:
df['DeviceInfo'] = df['DeviceInfo'].fillna('unknown_device' ).str.lower()
df['DeviceInfo_device'] = df['DeviceInfo'].apply(lambda x: ''.join([i for i in x if i.isalpha() ]))
df['DeviceInfo_version'] = df['DeviceInfo'].apply(lambda x: ''.join([i for i in x if i.isnumeric() ]))
df['id_30'] = df['id_30'].fillna('unknown_device' ).str.lower()
df['id_30_device'] = df['id_30'].apply(lambda x: ''.join([i for i in x if i.isalpha() ]))
df['id_30_version'] = df['id_30'].apply(lambda x: ''.join([i for i in x if i.isnumeric() ]))
df['id_31'] = df['id_31'].fillna('unknown_device' ).str.lower()
df['id_31_device'] = df['id_31'].apply(lambda x: ''.join([i for i in x if i.isalpha() ]))
from keras.models import Sequential, load_model
from keras.layers import Flatten, Dense, MaxPool2D, Conv2D, Dropout, BatchNormalization
from keras.optimizers import RMSprop
from keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras import layers<choose_model_class>
temp_df = train_df[['TransactionID']]
temp_df = temp_df.merge(train_identity, on=['TransactionID'], how='left')
 del temp_df['TransactionID']
train_df = pd.concat([train_df,temp_df], axis=1)
 temp_df = test_df[['TransactionID']]
temp_df = temp_df.merge(test_identity, on=['TransactionID'], how='left')
 del temp_df['TransactionID']
test_df = pd.concat([test_df,temp_df], axis=1 )
def create_model(train_ds, val_ds, epochs):
model = tf.keras.Sequential([
layers.Conv2D(32,(3, 3), activation = 'relu', padding = 'same', input_shape=(28,28,1)) ,
layers.Conv2D(32,(3, 3), activation = 'relu', padding = 'same'),
layers.BatchNormalization() ,
layers.MaxPooling2D(2, 2),
layers.Dropout(0.2),
layers.Conv2D(64,(3, 3), activation = 'relu', padding = 'same'),
layers.Conv2D(64,(3, 3), activation = 'relu', padding = 'same'),
layers.BatchNormalization() ,
layers.MaxPooling2D(2, 2),
layers.Dropout(0.2),
layers.Conv2D(128,(3, 3), activation = 'relu', padding = 'same'),
layers.Conv2D(128,(3, 3), activation = 'relu', padding = 'same'),
layers.BatchNormalization() ,
layers.MaxPooling2D(2, 2),
layers.Dropout(0.2),
layers.Flatten() ,
layers.Dense(512, activation = 'relu'),
layers.BatchNormalization() ,
layers.Dropout(0.2),
layers.Dense(256, activation = 'relu'),
layers.BatchNormalization() ,
layers.Dropout(0.35),
layers.Dense(10, activation = 'softmax')
 ])
 model.compile(optimizer='adam',
loss=tf.keras.losses.CategoricalCrossentropy() ,
metrics=['accuracy'])
 print(model.summary())
 return model<train_model>
i_cols = ['card1','card2','card3','card5',
'C1','C2','C3','C4','C5','C6','C7','C8','C9','C10','C11','C12','C13','C14',
'D1','D2','D3','D4','D5','D6','D7','D8',
'addr1','addr2',
'dist1','dist2',
'P_emaildomain', 'R_emaildomain',
'DeviceInfo','DeviceInfo_device','DeviceInfo_version',
'id_30','id_30_device','id_30_version',
'id_31_device',
'id_33',
'uid','uid2','uid3',
]
for col in i_cols:
temp_df = pd.concat([train_df[[col]], test_df[[col]]])
 fq_encode = temp_df[col].value_counts(dropna=False ).to_dict()
train_df[col+'_fq_enc'] = train_df[col].map(fq_encode)
 test_df[col+'_fq_enc'] = test_df[col].map(fq_encode)
 for col in ['DT_M','DT_W','DT_D']:
temp_df = pd.concat([train_df[[col]], test_df[[col]]])
 fq_encode = temp_df[col].value_counts().to_dict()
train_df[col+'_total'] = train_df[col].map(fq_encode)
 test_df[col+'_total'] = test_df[col].map(fq_encode)
 periods = ['DT_M','DT_W','DT_D']
i_cols = ['uid']
for period in periods:
for col in i_cols:
new_column = col + '_' + period
temp_df = pd.concat([train_df[[col,period]], test_df[[col,period]]])
 temp_df[new_column] = temp_df[col].astype(str)+ '_' +(temp_df[period] ).astype(str)
 fq_encode = temp_df[new_column].value_counts().to_dict()
train_df[new_column] =(train_df[col].astype(str)+ '_' + train_df[period].astype(str)).map(fq_encode)
 test_df[new_column] =(test_df[col].astype(str)+ '_' + test_df[period].astype(str)).map(fq_encode)
 train_df[new_column] /= train_df[period+'_total']
test_df[new_column] /= test_df[period+'_total']
cnn_model = create_model(X_train, y_train, 100 )<choose_model_class>
for col in list(train_df):
if train_df[col].dtype=='O':
print(col)
 train_df[col] = train_df[col].fillna('unseen_before_label')
 test_df[col] = test_df[col].fillna('unseen_before_label')
 train_df[col] = train_df[col].astype(str)
 test_df[col] = test_df[col].astype(str)
 le = LabelEncoder()
le.fit(list(train_df[col])+list(test_df[col]))
train_df[col] = le.transform(train_df[col])
 test_df[col] = le.transform(test_df[col])
 train_df[col] = train_df[col].astype('category')
 test_df[col] = test_df[col].astype('category' )
optimizer = RMSprop(lr = 0.001, rho = 0.9, epsilon=1e-08, decay =0.0)
 cnn_model.compile(optimizer = optimizer, loss = 'categorical_crossentropy', metrics = ['accuracy'] )<choose_model_class>
lgb_params = {
'objective':'binary',
'boosting_type':'gbdt',
'metric':'auc',
'n_jobs':-1,
'learning_rate':0.01,
'num_leaves': 2**8,
'max_depth':-1,
'tree_learner':'serial',
'colsample_bytree': 0.7,
'subsample_freq':1,
'subsample':0.7,
'n_estimators':800,
'max_bin':255,
'verbose':-1,
'seed': SEED,
'early_stopping_rounds':100,
}
learning_rate_reduction = ReduceLROnPlateau(monitor='val_loss',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )<train_model>
if LOCAL_TEST:
lgb_params['learning_rate'] = 0.01
lgb_params['n_estimators'] = 20000
lgb_params['early_stopping_rounds'] = 100
test_predictions = make_predictions(train_df, test_df, features_columns, TARGET, lgb_params)
 print(metrics.roc_auc_score(test_predictions[TARGET], test_predictions['prediction']))
else:
lgb_params['learning_rate'] = 0.005
lgb_params['n_estimators'] = 1800
lgb_params['early_stopping_rounds'] = 100
test_predictions = make_predictions(train_df, test_df, features_columns, TARGET, lgb_params, NFOLDS=10 )
<train_model><EOS> 
if not LOCAL_TEST:
test_predictions['isFraud'] = test_predictions['prediction']
test_predictions[['TransactionID','isFraud']].to_csv('submission.csv', index=False )