Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<compute_test_metric>
sns.set()
%matplotlib inline
warnings.filterwarnings('ignore')
 print(os.listdir(".. /input/ieee-fraud-detection"))
print("Ready!" )
def mcc(tp, tn, fp, fn):
sup = tp * tn - fp * fn
inf =(tp + fp)*(tp + fn)*(tn + fp)*(tn + fn)
 if inf == 0:
return 0
else:
return sup / np.sqrt(inf )<compute_test_metric>
%%time
train_transaction = pd.read_csv('.. /input/ieee-fraud-detection/train_transaction.csv', index_col='TransactionID')
 test_transaction = pd.read_csv('.. /input/ieee-fraud-detection/test_transaction.csv', index_col='TransactionID')
 train_identity = pd.read_csv('.. /input/ieee-fraud-detection/train_identity.csv', index_col='TransactionID')
 test_identity = pd.read_csv('.. /input/ieee-fraud-detection/test_identity.csv', index_col='TransactionID')
 print("Data is loaded!" )
def eval_mcc(y_true, y_prob, show=False):
idx = np.argsort(y_prob)
 y_true_sort = y_true[idx]
n = y_true.shape[0]
nump = 1.0 * np.sum(y_true)
 numn = n - nump
tp = nump
tn = 0.0
fp = numn
fn = 0.0
best_mcc = 0.0
best_id = -1
mccs = np.zeros(n)
 for i in range(n):
if y_true_sort[i] == 1:
tp -= 1.0
fn += 1.0
else:
fp -= 1.0
tn += 1.0
new_mcc = mcc(tp, tn, fp, fn)
 mccs[i] = new_mcc
if new_mcc >= best_mcc:
best_mcc = new_mcc
best_id = i
if show:
best_proba = y_prob[idx[best_id]]
y_pred =(y_prob > best_proba ).astype(int)
 return best_proba, best_mcc, y_pred
else:
return best_mcc<compute_test_metric>
missing_values_count = train_transaction.isnull().sum()
print(missing_values_count[0:10])
 total_cells = np.product(train_transaction.shape)
 total_missing = missing_values_count.sum()
print("% of missing data = ",(total_missing/total_cells)* 100 )
def mcc_eval(y_prob, dtrain):
y_true = dtrain.get_label()
best_mcc = eval_mcc(y_true, y_prob)
 return 'MCC', best_mcc<define_variables>
%%time
folder_path = '.. /input/ieee-fraud-detection/'
print('Loading data...')
 train_identity = pd.read_csv(f'{folder_path}train_identity.csv', index_col='TransactionID')
 print('\tSuccessfully loaded train_identity!')
 train_transaction = pd.read_csv(f'{folder_path}train_transaction.csv', index_col='TransactionID')
 print('\tSuccessfully loaded train_transaction!')
 test_identity = pd.read_csv(f'{folder_path}test_identity.csv', index_col='TransactionID')
 print('\tSuccessfully loaded test_identity!')
 test_transaction = pd.read_csv(f'{folder_path}test_transaction.csv', index_col='TransactionID')
 print('\tSuccessfully loaded test_transaction!')
 sub = pd.read_csv(f'{folder_path}sample_submission.csv')
 print('\tSuccessfully loaded sample_submission!')
 print('Data was successfully loaded!
' )
directory = '.. /input/'
trainfiles = ['train_date.csv',
'train_numeric.csv']
testfiles = ['test_date.csv',
'test_numeric.csv']<define_variables>
def id_split(dataframe):
dataframe['device_name'] = dataframe['DeviceInfo'].str.split('/', expand=True)[0]
dataframe['device_version'] = dataframe['DeviceInfo'].str.split('/', expand=True)[1]
dataframe['OS_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[0]
dataframe['version_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[1]
dataframe['browser_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[0]
dataframe['version_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[1]
dataframe['screen_width'] = dataframe['id_33'].str.split('x', expand=True)[0]
dataframe['screen_height'] = dataframe['id_33'].str.split('x', expand=True)[1]
dataframe['id_34'] = dataframe['id_34'].str.split(':', expand=True)[1]
dataframe['id_23'] = dataframe['id_23'].str.split(':', expand=True)[1]
dataframe.loc[dataframe['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
dataframe.loc[dataframe['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
dataframe.loc[dataframe['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
dataframe.loc[dataframe['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
dataframe.loc[dataframe['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
dataframe.loc[dataframe['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
dataframe.loc[dataframe['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
dataframe.loc[dataframe['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
dataframe.loc[dataframe.device_name.isin(dataframe.device_name.value_counts() [dataframe.device_name.value_counts() < 200].index), 'device_name'] = "Others"
dataframe['had_id'] = 1
gc.collect()
return dataframe
num_feats = ['Id',
'L3_S33_F3855',
'L3_S32_F3850',
'L3_S33_F3865',
'L1_S24_F1581',
'L3_S38_F3952',
'L1_S24_F1672',
'L1_S24_F1632',
'L1_S24_F1846',
'L1_S24_F1844',
'L1_S24_F1609',
'L1_S24_F1667',
'L1_S24_F1842',
'L0_S13_F356',
'L3_S29_F3342',
'L3_S29_F3407',
'L3_S34_F3876',
'L0_S11_F302',
'L3_S29_F3461',
'L3_S30_F3494',
'L0_S3_F100',
'L0_S1_F28',
'L0_S6_F122',
'L0_S0_F0',
'L0_S0_F20',
'L3_S30_F3704',
'Response']<define_variables>
train_identity = id_split(train_identity)
 test_identity = id_split(test_identity )
cols = [['Id']+date_cols,num_feats]<define_variables>
print('Merging data...')
 train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)
 print('Data was successfully merged!
')
 del train_identity, train_transaction, test_identity, test_transaction
print(f'Train dataset has {train.shape[0]} rows and {train.shape[1]} columns.')
 print(f'Test dataset has {test.shape[0]} rows and {test.shape[1]} columns.')
 gc.collect()
traindata = None
testdata = None<load_from_csv>
useful_features = ['TransactionAmt', 'ProductCD', 'card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'addr1', 'addr2', 'dist1',
'P_emaildomain', 'R_emaildomain', 'C1', 'C2', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11', 'C12', 'C13',
'C14', 'D1', 'D2', 'D3', 'D4', 'D5', 'D6', 'D8', 'D9', 'D10', 'D11', 'D12', 'D13', 'D14', 'D15', 'M2', 'M3',
'M4', 'M5', 'M6', 'M7', 'M8', 'M9', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V17',
'V19', 'V20', 'V29', 'V30', 'V33', 'V34', 'V35', 'V36', 'V37', 'V38', 'V40', 'V44', 'V45', 'V46', 'V47', 'V48',
'V49', 'V51', 'V52', 'V53', 'V54', 'V56', 'V58', 'V59', 'V60', 'V61', 'V62', 'V63', 'V64', 'V69', 'V70', 'V71',
'V72', 'V73', 'V74', 'V75', 'V76', 'V78', 'V80', 'V81', 'V82', 'V83', 'V84', 'V85', 'V87', 'V90', 'V91', 'V92',
'V93', 'V94', 'V95', 'V96', 'V97', 'V99', 'V100', 'V126', 'V127', 'V128', 'V130', 'V131', 'V138', 'V139', 'V140',
'V143', 'V145', 'V146', 'V147', 'V149', 'V150', 'V151', 'V152', 'V154', 'V156', 'V158', 'V159', 'V160', 'V161',
'V162', 'V163', 'V164', 'V165', 'V166', 'V167', 'V169', 'V170', 'V171', 'V172', 'V173', 'V175', 'V176', 'V177',
'V178', 'V180', 'V182', 'V184', 'V187', 'V188', 'V189', 'V195', 'V197', 'V200', 'V201', 'V202', 'V203', 'V204',
'V205', 'V206', 'V207', 'V208', 'V209', 'V210', 'V212', 'V213', 'V214', 'V215', 'V216', 'V217', 'V219', 'V220',
'V221', 'V222', 'V223', 'V224', 'V225', 'V226', 'V227', 'V228', 'V229', 'V231', 'V233', 'V234', 'V238', 'V239',
'V242', 'V243', 'V244', 'V245', 'V246', 'V247', 'V249', 'V251', 'V253', 'V256', 'V257', 'V258', 'V259', 'V261',
'V262', 'V263', 'V264', 'V265', 'V266', 'V267', 'V268', 'V270', 'V271', 'V272', 'V273', 'V274', 'V275', 'V276',
'V277', 'V278', 'V279', 'V280', 'V282', 'V283', 'V285', 'V287', 'V288', 'V289', 'V291', 'V292', 'V294', 'V303',
'V304', 'V306', 'V307', 'V308', 'V310', 'V312', 'V313', 'V314', 'V315', 'V317', 'V322', 'V323', 'V324', 'V326',
'V329', 'V331', 'V332', 'V333', 'V335', 'V336', 'V338', 'id_01', 'id_02', 'id_03', 'id_05', 'id_06', 'id_09',
'id_11', 'id_12', 'id_13', 'id_14', 'id_15', 'id_17', 'id_19', 'id_20', 'id_30', 'id_31', 'id_32', 'id_33',
'id_36', 'id_37', 'id_38', 'DeviceType', 'DeviceInfo', 'device_name', 'device_version', 'OS_id_30', 'version_id_30',
'browser_id_31', 'version_id_31', 'screen_width', 'screen_height', 'had_id']
for i, f in enumerate(trainfiles):
print(f)
 subset = None
for i, chunk in enumerate(pd.read_csv(directory + f,
usecols=cols[i],
chunksize=50000,
low_memory=False)) :
if subset is None:
subset = chunk.copy()
else:
subset = pd.concat([subset, chunk])
 del chunk
gc.collect()
if traindata is None:
traindata = subset.copy()
else:
traindata = pd.merge(traindata, subset.copy() , on="Id")
 del subset
gc.collect()<drop_column>
cols_to_drop = [col for col in train.columns if col not in useful_features]
cols_to_drop.remove('isFraud')
 cols_to_drop.remove('TransactionDT' )
del cols[1][-1]<load_from_csv>
train = train.drop(cols_to_drop, axis=1)
 test = test.drop(cols_to_drop, axis=1 )
for i, f in enumerate(testfiles):
print(f)
 subset = None
for i, chunk in enumerate(pd.read_csv(directory + f,
usecols=cols[i],
chunksize=50000,
low_memory=False)) :
if subset is None:
subset = chunk.copy()
else:
subset = pd.concat([subset, chunk])
 del chunk
gc.collect()
if testdata is None:
testdata = subset.copy()
else:
testdata = pd.merge(testdata, subset.copy() , on="Id")
 del subset
gc.collect()<set_options>
columns_a = ['TransactionAmt', 'id_02', 'D15']
columns_b = ['card1', 'card4', 'addr1']
for col_a in columns_a:
for col_b in columns_b:
for df in [train, test]:
df[f'{col_a}_to_mean_{col_b}'] = df[col_a] / df.groupby([col_b])[col_a].transform('mean')
 df[f'{col_a}_to_std_{col_b}'] = df[col_a] / df.groupby([col_b])[col_a].transform('std' )
del midr
gc.collect()<merge>
train['TransactionAmt_Log'] = np.log(train['TransactionAmt'])
 test['TransactionAmt_Log'] = np.log(test['TransactionAmt'])
 train['TransactionAmt_decimal'] =(( train['TransactionAmt'] - train['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 test['TransactionAmt_decimal'] =(( test['TransactionAmt'] - test['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 train['Transaction_day_of_week'] = np.floor(( train['TransactionDT'] /(3600 * 24)- 1)% 7)
 test['Transaction_day_of_week'] = np.floor(( test['TransactionDT'] /(3600 * 24)- 1)% 7)
 train['Transaction_hour'] = np.floor(train['TransactionDT'] / 3600)% 24
test['Transaction_hour'] = np.floor(test['TransactionDT'] / 3600)% 24
for feature in ['id_02__id_20', 'id_02__D8', 'D11__DeviceInfo', 'DeviceInfo__P_emaildomain', 'P_emaildomain__C2',
'card2__dist1', 'card1__card5', 'card2__id_20', 'card5__P_emaildomain', 'addr1__card1']:
f1, f2 = feature.split('__')
 train[feature] = train[f1].astype(str)+ '_' + train[f2].astype(str)
 test[feature] = test[f1].astype(str)+ '_' + test[f2].astype(str)
 le = LabelEncoder()
le.fit(list(train[feature].astype(str ).values)+ list(test[feature].astype(str ).values))
train[feature] = le.transform(list(train[feature].astype(str ).values))
test[feature] = le.transform(list(test[feature].astype(str ).values))
for feature in ['card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'id_36']:
train[feature + '_count_full'] = train[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
test[feature + '_count_full'] = test[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
for feature in ['id_01', 'id_31', 'id_33', 'id_36']:
train[feature + '_count_dist'] = train[feature].map(train[feature].value_counts(dropna=False))
test[feature + '_count_dist'] = test[feature].map(test[feature].value_counts(dropna=False))
traindata = traindata.merge(new_features, on='Id')
 traindata = traindata.merge(station_ohe, on='Id')
 testdata = testdata.merge(new_features, on='Id')
 testdata = testdata.merge(station_ohe, on='Id' )<set_options>
for c in ['P_emaildomain', 'R_emaildomain']:
train[c + '_bin'] = train[c].map(emails)
 test[c + '_bin'] = test[c].map(emails)
 train[c + '_suffix'] = train[c].map(lambda x: str(x ).split('.')[-1])
 test[c + '_suffix'] = test[c].map(lambda x: str(x ).split('.')[-1])
 train[c + '_suffix'] = train[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 test[c + '_suffix'] = test[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us' )
del new_features
del station_ohe
gc.collect()<feature_engineering>
%%time
for col in train.columns:
if train[col].dtype == 'object':
le = LabelEncoder()
le.fit(list(train[col].astype(str ).values)+ list(test[col].astype(str ).values))
train[col] = le.transform(list(train[col].astype(str ).values))
test[col] = le.transform(list(test[col].astype(str ).values))
testdata['Response'] = 0<split>
%%time
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test )
visibletraindata = traindata[::2]<define_variables>
X = train.sort_values('TransactionDT' ).drop(['isFraud', 'TransactionDT'], axis=1)
 y = train.sort_values('TransactionDT')['isFraud']
X_test = test.drop(['TransactionDT'], axis=1)
 del train, test
gc.collect()
blindtraindata = traindata[1::2]<set_options>
params = {'num_leaves': 524,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.3797454081646243,
'bagging_fraction': 0.4181193142567742,
'min_data_in_leaf': 124,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.006883242363721497,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
'random_state': 47,
}
del traindata
gc.collect()<merge>
%%time
NFOLDS = 5
folds = KFold(n_splits=NFOLDS)
 columns = X.columns
splits = folds.split(X, y)
 y_preds = np.zeros(X_test.shape[0])
 y_oof = np.zeros(X.shape[0])
 score = 0
feature_importances = pd.DataFrame()
feature_importances['feature'] = columns
for fold_n,(train_index, valid_index)in enumerate(splits):
X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
dtrain = lgb.Dataset(X_train, label=y_train)
 dvalid = lgb.Dataset(X_valid, label=y_valid)
 clf = lgb.train(params, dtrain, 10000, valid_sets = [dtrain, dvalid], verbose_eval=200, early_stopping_rounds=500)
 feature_importances[f'fold_{fold_n + 1}'] = clf.feature_importance()
y_pred_valid = clf.predict(X_valid)
 y_oof[valid_index] = y_pred_valid
print(f"Fold {fold_n + 1} | AUC: {roc_auc_score(y_valid, y_pred_valid)}")
 score += roc_auc_score(y_valid, y_pred_valid)/ NFOLDS
y_preds += clf.predict(X_test)/ NFOLDS
del X_train, X_valid, y_train, y_valid
gc.collect()
print(f"
Mean AUC = {score}")
 print(f"Out of folds AUC = {roc_auc_score(y, y_oof)}" )
def LeaveOneOut(data1, data2, columnName, useLOO=False):
grpOutcomes = data1.groupby(columnName)['Response'].mean().reset_index()
grpCount = data1.groupby(columnName)['Response'].count().reset_index()
grpOutcomes['cnt'] = grpCount.Response
if(useLOO):
grpOutcomes = grpOutcomes[grpOutcomes.cnt > 1]
grpOutcomes.drop('cnt', inplace=True, axis=1)
 outcomes = data2['Response'].values
x = pd.merge(data2[[columnName, 'Response']], grpOutcomes,
suffixes=('x_', ''),
how='left',
on=columnName,
left_index=True)['Response']
if(useLOO):
x =(( x*x.shape[0])-outcomes)/(x.shape[0]-1)
 return x.fillna(x.mean() )<drop_column>
sub['isFraud'] = y_preds
sub.to_csv("submission.csv", index=False )
<init_hyperparams><EOS> 
feature_importances['average'] = feature_importances[[f'fold_{fold_n + 1}' for fold_n in range(folds.n_splits)]].mean(axis=1)
 feature_importances.to_csv('feature_importances.csv')
 plt.figure(figsize=(16, 16))
sns.barplot(data=feature_importances.sort_values(by='average', ascending=False ).head(50), x='average', y='feature');
plt.title('Top 50 feature importance over {} folds average'.format(folds.n_splits)) ;