Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<split>
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import gc
x_train, x_test, y_train, y_test = train_test_split(X, y, test_size =
0.25, random_state = 0 )<import_modules>
train_identity = pd.read_csv('.. /input/train_identity.csv')
 train_transaction = pd.read_csv('.. /input/train_transaction.csv' )
from sklearn import preprocessing<normalization>
train_transaction['isFraud'].value_counts()
scaler = preprocessing.StandardScaler().fit(x_train )<normalization>
train = train_transaction.merge(train_identity, on='TransactionID', how='left')
 del train_identity, train_transaction
gc.collect()
X_scaled = scaler.transform(x_train )<normalization>
test_identity = pd.read_csv('.. /input/test_identity.csv')
 test_transaction = pd.read_csv('.. /input/test_transaction.csv')
 test = test_transaction.merge(test_identity, on='TransactionID', how='left')
 del test_identity, test_transaction
gc.collect()
scaler = preprocessing.StandardScaler().fit(x_test )<normalization>
train['TransactionDecimal'] =(( train['TransactionAmt'] - train['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 test['TransactionDecimal'] =(( test['TransactionAmt'] - test['TransactionAmt'].astype(int)) * 1000 ).astype(int )
X_testscaled = scaler.transform(x_test )<train_on_grid>
train['id_02_bins'] = train['id_02']
train['id_02_bins'] = pd.cut(train['id_02_bins'], 10, labels=False)
 test['id_02_bins'] = test['id_02']
test['id_02_bins'] = pd.cut(test['id_02_bins'], 10, labels=False )
params_to_test = {
'n_estimators':[50,100,150,170,180,190,200,210],
'max_depth':[3,5,6]
}
rf_model = RandomForestClassifier(random_state=42)
 grid_search = GridSearchCV(rf_model, param_grid=params_to_test, cv=10, scoring='f1_macro', n_jobs=4)
 grid_search.fit(X_scaled, y_train)
 best_params = grid_search.best_params_
best_model = RandomForestClassifier(**best_params )<train_model>
train['id_11_flag'] = np.where(train['id_11'].isnull() ,'F','T')
 test['id_11_flag'] = np.where(test['id_11'].isnull() ,'F','T')
 train['id_11_flag'] = train['id_11_flag'].astype('category')
 test['id_11_flag'] = test['id_11_flag'].astype('category')
 train['id_11_residual'] = train['id_11']
train['id_11_residual'] = np.where(train['id_11'] == 100.0,'100',np.where(train['id_11_residual'].isnull() ,'None','Residual'))
train['id_11_residual'] = train['id_11_residual'].astype('category')
 test['id_11_residual'] = test['id_11']
test['id_11_residual'] = np.where(test['id_11'] == 100.0,'100',np.where(test['id_11_residual'].isnull() ,'None','Residual'))
test['id_11_residual'] = test['id_11_residual'].astype('category' )
best_model.fit(X_scaled,y_train )<predict_on_test>
train['Transaction_DOW'] = np.floor(( train['TransactionDT'] /(3600 * 24)- 1)% 7)
 test['Transaction_DOW'] = np.floor(( test['TransactionDT'] /(3600 * 24)- 1)% 7)
 train['Transaction_H'] = np.floor(train['TransactionDT'] / 3600)% 24
test['Transaction_H'] = np.floor(test['TransactionDT'] / 3600)% 24
betterpred = best_model.predict(X_testscaled )<import_modules>
warnings.filterwarnings("ignore" )
from sklearn.metrics import accuracy_score<compute_test_metric>
cat_types = ['ProductCD','card1','card2','card3','card4','card5','card6','addr1','addr2','P_emaildomain','R_emaildomain',
'M1','M2','M3','M4','M5','M6','M7','M8','M9','DeviceType','DeviceInfo','id_12','id_13','id_14','id_15','id_16',
'id_17','id_18','id_19','id_20','id_21','id_22','id_23','id_24','id_25','id_26','id_27','id_28','id_29','id_30',
'id_31','id_32','id_33','id_34','id_35','id_36','id_37','id_38','id_02_bins','id_11_flag','id_11_residual']
train[cat_types] = train[cat_types].astype('object')
 test[cat_types] = test[cat_types].astype('object' )
predacc = round(accuracy_score(betterpred, y_test)* 100, 2)
 print(predacc,'%' )<normalization>
train = train.fillna(-999 )
scale1 = preprocessing.StandardScaler().fit(X)
 X_scaled1 = scale1.transform(X )<drop_column>
for f in train.columns:
if train[f].dtype=='object':
lbl = LabelEncoder()
lbl.fit(list(train[f].values)+ list(test[f].values))
train[f] = lbl.transform(list(train[f].values))
test[f] = lbl.transform(list(test[f].values))
train = train.reset_index()
test = test.reset_index()
X_test = test.drop(['Survived','PassengerId'],axis=1 )<normalization>
features = list(train)
 features.remove('isFraud')
 target = 'isFraud'
predscale1 = preprocessing.StandardScaler().fit(X_test)
 pred_scale = predscale1.transform(X_test )<predict_on_test>
bayesian_tr_idx, bayesian_val_idx = train_test_split(train, test_size = 0.3, random_state = 42, stratify = train[target])
 bayesian_tr_idx = bayesian_tr_idx.index
bayesian_val_idx = bayesian_val_idx.index
testpred = best_model.predict(pred_scale )<data_type_conversions>
def LGB_bayesian(
 num_leaves,
bagging_fraction,
feature_fraction,
min_child_weight,
min_data_in_leaf,
max_depth,
reg_alpha,
reg_lambda 
):
num_leaves = int(num_leaves)
 min_data_in_leaf = int(min_data_in_leaf)
 max_depth = int(max_depth)
 assert type(num_leaves)== int
assert type(min_data_in_leaf)== int
assert type(max_depth)== int
param = {
'num_leaves': num_leaves,
'min_data_in_leaf': min_data_in_leaf,
'min_child_weight': min_child_weight,
'bagging_fraction' : bagging_fraction,
'feature_fraction' : feature_fraction,
'max_depth': max_depth,
'reg_alpha': reg_alpha,
'reg_lambda': reg_lambda,
'objective': 'binary',
'save_binary': True,
'seed': 11,
'feature_fraction_seed': 11,
'bagging_seed': 11,
'drop_seed': 11,
'data_random_seed': 11,
'boosting_type': 'gbdt',
'verbose': 1,
'is_unbalance': False,
'boost_from_average': True,
'metric':'auc'}
oof = np.zeros(len(train))
trn_data= lgb.Dataset(train.iloc[bayesian_tr_idx][features].values, label=train.iloc[bayesian_tr_idx][target].values)
 val_data= lgb.Dataset(train.iloc[bayesian_val_idx][features].values, label=train.iloc[bayesian_val_idx][target].values)
 clf = lgb.train(param, trn_data, num_boost_round=50, valid_sets = [trn_data, val_data], verbose_eval=0, early_stopping_rounds = 50)
 oof[bayesian_val_idx] = clf.predict(train.iloc[bayesian_val_idx][features].values, num_iteration=clf.best_iteration)
 score = roc_auc_score(train.iloc[bayesian_val_idx][target].values, oof[bayesian_val_idx])
 return score
output = pd.DataFrame({'PassengerId': test.PassengerId, 'Survived': testpred})
 output.info()
output['Survived'] = output['Survived'].astype('int64')
 output.head()<count_values>
bounds_LGB = {
'num_leaves':(31, 500),
'min_data_in_leaf':(20, 200),
'bagging_fraction' :(0.1, 0.9),
'feature_fraction' :(0.1, 0.9),
'min_child_weight':(0.00001, 0.01),
'reg_alpha':(1, 2),
'reg_lambda':(1, 2),
'max_depth':(-1,50),
}
output['Survived'].value_counts()<import_modules>
LGB_BO = BayesianOptimization(LGB_bayesian, bounds_LGB, random_state=42 )
from sklearn.svm import SVC<choose_model_class>
init_points = 10
n_iter = 15
model = SVC()<train_model>
with warnings.catch_warnings() :
warnings.filterwarnings('ignore')
 LGB_BO.maximize(init_points=init_points, n_iter=n_iter, acq='ucb', xi=0.0, alpha=1e-6 )
model.fit(X_scaled,y_train )<predict_on_test>
train_identity = pd.read_csv('.. /input/train_identity.csv')
 train_transaction = pd.read_csv('.. /input/train_transaction.csv')
 train = train_transaction.merge(train_identity, on='TransactionID', how='left')
 del train_identity, train_transaction
gc.collect()
test_identity = pd.read_csv('.. /input/test_identity.csv')
 test_transaction = pd.read_csv('.. /input/test_transaction.csv')
 test = test_transaction.merge(test_identity, on='TransactionID', how='left')
 del test_identity, test_transaction
gc.collect()
train['TransactionDecimal'] =(( train['TransactionAmt'] - train['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 test['TransactionDecimal'] =(( test['TransactionAmt'] - test['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 train['id_02_bins'] = train['id_02']
train['id_02_bins'] = pd.cut(train['id_02_bins'], 10, labels=False)
 test['id_02_bins'] = test['id_02']
test['id_02_bins'] = pd.cut(test['id_02_bins'], 10, labels=False)
 train['id_11_flag'] = np.where(train['id_11'].isnull() ,'F','T')
 test['id_11_flag'] = np.where(test['id_11'].isnull() ,'F','T')
 train['id_11_flag'] = train['id_11_flag'].astype('category')
 test['id_11_flag'] = test['id_11_flag'].astype('category')
 train['id_11_residual'] = train['id_11']
train['id_11_residual'] = np.where(train['id_11'] == 100.0,'100',np.where(train['id_11_residual'].isnull() ,'None','Residual'))
train['id_11_residual'] = train['id_11_residual'].astype('category')
 test['id_11_residual'] = test['id_11']
test['id_11_residual'] = np.where(test['id_11'] == 100.0,'100',np.where(test['id_11_residual'].isnull() ,'None','Residual'))
test['id_11_residual'] = test['id_11_residual'].astype('category')
 train['Transaction_DOW'] = np.floor(( train['TransactionDT'] /(3600 * 24)- 1)% 7)
 test['Transaction_DOW'] = np.floor(( test['TransactionDT'] /(3600 * 24)- 1)% 7)
 train['Transaction_H'] = np.floor(train['TransactionDT'] / 3600)% 24
test['Transaction_H'] = np.floor(test['TransactionDT'] / 3600)% 24
predictions = model.predict(X_testscaled )<compute_test_metric>
cat_types = ['ProductCD','card1','card2','card3','card4','card5','card6','addr1','addr2','P_emaildomain','R_emaildomain',
'M1','M2','M3','M4','M5','M6','M7','M8','M9','DeviceType','DeviceInfo','id_12','id_13','id_14','id_15','id_16',
'id_17','id_18','id_19','id_20','id_21','id_22','id_23','id_24','id_25','id_26','id_27','id_28','id_29','id_30',
'id_31','id_32','id_33','id_34','id_35','id_36','id_37','id_38']
train[cat_types] = train[cat_types].astype('category')
 test[cat_types] = test[cat_types].astype('category' )
svcacc = round(accuracy_score(predictions, y_test)* 100, 2)
 print(svcacc,'%' )<predict_on_test>
train_ID = train['TransactionID']
test_ID = test['TransactionID']
y = train['isFraud']
X = train.drop(['TransactionID','TransactionDT','isFraud'], axis=1)
 X_test = test.drop(['TransactionID','TransactionDT'], axis=1)
 del train, test
gc.collect()
testpredictions = model.predict(pred_scale )<prepare_output>
params = {'num_leaves': int(LGB_BO.max['params']['num_leaves']),
'min_child_weight': LGB_BO.max['params']['min_child_weight'],
'feature_fraction': LGB_BO.max['params']['feature_fraction'],
'bagging_fraction': LGB_BO.max['params']['bagging_fraction'],
'min_data_in_leaf': int(LGB_BO.max['params']['min_data_in_leaf']),
'objective': 'binary',
'max_depth': int(LGB_BO.max['params']['max_depth']),
'learning_rate': 0.006883242363721497,
"boosting_type": "gbdt",
"bagging_seed": 11,
'seed': 11,
'drop_seed': 11,
'feature_fraction_seed': 11,
'data_random_seed': 11,
'metric': 'auc',
'verbosity': 1,
'reg_alpha': LGB_BO.max['params']['reg_alpha'],
'reg_lambda': LGB_BO.max['params']['reg_lambda'],
'random_state': 47
}
output2 = pd.DataFrame({'PassengerId': test.PassengerId, 'Survived': testpredictions})
 output2.info()
output2['Survived'] = output['Survived'].astype('int64')
 output2.head()<count_values>
NFOLDS = 5
folds = KFold(n_splits=NFOLDS)
 columns = X.columns
splits = folds.split(X, y)
 y_preds = np.zeros(X_test.shape[0])
 y_oof = np.zeros(X.shape[0])
 score = 0
for fold_n,(train_index, valid_index)in enumerate(splits):
X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
dtrain = lgb.Dataset(X_train, label=y_train)
 dvalid = lgb.Dataset(X_valid, label=y_valid)
 clf = lgb.train(params, dtrain, 10000, valid_sets = [dtrain, dvalid], verbose_eval=200, early_stopping_rounds=500)
 y_pred_valid = clf.predict(X_valid)
 y_oof[valid_index] = y_pred_valid
print(f"Fold {fold_n + 1} | AUC: {roc_auc_score(y_valid, y_pred_valid)}")
 score += roc_auc_score(y_valid, y_pred_valid)/ NFOLDS
y_preds += clf.predict(X_test)/ NFOLDS
del X_train, X_valid, y_train, y_valid
gc.collect()
print(f"
Mean AUC = {score}")
 print(f"Out of folds AUC = {roc_auc_score(y, y_oof)}" )
<count_values><EOS> 
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['isFraud'] = y_preds
sub.to_csv('submission.csv', index=False )