Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<choose_model_class>
BUILD95 = True
BUILD96 = True
str_type = ['ProductCD', 'card4', 'card6', 'P_emaildomain', 'R_emaildomain','M1', 'M2', 'M3', 'M4','M5',
'M6', 'M7', 'M8', 'M9', 'id_12', 'id_15', 'id_16', 'id_23', 'id_27', 'id_28', 'id_29', 'id_30',
'id_31', 'id_33', 'id_34', 'id_35', 'id_36', 'id_37', 'id_38', 'DeviceType', 'DeviceInfo']
str_type += ['id-12', 'id-15', 'id-16', 'id-23', 'id-27', 'id-28', 'id-29', 'id-30',
'id-31', 'id-33', 'id-34', 'id-35', 'id-36', 'id-37', 'id-38']
cols = ['TransactionID', 'TransactionDT', 'TransactionAmt',
'ProductCD', 'card1', 'card2', 'card3', 'card4', 'card5', 'card6',
'addr1', 'addr2', 'dist1', 'dist2', 'P_emaildomain', 'R_emaildomain',
'C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11',
'C12', 'C13', 'C14', 'D1', 'D2', 'D3', 'D4', 'D5', 'D6', 'D7', 'D8',
'D9', 'D10', 'D11', 'D12', 'D13', 'D14', 'D15', 'M1', 'M2', 'M3', 'M4',
'M5', 'M6', 'M7', 'M8', 'M9']
v = [1, 3, 4, 6, 8, 11]
v += [13, 14, 17, 20, 23, 26, 27, 30]
v += [36, 37, 40, 41, 44, 47, 48]
v += [54, 56, 59, 62, 65, 67, 68, 70]
v += [76, 78, 80, 82, 86, 88, 89, 91]
v += [107, 108, 111, 115, 117, 120, 121, 123]
v += [124, 127, 129, 130, 136]
v += [138, 139, 142, 147, 156, 162]
v += [165, 160, 166]
v += [178, 176, 173, 182]
v += [187, 203, 205, 207, 215]
v += [169, 171, 175, 180, 185, 188, 198, 210, 209]
v += [218, 223, 224, 226, 228, 229, 235]
v += [240, 258, 257, 253, 252, 260, 261]
v += [264, 266, 267, 274, 277]
v += [220, 221, 234, 238, 250, 271]
v += [294, 284, 285, 286, 291, 297]
v += [303, 305, 307, 309, 310, 320]
v += [281, 283, 289, 296, 301, 314]
cols += ['V'+str(x)for x in v]
dtypes = {}
for c in cols+['id_0'+str(x)for x in range(1,10)]+['id_'+str(x)for x in range(10,34)]+\
['id-0'+str(x)for x in range(1,10)]+['id-'+str(x)for x in range(10,34)]:
dtypes[c] = 'float32'
for c in str_type: dtypes[c] = 'category'
model = LogisticRegression(random_state=10, max_iter = 1000)
 logit_params = {
"C": [1, 3, 10, 20, 30, 40],
"solver": ["lbfgs", "liblinear"]
}
logit_gs = GridSearchCV(model, logit_params, scoring="f1", cv = 5, n_jobs=4)
<train_model>
%%time
X_train = pd.read_csv('.. /input/ieee-fraud-detection/train_transaction.csv',index_col='TransactionID', dtype=dtypes, usecols=cols+['isFraud'])
 train_id = pd.read_csv('.. /input/ieee-fraud-detection/train_identity.csv',index_col='TransactionID', dtype=dtypes)
 X_train = X_train.merge(train_id, how='left', left_index=True, right_index=True)
 X_test = pd.read_csv('.. /input/ieee-fraud-detection/test_transaction.csv',index_col='TransactionID', dtype=dtypes, usecols=cols)
 test_id = pd.read_csv('.. /input/ieee-fraud-detection/test_identity.csv',index_col='TransactionID', dtype=dtypes)
 fix = {o:n for o, n in zip(test_id.columns, train_id.columns)}
test_id.rename(columns=fix, inplace=True)
 X_test = X_test.merge(test_id, how='left', left_index=True, right_index=True)
 y_train = X_train['isFraud'].copy()
del train_id, test_id, X_train['isFraud']; x = gc.collect()
print('Train shape',X_train.shape,'test shape',X_test.shape )
logit_gs.fit(train_data, train_y )<find_best_score>
for i in range(1,16):
if i in [1,2,3,5,9]: continue
X_train['D'+str(i)] = X_train['D'+str(i)] - X_train.TransactionDT/np.float32(24*60*60)
 X_test['D'+str(i)] = X_test['D'+str(i)] - X_test.TransactionDT/np.float32(24*60*60 )
print(logit_gs.best_params_)
 print(logit_gs.best_score_ )<choose_model_class>
%%time
for i,f in enumerate(X_train.columns):
if(np.str(X_train[f].dtype)=='category')|(X_train[f].dtype=='object'):
df_comb = pd.concat([X_train[f],X_test[f]],axis=0)
 df_comb,_ = df_comb.factorize(sort=True)
 if df_comb.max() >32000: print(f,'needs int32')
 X_train[f] = df_comb[:len(X_train)].astype('int16')
 X_test[f] = df_comb[len(X_train):].astype('int16')
 elif f not in ['TransactionAmt','TransactionDT']:
mn = np.min(( X_train[f].min() ,X_test[f].min()))
X_train[f] -= np.float32(mn)
 X_test[f] -= np.float32(mn)
 X_train[f].fillna(-1,inplace=True)
 X_test[f].fillna(-1,inplace=True )
svc_model = SVC()
test_parameters = {
"C": [1, 3, 10, 30, 100],
"kernel": ["linear", "poly", "rbf" , "sigmoid"],
}
svc_gs = GridSearchCV(svc_model, test_parameters, scoring="f1", cv=5, n_jobs=4 )<train_model>
def encode_FE(df1, df2, cols):
for col in cols:
df = pd.concat([df1[col],df2[col]])
 vc = df.value_counts(dropna=True, normalize=True ).to_dict()
vc[-1] = -1
nm = col+'_FE'
df1[nm] = df1[col].map(vc)
 df1[nm] = df1[nm].astype('float32')
 df2[nm] = df2[col].map(vc)
 df2[nm] = df2[nm].astype('float32')
 print(nm,', ',end='')
 def encode_LE(col,train=X_train,test=X_test,verbose=True):
df_comb = pd.concat([train[col],test[col]],axis=0)
 df_comb,_ = df_comb.factorize(sort=True)
 nm = col
if df_comb.max() >32000:
train[nm] = df_comb[:len(train)].astype('int32')
 test[nm] = df_comb[len(train):].astype('int32')
 else:
train[nm] = df_comb[:len(train)].astype('int16')
 test[nm] = df_comb[len(train):].astype('int16')
 del df_comb; x=gc.collect()
if verbose: print(nm,', ',end='')
 def encode_AG(main_columns, uids, aggregations=['mean'], train_df=X_train, test_df=X_test,
fillna=True, usena=False):
for main_column in main_columns:
for col in uids:
for agg_type in aggregations:
new_col_name = main_column+'_'+col+'_'+agg_type
temp_df = pd.concat([train_df[[col, main_column]], test_df[[col,main_column]]])
 if usena: temp_df.loc[temp_df[main_column]==-1,main_column] = np.nan
temp_df = temp_df.groupby([col])[main_column].agg([agg_type] ).reset_index().rename(
 columns={agg_type: new_col_name})
 temp_df.index = list(temp_df[col])
 temp_df = temp_df[new_col_name].to_dict()
train_df[new_col_name] = train_df[col].map(temp_df ).astype('float32')
 test_df[new_col_name] = test_df[col].map(temp_df ).astype('float32')
 if fillna:
train_df[new_col_name].fillna(-1,inplace=True)
 test_df[new_col_name].fillna(-1,inplace=True)
 print("'"+new_col_name+"'",', ',end='')
 def encode_CB(col1,col2,df1=X_train,df2=X_test):
nm = col1+'_'+col2
df1[nm] = df1[col1].astype(str)+'_'+df1[col2].astype(str)
 df2[nm] = df2[col1].astype(str)+'_'+df2[col2].astype(str)
 encode_LE(nm,verbose=False)
 print(nm,', ',end='')
 def encode_AG2(main_columns, uids, train_df=X_train, test_df=X_test):
for main_column in main_columns:
for col in uids:
comb = pd.concat([train_df[[col]+[main_column]],test_df[[col]+[main_column]]],axis=0)
 mp = comb.groupby(col)[main_column].agg(['nunique'])['nunique'].to_dict()
train_df[col+'_'+main_column+'_ct'] = train_df[col].map(mp ).astype('float32')
 test_df[col+'_'+main_column+'_ct'] = test_df[col].map(mp ).astype('float32')
 print(col+'_'+main_column+'_ct, ',end='' )
svc_gs.fit(train_scaled, train_y )<find_best_params>
%%time
X_train['cents'] =(X_train['TransactionAmt'] - np.floor(X_train['TransactionAmt'])).astype('float32')
 X_test['cents'] =(X_test['TransactionAmt'] - np.floor(X_test['TransactionAmt'])).astype('float32')
 print('cents, ', end='')
 encode_FE(X_train,X_test,['addr1','card1','card2','card3','P_emaildomain'])
 encode_CB('card1','addr1')
 encode_CB('card1_addr1','P_emaildomain')
 encode_FE(X_train,X_test,['card1_addr1','card1_addr1_P_emaildomain'])
 encode_AG(['TransactionAmt','D9','D11'],['card1','card1_addr1','card1_addr1_P_emaildomain'],['mean','std'],usena=True )
print(svc_gs.best_params_)
 print(svc_gs.best_score_ )<choose_model_class>
cols = list(X_train.columns)
 cols.remove('TransactionDT')
 for c in ['D6','D7','D8','D9','D12','D13','D14']:
cols.remove(c)
 for c in ['C3','M5','id_08','id_33']:
cols.remove(c)
 for c in ['card4','id_07','id_14','id_21','id_30','id_32','id_34']:
cols.remove(c)
 for c in ['id_'+str(x)for x in range(22,28)]:
cols.remove(c )
lgb_model = LGBMClassifier()
test_parameters = {
"n_estimators": [int(x)for x in np.linspace(5, 30, 6)],
"reg_alpha": [0, 0.75, 1, 1.25],
"learning_rate": [0.5, 0.4, 0.35, 0.3, 0.25, 0.2],
"subsample": [0.5, 0.75, 1]
}
lgb_gs = GridSearchCV(lgb_model, test_parameters, scoring=scoring_method, cv=8, n_jobs=4 )<train_model>
idxT = X_train.index[:3*len(X_train)//4]
idxV = X_train.index[3*len(X_train)//4:]
lgb_gs.fit(train_data, train_y )<find_best_score>
print("XGBoost version:", xgb.__version__)
 if BUILD95:
clf = xgb.XGBClassifier(
 n_estimators=2000,
max_depth=12,
learning_rate=0.02,
subsample=0.8,
colsample_bytree=0.4,
missing=-1,
eval_metric='auc',
tree_method='gpu_hist' 
)
 h = clf.fit(X_train.loc[idxT,cols], y_train[idxT],
eval_set=[(X_train.loc[idxV,cols],y_train[idxV])],
verbose=50, early_stopping_rounds=100 )
print(lgb_gs.best_params_)
 print(lgb_gs.best_score_ )<choose_model_class>
START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d')
 X_train['DT_M'] = X_train['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 X_train['DT_M'] =(X_train['DT_M'].dt.year-2017)*12 + X_train['DT_M'].dt.month
X_test['DT_M'] = X_test['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 X_test['DT_M'] =(X_test['DT_M'].dt.year-2017)*12 + X_test['DT_M'].dt.month
rf_model = RandomForestClassifier()
rf_params ={
'bootstrap': [True, False],
'max_depth': [10, None],
'max_features': ['auto', 'sqrt'],
'min_samples_leaf': [1, 2, 4],
'min_samples_split': [2, 5, 10],
'n_estimators': [5, 10, 15, 20, 25, 30]}
rf_gs = GridSearchCV(rf_model, rf_params, scoring=scoring_method, cv=8, n_jobs=4 )<train_model>
if BUILD95:
oof = np.zeros(len(X_train))
preds = np.zeros(len(X_test))
skf = GroupKFold(n_splits=6)
 for i,(idxT, idxV)in enumerate(skf.split(X_train, y_train, groups=X_train['DT_M'])) :
month = X_train.iloc[idxV]['DT_M'].iloc[0]
print('Fold',i,'withholding month',month)
 print(' rows of train =',len(idxT),'rows of holdout =',len(idxV))
clf = xgb.XGBClassifier(
 n_estimators=5000,
max_depth=12,
learning_rate=0.02,
subsample=0.8,
colsample_bytree=0.4,
missing=-1,
eval_metric='auc',
tree_method='gpu_hist' 
)
 h = clf.fit(X_train[cols].iloc[idxT], y_train.iloc[idxT],
eval_set=[(X_train[cols].iloc[idxV],y_train.iloc[idxV])],
verbose=100, early_stopping_rounds=200)
 oof[idxV] += clf.predict_proba(X_train[cols].iloc[idxV])[:,1]
preds += clf.predict_proba(X_test[cols])[:,1]/skf.n_splits
del h, clf
x=gc.collect()
print('
print('XGB95 OOF CV=',roc_auc_score(y_train,oof))
rf_gs.fit(train_data, train_y )<find_best_score>
if BUILD95:
plt.hist(oof,bins=100)
 plt.ylim(( 0,5000))
plt.title('XGB OOF')
 plt.show()
X_train['oof'] = oof
X_train.reset_index(inplace=True)
 X_train[['TransactionID','oof']].to_csv('oof_xgb_95.csv')
 X_train.set_index('TransactionID',drop=True,inplace=True)
 else: X_train['oof'] = 0
print(rf_gs.best_params_)
 print(rf_gs.best_score_ )<choose_model_class>
if BUILD95:
sample_submission = pd.read_csv('.. /input/ieee-fraud-detection/sample_submission.csv')
 sample_submission.isFraud = preds
sample_submission.to_csv('sub_xgb_95.csv',index=False)
 plt.hist(sample_submission.isFraud,bins=100)
 plt.ylim(( 0,5000))
plt.title('XGB95 Submission')
 plt.show()
ensemble_model = VotingClassifier(estimators=[ 
("logit", logit_gs.best_estimator_), 
("rf", rf_gs.best_estimator_), 
("svc", svc_gs.best_estimator_), 
("lgb", lgb_gs.best_estimator_),
], voting = "hard" )<train_model>
X_train['day'] = X_train.TransactionDT /(24*60*60)
 X_train['uid'] = X_train.card1_addr1.astype(str)+'_'+np.floor(X_train.day-X_train.D1 ).astype(str)
 X_test['day'] = X_test.TransactionDT /(24*60*60)
 X_test['uid'] = X_test.card1_addr1.astype(str)+'_'+np.floor(X_test.day-X_test.D1 ).astype(str )
ensemble_model.fit(train_data, train_y )<compute_test_metric>
%%time
encode_FE(X_train,X_test,['uid'])
 encode_AG(['TransactionAmt','D4','D9','D10','D15'],['uid'],['mean','std'],fillna=True,usena=True)
 encode_AG(['C'+str(x)for x in range(1,15)if x!=3],['uid'],['mean'],X_train,X_test,fillna=True,usena=True)
 encode_AG(['M'+str(x)for x in range(1,10)],['uid'],['mean'],fillna=True,usena=True)
 encode_AG2(['P_emaildomain','dist1','DT_M','id_02','cents'], ['uid'], train_df=X_train, test_df=X_test)
 encode_AG(['C14'],['uid'],['std'],X_train,X_test,fillna=True,usena=True)
 encode_AG2(['C13','V314'], ['uid'], train_df=X_train, test_df=X_test)
 encode_AG2(['V127','V136','V309','V307','V320'], ['uid'], train_df=X_train, test_df=X_test)
 X_train['outsider15'] =(np.abs(X_train.D1-X_train.D15)>3 ).astype('int8')
 X_test['outsider15'] =(np.abs(X_test.D1-X_test.D15)>3 ).astype('int8')
 print('outsider15' )
ensemble_model.score(train_data, train_y )<predict_on_test>
cols = list(X_train.columns)
 cols.remove('TransactionDT')
 for c in ['D6','D7','D8','D9','D12','D13','D14']:
cols.remove(c)
 for c in ['oof','DT_M','day','uid']:
cols.remove(c)
 for c in ['C3','M5','id_08','id_33']:
cols.remove(c)
 for c in ['card4','id_07','id_14','id_21','id_30','id_32','id_34']:
cols.remove(c)
 for c in ['id_'+str(x)for x in range(22,28)]:
cols.remove(c )
preds = ensemble_model.predict(test_data )<save_to_csv>
idxT = X_train.index[:3*len(X_train)//4]
idxV = X_train.index[3*len(X_train)//4:]
output = pd.DataFrame({'PassengerId': test_data.index,
'Survived': preds})
 output.to_csv('submission.csv', index=False )<load_from_csv>
if BUILD96:
clf = xgb.XGBClassifier(
 n_estimators=2000,
max_depth=12,
learning_rate=0.02,
subsample=0.8,
colsample_bytree=0.4,
missing=-1,
eval_metric='auc',
tree_method='gpu_hist' 
)
 h = clf.fit(X_train.loc[idxT,cols], y_train[idxT],
eval_set=[(X_train.loc[idxV,cols],y_train[idxV])],
verbose=50, early_stopping_rounds=100 )
train = pd.read_csv("/kaggle/input/titanic/train.csv")
 test = pd.read_csv("/kaggle/input/titanic/test.csv")
 train['type'] = 'train'
test['type'] = 'test'
alldata = pd.concat([train,test])
 alldata.head()<feature_engineering>
if BUILD96:
oof = np.zeros(len(X_train))
preds = np.zeros(len(X_test))
skf = GroupKFold(n_splits=6)
 for i,(idxT, idxV)in enumerate(skf.split(X_train, y_train, groups=X_train['DT_M'])) :
month = X_train.iloc[idxV]['DT_M'].iloc[0]
print('Fold',i,'withholding month',month)
 print(' rows of train =',len(idxT),'rows of holdout =',len(idxV))
clf = xgb.XGBClassifier(
 n_estimators=5000,
max_depth=12,
learning_rate=0.02,
subsample=0.8,
colsample_bytree=0.4,
missing=-1,
eval_metric='auc',
tree_method='gpu_hist' 
)
 h = clf.fit(X_train[cols].iloc[idxT], y_train.iloc[idxT],
eval_set=[(X_train[cols].iloc[idxV],y_train.iloc[idxV])],
verbose=100, early_stopping_rounds=200)
 oof[idxV] += clf.predict_proba(X_train[cols].iloc[idxV])[:,1]
preds += clf.predict_proba(X_test[cols])[:,1]/skf.n_splits
del h, clf
x=gc.collect()
print('
print('XGB96 OOF CV=',roc_auc_score(y_train,oof))
alldata['titles']=alldata['titles'].str.replace('Don','Mr')
 alldata['titles']=alldata['titles'].str.replace('Rev','Mr')
 alldata['titles']=alldata['titles'].str.replace('Dr','Mr')
 alldata['titles']=alldata['titles'].str.replace('Capt','Mr')
 alldata['titles']=alldata['titles'].str.replace('Jonkheer','Mr')
 alldata['titles']=alldata['titles'].str.replace('Lady','Mrs')
 alldata['titles']=alldata['titles'].str.replace('Mme','Mrs')
 alldata['titles']=alldata['titles'].str.replace('Ms','Mrs')
 alldata['titles']=alldata['titles'].str.replace('Mlle','Mrs')
 alldata['titles']=alldata['titles'].str.replace('the Countess','Mrs')
 alldata['titles']=alldata['titles'].str.replace('Sir','Master')
 alldata['titles']=alldata['titles'].str.replace('Major','Master')
 alldata['titles']=alldata['titles'].str.replace('Col','Master' )<count_values>
if BUILD96:
plt.hist(oof,bins=100)
 plt.ylim(( 0,5000))
plt.title('XGB OOF')
 plt.show()
X_train['oof'] = oof
X_train.reset_index(inplace=True)
 X_train[['TransactionID','oof']].to_csv('oof_xgb_96.csv')
 X_train.set_index('TransactionID',drop=True,inplace=True )
alldata['titles'].value_counts()<drop_column>
if BUILD96:
sample_submission = pd.read_csv('.. /input/ieee-fraud-detection/sample_submission.csv')
 sample_submission.isFraud = preds
sample_submission.to_csv('sub_xgb_96.csv',index=False)
 plt.hist(sample_submission.isFraud,bins=100)
 plt.ylim(( 0,5000))
plt.title('XGB96 Submission')
 plt.show()
alldata.drop('Name',axis=1,inplace=True )<drop_column>
X_test['isFraud'] = sample_submission.isFraud.values
X_train['isFraud'] = y_train.values
comb = pd.concat([X_train[['isFraud']],X_test[['isFraud']]],axis=0)
 uids = pd.read_csv('/kaggle/input/ieee-submissions-and-uids/uids_v4_no_multiuid_cleaning.. csv',usecols=['TransactionID','uid'] ).rename({'uid':'uid2'},axis=1)
 comb = comb.merge(uids,on='TransactionID',how='left')
 mp = comb.groupby('uid2' ).isFraud.agg(['mean'])
 comb.loc[comb.uid2>0,'isFraud'] = comb.loc[comb.uid2>0].uid2.map(mp['mean'])
 uids = pd.read_csv('/kaggle/input/ieee-submissions-and-uids/uids_v1_no_multiuid_cleaning.csv',usecols=['TransactionID','uid'] ).rename({'uid':'uid3'},axis=1)
 comb = comb.merge(uids,on='TransactionID',how='left')
 mp = comb.groupby('uid3' ).isFraud.agg(['mean'])
 comb.loc[comb.uid3>0,'isFraud'] = comb.loc[comb.uid3>0].uid3.map(mp['mean'])
 sample_submission.isFraud = comb.iloc[len(X_train):].isFraud.values
sample_submission.to_csv('sub_xgb_96_PP.csv',index=False )