Kaggle competition: IEEE-CIS Fraud Detection
alldata.drop('Cabin',axis=1,inplace=True )<count_missing_values>
from pandas.api.types import is_string_dtype, is_numeric_dtype, is_categorical_dtype
from fastai.tabular.all import *
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from sklearn.tree import DecisionTreeRegressor, DecisionTreeClassifier
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import train_test_split
import xgboost as xgb
from IPython.display import Image, display_svg, SVG
alldata.isnull().sum()<feature_engineering>
from sklearn.metrics import confusion_matrix, roc_auc_score, plot_confusion_matrix, precision_score, recall_score, classification_report, plot_roc_curve, roc_curve
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold, GroupKFold
alldata['Family_size'] = alldata['SibSp'] + alldata['Parch']<create_dataframe>
import gc
import random as r
import joblib
agegroups = alldata['Age']
agegroups
groups = []
for ages in agegroups:
if ages < 18:
groups.append('Child')
 else:
groups.append('Adult')
 groups = pd.DataFrame(groups,columns=['Child'])
 len(groups)
<categorify>
pd.options.display.max_rows = None
pd.options.display.max_columns = None
alldata['Child'] = groups
alldata.head()
alldata['Child'] = pd.get_dummies(alldata['Child'],drop_first=True)
 alldata.head()<feature_engineering>
main_path = ".. /input/fraud-detection-categorified-and-split/"
alldata['titles'] = pd.Categorical(alldata['titles'])
<feature_engineering>
buildLv = False
buildSkf = True
AV=True
alldata['titles'] = alldata['titles'].cat.codes<drop_column>
%%time
dset_str="all"
to = load_pickle(main_path+"to_"+dset_str+"c.pkl")
 to_tst = load_pickle(main_path+"to_tst_"+dset_str+"c.pkl")
 xs,y = to.train.xs,to.train.y
valid_xs,valid_y = to.valid.xs,to.valid.y
test_xs = to_tst.xs.copy()
xs.shape, test_xs.shape
alldata.drop('titles1',axis=1 )<count_values>
del to, to_tst; x=gc.collect()
alldata.loc[alldata['Sex']=='male','Embarked'].value_counts()<count_values>
def m_rep(m,xs,y): return classification_report(y, m.predict(xs), labels=[1,0], digits=4, output_dict=True)
 def metrics(m, xs, y , valid_xs, valid_y):
tr_rep = m_rep(m,xs,y)
 vd_rep = m_rep(m,valid_xs,valid_y)
 tr_auc = roc_auc_score(y,m.predict_proba(xs)[:,1])
 oob_auc=0.0000
vd_auc = roc_auc_score(valid_y,m.predict_proba(valid_xs)[:,1])
 print('{:.4f} ; {:.4f} ; {:.4f}' 
.format(tr_auc, oob_auc, vd_auc))
alldata.loc[alldata['Sex']=='female','Embarked'].value_counts()<count_values>
def format_test_preds(preds,test_xs,comment):
print(pd.DataFrame(preds ).describe())
 df_pred = pd.DataFrame({"TransactionID":test_xs.TransactionID.to_list() ,
"isFraud": preds})
 df_pred.to_csv(comment+"my_subs.csv",index=False)
 print(df_pred.shape)
 del df_pred; x=gc.collect()
alldata.loc[alldata['Pclass']==1,'Embarked'].value_counts()<count_values>
def xgb_lvd_fun(get_feat_imp):
clf = xgb.XGBClassifier(n_estimators=2000,
max_depth=8,
learning_rate= 0.05,
subsample= 0.6,
colsample_bytree= 0.4,
random_state = r.randint(0,9999),
use_label_encoder=False,
tree_method='gpu_hist')
 m = clf.fit(xs[cols].iloc[idxT], y.iloc[idxT], eval_set=[(xs[cols].iloc[idxV],y.iloc[idxV])],
eval_metric= "auc", verbose=100, early_stopping_rounds=100)
 tr_auc = roc_auc_score(y.iloc[idxT], m.predict_proba(xs[cols].iloc[idxT])[:,1])
 vd_auc = roc_auc_score(y.iloc[idxV],m.predict_proba(xs[cols].iloc[idxV])[:,1])
 te_pred = m.predict_proba(test_xs[cols])[:,1]
if get_feat_imp:
fi = xgb_fi(m, xs[cols])
 fi.to_csv("fi_lv.csv",index=False)
 print(fi)
 del m, clf; x=gc.collect()
return [tr_auc, vd_auc], te_pred
alldata.loc[alldata['Pclass']==2,'Embarked'].value_counts()<count_values>
def xgb_pred_fun() :
skf = KFold(n_splits = 5, shuffle = False)
 tr_pred = np.zeros(len(xs))
oof_pred = np.zeros(len(xs))
te_pred = np.zeros(len(test_xs))
for i,(idxT, idxV)in enumerate(skf.split(xs,y)) :
print('Fold',i)
 print(' n_rows of train =',len(idxT),'rows of holdout =',min(idxV),"to",max(idxV))
clf = xgb.XGBClassifier(n_estimators=1000,
max_depth=12,
learning_rate= 0.02,
subsample= 0.8,
colsample_bytree= 0.4,
random_state = r.randint(0,9999),
use_label_encoder=False,
tree_method='gpu_hist')
 m = clf.fit(xs[cols].iloc[idxT], y.iloc[idxT], eval_set=[(xs[cols].iloc[idxV],y.iloc[idxV])],
eval_metric= "auc", verbose=100, early_stopping_rounds=100)
 tr_pred[idxT] += m.predict_proba(xs[cols].iloc[idxT])[:,1]/(skf.n_splits-1)
 oof_pred[idxV] += m.predict_proba(xs[cols].iloc[idxV])[:,1]
te_pred += m.predict_proba(test_xs[cols])[:,1]/skf.n_splits
if i==4:
fi = xgb_fi(m, xs[cols])
 fi.to_csv("fi_lv.csv",index=False)
 print(fi)
 del m; x=gc.collect()
print('{:.4f} ; {:.4f} ; {:.4f}' 
.format(roc_auc_score(y,tr_pred),roc_auc_score(y,oof_pred),0.000))
return te_pred
alldata.loc[alldata['Pclass']==3,'Embarked'].value_counts()<data_type_conversions>
v = []
v += ['V1', 'V3', 'V11', 'V9', 'V5', 'V7']
v += ['V13', 'V17', 'V24', 'V14', 'V20', 'V27', 'V34', 'V26', 'V30']
v += ['V36', 'V37', 'V47', 'V40', 'V48', 'V52', 'V41', 'V45']
v += ['V54', 'V65', 'V60', 'V67', 'V56', 'V68', 'V62', 'V55', 'V70']
v += ['V76', 'V89', 'V91', 'V81', 'V82', 'V87', 'V78', 'V88']
v += ['V127', 'V121', 'V99', 'V110', 'V104', 'V130', 'V129', 'V131', 'V109', 'V136', 'V116', 'V120', 'V125', 'V113', 'V118', 'V98', 'V107', 'V117', 'V115']
v += ['V138', 'V140', 'V142', 'V147', 'V155', 'V162']
v += ['V165', 'V160', 'V166']
v += ['V203', 'V207', 'V216', 'V187', 'V176', 'V173', 'V183', 'V215']
v += ['V169', 'V195', 'V201', 'V171', 'V174', 'V175', 'V209', 'V185', 'V188', 'V210', 'V198', 'V180']
v += ['V274', 'V264', 'V265', 'V261', 'V235', 'V223', 'V258', 'V260', 'V246', 'V252', 'V241', 'V266', 'V240', 'V277', 'V228', 'V226']
v += ['V220', 'V239', 'V271', 'V221', 'V234', 'V251']
v += ['V307', 'V291', 'V285', 'V290', 'V312', 'V297', 'V305', 'V320', 'V303', 'V286', 'V309', 'V284', 'V310']
v += ['V281', 'V301', 'V282', 'V315', 'V289', 'V296', 'V314', 'V283']
v += ['V332', 'V338', 'V337', 'V336', 'V325', 'V326', 'V328', 'V335']
len(v)
 def set_approach(a,b):
return list(set(a)-set(b))
tf_V = [bool(re.search("^V"+"[0-9]+",col)) for col in xs.columns.to_list() ]
cols = xs.columns[list(~np.array(tf_V)) ].to_list()
alldata['Embarked']=alldata['Embarked'].fillna('S' )<count_missing_values>
cols +=v
print(len(cols))
alldata.isnull().sum()<create_dataframe>
cols_rem = ['TransactionID', 'TransactionDT']
cols_rem +=["DayNum","HrOfDay","WkDayNum",'R_emaildomain1', 'R_emaildomain2', 'P_emaildomain1', 'P_emaildomain2','id_31_browser',"DeviceInfo_make"]
cols = set_approach(cols,cols_rem )
pclass_list = list(alldata['Pclass'].unique())
 df_averages = []
for classes in pclass_list:
df_averages.append(( alldata.loc[alldata['Pclass']==classes]['Age'].mean()))
averages = pd.DataFrame(df_averages,index=pclass_list,columns=['Average age in class'])
 averages<create_dataframe>
def ohe(df,cols):
for col in cols:
df[col+"_OHE"] = df[col].isna().astype(int )
pclass_list = list(alldata['Pclass'].unique())
 df_fares = []
for fares in pclass_list:
df_fares.append(( alldata.loc[alldata['Pclass']==fares]['Fare'].median()))
averages2 = pd.DataFrame(df_fares,index=pclass_list,columns=['Average fare in class'])
 averages2<feature_engineering>
cols_ohe = ["D2","D9"]
ohe(xs,cols_ohe)
 ohe(test_xs,cols_ohe)
 cols += ["D2_OHE","D9_OHE"]
alldata.loc[(alldata['Age'].isnull())&(alldata['Pclass']== 1),'Age'] = averages.loc[1,'Average age in class']
alldata.loc[(alldata['Age'].isnull())&(alldata['Pclass']== 2),'Age'] = averages.loc[2,'Average age in class']
alldata.loc[(alldata['Age'].isnull())&(alldata['Pclass']== 3),'Age'] = averages.loc[3,'Average age in class']
alldata.loc[(alldata['Fare'].isnull())&(alldata['Pclass']== 1),'Fare'] = averages2.loc[1,'Average fare in class']
alldata.loc[(alldata['Fare'].isnull())&(alldata['Pclass']== 2),'Fare'] = averages2.loc[2,'Average fare in class']
alldata.loc[(alldata['Fare'].isnull())&(alldata['Pclass']== 3),'Fare'] = averages2.loc[3,'Average fare in class']
alldata.isnull().sum()<count_values>
cols_rem =["D2","D9","D12"]
cols = set_approach(cols,cols_rem )
farebandlist = list(alldata['FareBand'].unique())
 farelist =[]
for fares in farebandlist:
farelist.append(alldata.loc[(alldata['type']=='train')&(alldata['FareBand']== fares),'Survived'].value_counts())
 farelist = pd.DataFrame(farelist,index=farebandlist)
 farelist.columns 
<feature_engineering>
cols += ["DayNum","HrOfDay","WkDayNum"]
farelist[[0.0, 1.0]] = farelist[[0.0, 1.0]].apply(lambda x: x/x.sum() , axis=1)
 farelist<data_type_conversions>
def Dn(df,cols):
for col in cols:
df[col+"n"] = df["DayNum"] - df[col]
alldata['FareBand'] = alldata['FareBand'].astype(np.int64)
 alldata.info()<categorify>
d_cols = ['D1',"D2", 'D3', 'D4', 'D5', 'D6', 'D7', "D8", "D9", 'D10', 'D11', "D12",
'D13', 'D14', 'D15']
Dn(xs,d_cols)
 Dn(test_xs,d_cols)
 d_cols = [string +"n" for string in d_cols]
cols +=d_cols
sex1 = pd.get_dummies(alldata['Sex'],drop_first=True)
 embarked1 = pd.get_dummies(alldata['Embarked'],drop_first=True)
 alldata.drop(['Sex','Embarked'],axis=1,inplace=True)
<concatenate>
def encode_CB2(df1,df2,uid):
newcol = "_".join(uid)
 df1[newcol] = df1[uid].astype(str ).apply(lambda x: '_'.join(x), axis=1)
 df2[newcol] = df2[uid].astype(str ).apply(lambda x: '_'.join(x), axis=1)
 temp_df = pd.concat([df1[newcol],df2[newcol]],axis=0)
 temp_df,_ = temp_df.factorize(sort=True)
 if temp_df.max() >32000:
df1[newcol+"_fact"] = temp_df[:len(df1)].astype('int32')
 df2[newcol+"_fact"] = temp_df[len(df1):].astype('int32')
 else:
df1[newcol+"_fact"] = temp_df[:len(df1)].astype('int16')
 df2[newcol+"_fact"] = temp_df[len(df1):].astype('int16')
 print(newcol+"_fact")
 return [newcol+"_fact"]
alldata = pd.concat([alldata,sex1,embarked1],axis=1 )<drop_column>
def encode_ag(df,df_te,uid,cols_ag,func_list,ag=True):
new_cols_ret = []
temp_df = pd.concat([df[uid+cols_ag],df_te[uid+cols_ag]],axis=0 ).reset_index(drop=True)
 temp_df[uid] = temp_df[uid].fillna(-9999)
 grouped = temp_df.groupby(uid)
 for func in func_list:
temp2_df = grouped[cols_ag].transform(func ).reset_index(drop=True)
 if ag:
new_cols = [col+"_"+"_".join(uid)+"_"+func+"2" for col in cols_ag]
else:
new_cols = ["_".join(uid)+"_FE2" for col in cols_ag]
new_cols_ret +=new_cols
df[new_cols] = temp2_df[0:len(df)].fillna(-9999 ).astype('float32')
 df_te[new_cols] = temp2_df[len(df):].fillna(-9999 ).reset_index(drop=True ).astype('float32')
 print(new_cols_ret)
 return new_cols_ret
alldata.drop(['Ticket','Fare','Age'],axis=1,inplace=True )<filter>
uid = ["D1n","card1","addr1","P_emaildomain"]
print(xs[uid].isna().sum())
train = alldata.loc[alldata['type']=='train']
test = alldata.loc[alldata['type']=='test']<drop_column>
cols+= encode_CB2(xs,test_xs,["card1","addr1"])
 cols+= encode_CB2(xs,test_xs,uid)
 cols += encode_ag(xs,test_xs,["addr1"],["TransactionID"],["count"],ag=False)
 cols += encode_ag(xs,test_xs,["card1"],["TransactionID"],["count"],ag=False)
 cols += encode_ag(xs,test_xs,["card2"],["TransactionID"],["count"],ag=False)
 cols += encode_ag(xs,test_xs,["dist1"],["TransactionID"],["count"],ag=False)
 cols += encode_ag(xs,test_xs,["card1","addr1"],["TransactionID"],["count"],ag=False)
 cols += encode_ag(xs,test_xs,uid,["TransactionID"],["count"],ag=False)
 cols += encode_ag(xs,test_xs,["card1","D1n"],["TransactionID"],["count"],ag=False)
 cols += encode_ag(xs,test_xs,["P_emaildomain"],["TransactionID"],["count"],ag=False)
 cols += encode_ag(xs,test_xs,["TransactionAmt"],["TransactionID"],["count"],ag=False)
 cols += encode_ag(xs,test_xs,["cents"],["TransactionID"],["count"],ag=False)
 cols += encode_ag(xs,test_xs,["ProductCD"],["TransactionID"],["count"],ag=False)
 cols += encode_ag(xs,test_xs,["HrOfDay"],["TransactionID"],["count"],ag=False)
 cols_ag_av = ["D3n","M7","D9","D10n","D13n","D5n","D15n","D4n","D6n","D11n","HrOfDay","D14n","id_13"]
cols_ag_other = ["TransactionAmt","cents"]
cols_C = ["C"+str(i)for i in range(1,15)]
cols_M = ['M'+str(x)for x in range(1,10)if x!= 7]
cols += encode_ag(xs,test_xs,uid,cols_ag_av+cols_ag_other, ["std","mean"])
 cols += encode_ag(xs,test_xs,["card1","addr1","P_emaildomain"],cols_ag_av+cols_ag_other, ["std","mean"])
 cols += encode_ag(xs,test_xs,uid,cols_C, ["std","mean"])
 cols_ag_v = ["V9","V314","V3","V171","V81","V289","V88","V89","V271","V326","V7","V52","V1"]
cols += encode_ag(xs,test_xs,uid,cols_ag_v, ["std","mean"])
 cols_ag_numb = ["addr2","ProductCD","dist1","DeviceType"]
cols += encode_ag(xs,test_xs,uid,cols_ag_numb+cols_M, ["nunique"] )
train = train.drop(['type'],axis=1)
<count_missing_values>
cols = set_approach(cols,["DayNum"] )
train.isnull().sum()<count_missing_values>
i=4
if i==4: get_feat_imp=True
get_feat_imp
train.isnull().sum()<drop_column>
msk = np.arange(len(xs)) <0.8*len(xs)
 idxT = list(np.where(msk)[0])
 idxV = list(np.where(~msk)[0] )
test = test.drop(['type'],axis=1 )<count_missing_values>
%%time
if buildLv:
get_feat_imp=False
n=5
preds = np.zeros(len(test_xs))
auc_lst = []
for i in range(n):
print("iteation",i)
 if i==4: get_feat_imp=True
a,temp = xgb_lvd_fun(get_feat_imp)
 preds += temp/n
auc_lst.append(a)
 del temp; x=gc.collect()
auc_lst = np.array(auc_lst)
 print(auc_lst.mean(axis=0))
print(auc_lst.std(axis=0))
format_test_preds(preds,test_xs,"baselineLV")
test.isnull().sum()<count_missing_values>
%%time
if buildSkf:
preds = xgb_pred_fun()
format_test_preds(preds,test_xs,"baseline")
<prepare_x_and_y><EOS> 
if AV:
df_dom = pd.concat([xs[cols], test_xs[cols]])
 is_test = pd.DataFrame([0]*len(xs)+ [1]*len(test_xs))
idxT,idxV = train_test_split(np.arange(len(df_dom)) ,test_size=0.2)
 del xs, test_xs; x=gc.collect()
clf = xgb.XGBClassifier(n_estimators=1000,
max_depth=8,
learning_rate= 0.05,
subsample= 0.6,
colsample_bytree= 0.4,
random_state = r.randint(0,9999),
use_label_encoder=False,
tree_method='gpu_hist')
 m = clf.fit(df_dom.iloc[idxT], is_test.iloc[idxT], eval_set=[(df_dom[cols].iloc[idxV],is_test.iloc[idxV])],
eval_metric= "auc", verbose=100, early_stopping_rounds=100)
 fi = xgb_fi(m, df_dom,df_dom)
 fi.to_csv("fi_av.csv",index=False)
 fi