Kaggle competition: IEEE-CIS Fraud Detection
%matplotlib inline
plt.style.use('ggplot')
 pd.options.display.precision = 15
warnings.filterwarnings("ignore")
<categorify>
pd.options.display.max_rows = 500
pd.options.display.max_columns = 100
def reduce_mem_usage(df, verbose=True):
start_mem = df.memory_usage(deep=True ).sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage(deep=True ).sum() / 1024**2
if verbose:
print(f'Memory usage of dataframe is {start_mem} MB --> {end_mem} MB(Decreased by {100 *(start_mem - end_mem)/ start_mem})')
 return df 
<load_from_csv>
train = pd.read_csv('/kaggle/input/ieee-fraud-detection/train_transaction.csv')
 train_ind = pd.read_csv('/kaggle/input/ieee-fraud-detection/train_identity.csv')
 test = pd.read_csv('/kaggle/input/ieee-fraud-detection/test_transaction.csv')
 test_ind = pd.read_csv('/kaggle/input/ieee-fraud-detection/test_identity.csv' )
%%time
folder_path = '.. /input/'
train_identity = pd.read_csv(f'{folder_path}train_identity.csv')
 train_transaction = pd.read_csv(f'{folder_path}train_transaction.csv')
 test_identity = pd.read_csv(f'{folder_path}test_identity.csv')
 test_transaction = pd.read_csv(f'{folder_path}test_transaction.csv')
 sub = pd.read_csv(f'{folder_path}sample_submission.csv')
 train = pd.merge(train_transaction, train_identity, on='TransactionID', how='left')
 test = pd.merge(test_transaction, test_identity, on='TransactionID', how='left')
 identity_col=list(train_identity.columns)
 trasac_col=list(train_transaction.columns)
 del train_identity, train_transaction, test_identity, test_transaction
train=reduce_mem_usage(train)
 test=reduce_mem_usage(test)
 gc.collect()<feature_engineering>
train_len = len(train)
def datetime_trans(train,start_date='2017-11-30'):
startdate=datetime.datetime.strptime(start_date,"%Y-%m-%d")
 train['TransactionDT']=train['TransactionDT'].fillna(train['TransactionDT'].mean())
 train['date']=train['TransactionDT'].apply(lambda x : datetime.timedelta(seconds=x)+startdate)
 train['weekday']=train['date'].apply(lambda x :x.weekday())
 train['month']=(train['date'].dt.year-2017)*12+train['date'].dt.month
train['hour']=train['date'].apply(lambda x :x.hour)
 train['day']=(train['date'].dt.year-2017)*365+train['date'].dt.dayofyear
train['year_weekday']=train['date'].apply(lambda x : str(x.year)+'_'+str(x.weekday()))
train['weekday_hour']=train['date'].apply(lambda x :str(x.weekday())+'_'+str(x.hour))
date_col=['weekday','month','day','hour','year_weekday','weekday_hour']
datetime_trans(train)
 datetime_trans(test)
<data_type_conversions>
train = train.merge(train_ind, how = 'left', on ='TransactionID')
 test = test.merge(test_ind, how = 'left', on ='TransactionID')
 del train_ind,test_ind
def addNewFeatures(data):
data['uid'] = data['card1'].astype(str)+'_'+data['card2'].astype(str)
 data['uid2'] = data['uid'].astype(str)+'_'+data['card3'].astype(str)+'_'+data['card5'].astype(str)
 data['uid3'] = data['uid2'].astype(str)+'_'+data['addr1'].astype(str)+'_'+data['addr2'].astype(str)
 data['uid4'] = data['addr1'].astype(str)+'_'+data['addr2'].astype(str)
 data['D9'] = np.where(data['D9'].isna() ,0,1)
 return data
train = addNewFeatures(train)
 test = addNewFeatures(test )<categorify>
all_data = pd.concat([train, test])
 del train,test
gc.collect()
agg_cols = ['card1','card2','card3','card5','uid','uid2','uid3','uid4']
def add_agg_col(col_prefix,agg_col,col_suffix='TransactionAmt'):
if isinstance(agg_col,list):
temp_df=pd.concat([train[[col_prefix,col_suffix]],test[[col_prefix,col_suffix]]])
 temp_df=temp_df.groupby(col_prefix)[col_suffix].agg(agg_col)
 for c in agg_col:
new_col=col_prefix+'_'+c+'_'+col_suffix
train[new_col]=train[col_prefix].map(temp_df[c])
 test[new_col]=test[col_prefix].map(temp_df[c])
 else:
raise TypeError('agg_col must be List')
 for i in agg_cols:
add_agg_col(i,['mean','std'])
 print(f'{i} for ['mean','std'] aggregate is done!')
<categorify>
START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d')
 all_data['DT_time'] = all_data['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 all_data['count'] = 1
all_data['diff_days_from_first_transaction'] = all_data['D1'].fillna(0 ).apply(lambda x:(datetime.timedelta(days = x)))
 all_data['client_firstdate'] = all_data['DT_time']- all_data['diff_days_from_first_transaction']
all_data['client_firstdate_days'] =(all_data['DT_time']- all_data['diff_days_from_first_transaction'] ).apply(lambda x:str(x.date()))
train['id_02_to_mean_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('mean')
 train['id_02_to_mean_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('mean')
 train['id_02_to_std_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('std')
 train['id_02_to_std_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('std')
 test['id_02_to_mean_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('mean')
 test['id_02_to_mean_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('mean')
 test['id_02_to_std_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('std')
 test['id_02_to_std_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('std')
 train['D15_to_mean_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('mean')
 train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')
 train['D15_to_std_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('std')
 train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')
 test['D15_to_mean_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('mean')
 test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')
 test['D15_to_std_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('std')
 test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std')
 train['D15_to_mean_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('mean')
 train['D15_to_mean_addr2'] = train['D15'] / train.groupby(['addr2'])['D15'].transform('mean')
 train['D15_to_std_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('std')
 train['D15_to_std_addr2'] = train['D15'] / train.groupby(['addr2'])['D15'].transform('std')
 test['D15_to_mean_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('mean')
 test['D15_to_mean_addr2'] = test['D15'] / test.groupby(['addr2'])['D15'].transform('mean')
 test['D15_to_std_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('std')
 test['D15_to_std_addr2'] = test['D15'] / test.groupby(['addr2'])['D15'].transform('std')
 train=train.replace([np.inf,-np.inf],np.nan)
 test=test.replace([np.inf,-np.inf],np.nan )<categorify>
anaysis_fea = [ 'TransactionID',
'isFraud',
'TransactionDT',
'TransactionAmt',
'ProductCD',
'device_hash','card_hash', 'V307','id_30','id_31','id_32','id_33','DeviceType','DeviceInfo',
'card1','card2','card3','card4','card5','card6','client_firstdate_days','dist1','dist2','P_emaildomain','addr1','addr2','train_or_test','count','DT_time','diff_days_from_first_transaction','client_firstdate']
drop_fea = [col for col in all_data.columns if col not in anaysis_fea]
def p_r_domain(train,test,col):
train[col]=train[col].fillna('Nan' ).apply(lambda x : x.split('.')[0])
 test[col]=test[col].fillna('Nan' ).apply(lambda x : x.split('.')[0])
 for c in ['P_emaildomain','R_emaildomain']:
p_r_domain(train,test,c )<feature_engineering>
all_data = all_data.drop(drop_fea,axis=1 )
freq_cols = ['card1','card2','card3','card5',
'C1','C2','C3','C4','C5','C6','C7','C8','C9','C10','C11','C12','C13','C14',
'D1','D2','D3','D4','D5','D6','D7','D8',
'addr1','addr2',
'dist1','dist2',
'P_emaildomain', 'R_emaildomain',
'DeviceInfo','DeviceType',
'id_30','id_33',
'uid','uid2','uid3','uid4'
]+date_col
def set_freq_col(train,test,col):
prefix='_fq'
temp_df=pd.concat([train[[col]],test[[col]]])
 fq=temp_df[col].value_counts(dropna=False)
 train[col+prefix]=train[col].map(fq)
 test[col+prefix]=test[col].map(fq)
 for c in freq_cols:
set_freq_col(train,test,c)
 periods = ['month','year_weekday','weekday_hour']
uids = ['uid','uid2','uid3','uid4']
def set_uid_period(train,test,periods,uids):
for period in periods:
for col in uids:
new_column = col + '_' + period
temp_df = pd.concat([train[[col,period]], test[[col,period]]])
 temp_df[new_column] = temp_df[col].astype(str)+ '_' +(temp_df[period] ).astype(str)
 fq_encode = temp_df[new_column].value_counts()
train[new_column] =(train[col].astype(str)+ '_' + train[period].astype(str)).map(fq_encode)
 test[new_column] =(test[col].astype(str)+ '_' + test[period].astype(str)).map(fq_encode)
 train[new_column] /= train[period+'_fq']
test[new_column] /= test[period+'_fq']
set_uid_period(train,test,periods,uids )<prepare_x_and_y>
gc.collect()
tr_na_count=train.isnull().sum() /len(train)
 tr_drop_cols=[c for c in train.columns if tr_na_count[c]>0.85]
tr_big_cols=[c for c in train.columns if train[c].value_counts(normalize=True,dropna=False ).values[0]>0.85]
drop_cols=list(set(tr_drop_cols+tr_big_cols))
drop_cols.remove('isFraud')
 y_=train['isFraud']
train.drop(columns=drop_cols+['isFraud'],inplace=True)
 test.drop(columns=drop_cols,inplace=True)
 excess_col=['date','TransactionDT','TransactionID']
train.drop(columns=excess_col,inplace=True)
 test.drop(columns=excess_col,inplace=True )<categorify>
all_data['card1'] = all_data['card1'].fillna(0)
 all_data['card2'] = all_data['card2'].fillna(0)
 all_data['card3'] = all_data['card3'].fillna(0)
 all_data['card5'] = all_data['card5'].fillna(0)
 all_data['card4'] = all_data['card4'].fillna('nan')
 all_data['card6'] = all_data['card6'].fillna('nan' )
def imputing_na(train,col):
if train[col].dtypes == object:
imp=SimpleImputer(strategy='constant',fill_value='Nan' ).fit_transform(train[col].values.reshape(-1,1))
train[col]=pd.Series(imp[:,0])
 else:
imp=SimpleImputer(strategy='constant',fill_value=-999 ).fit_transform(train[col].values.reshape(-1,1))
train[col]=pd.Series(imp[:,0])
 for c in train.columns:
imputing_na(train,c)
 imputing_na(test,c)
<categorify>
all_data['card_hash'] = all_data.apply(lambda x: card_info_hash(x), axis=1 )
numerical_cols = train.select_dtypes(exclude = 'object' ).columns
categorical_cols = train.select_dtypes(include = 'object' ).columns
def labelencoder(train,test,col):
cod=list(train[col].values)+list(test[col].values)
 le=LabelEncoder().fit(cod)
 train[col]=le.transform(train[col])
 test[col]=le.transform(test[col])
 for c in categorical_cols:
labelencoder(train,test,c)
<init_hyperparams>
def get_data_by_card_hash(data, card_hash):
mask = data['card_hash']==card_hash
return data.loc[mask,:].copy()
def get_data_by_device_hash(data, device_hash):
mask = data['device_hash']==device_hash
return data.loc[mask,:].copy()
def get_data_by_card_and_startdate(data, card_hash, device_hash):
mask =(data['client_firstdate_days']==card_hash)&(data['card_hash']==device_hash)
 return data.loc[mask,:].copy()
params = {'num_leaves': int(( 2**10)*0.72),
'min_child_weight': 0.17,
'feature_fraction': 0.72,
'bagging_fraction': 0.72,
'min_data_in_leaf': 179,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.006,
"boosting_type": "gbdt",
"bagging_seed": 13,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3299927210061127,
'reg_lambda': 0.3885237330340494,
'random_state': 4,
}<split>
all_data['count']=1
grp = all_data.iloc[:train_len].groupby(['client_firstdate_days','card_hash'])['count'].agg('sum')
 display_group = get_data_by_card_and_startdate(all_data,grp[grp>10].index[0][0],grp[grp>10].index[0][1])
 display_group[['DT_time','TransactionAmt','V307','id_30','id_31','id_32','id_33','DeviceType','DeviceInfo','dist1','dist2','P_emaildomain']]
%%time
NFOLDS = 7
folds = KFold(n_splits=NFOLDS)
 columns = train.columns
splits = folds.split(train, y_)
 y_preds = np.zeros(test.shape[0])
 y_oof = np.zeros(train.shape[0])
 score = 0
feature_importances = pd.DataFrame()
feature_importances['feature'] = columns
for fold_n,(train_index, valid_index)in enumerate(splits):
X_train, X_valid = train[columns].iloc[train_index], train[columns].iloc[valid_index]
y_train, y_valid = y_.iloc[train_index], y_.iloc[valid_index]
dtrain = lgb.Dataset(X_train, label=y_train)
 dvalid = lgb.Dataset(X_valid, label=y_valid)
 clf = lgb.train(params, dtrain, 10000, valid_sets = [dtrain, dvalid], verbose_eval=200, early_stopping_rounds=300)
 feature_importances[f'fold_{fold_n + 1}'] = clf.feature_importance()
y_pred_valid = clf.predict(X_valid)
 y_oof[valid_index] = y_pred_valid
print(f"Fold {fold_n + 1} | AUC: {roc_auc_score(y_valid, y_pred_valid)}")
 score += roc_auc_score(y_valid, y_pred_valid)/ NFOLDS
y_preds += clf.predict(test)/ NFOLDS
del X_train, X_valid, y_train, y_valid
gc.collect()
print(f"
Mean AUC = {score}" )<prepare_output>
s = all_data.iloc[:train_len].groupby(['client_firstdate_days','card_hash'])['isFraud'].agg(['mean', 'count'] )
sub['isFraud']= y_preds<save_to_csv>
Test_ID=[]
for ind in tqdm(s[(s['mean']==1)].index):
very_strange_thing = get_data_by_card_and_startdate(all_data, ind[0],ind[1])
 Test_ID.extend(very_strange_thing[very_strange_thing['isFraud'].isna() ]['TransactionID'].tolist() )
sub.to_csv('submission.csv', index=False)
<load_from_csv>
submit = pd.read_csv('.. /input/rank-blend/easy_blend4.csv' )
sub=pd.read_csv('.. /input/ieee-fraud-detection/sample_submission.csv' )<load_from_csv>
mask = submit['TransactionID'].isin(Test_ID)
 submit.loc[mask,'isFraud'] =1
df0=pd.read_csv('.. /input/a-method-to-valid-offline-lb-9506/simple_ensemble6.csv')
 df1=pd.read_csv('.. /input/lgb-fe-0-9492-lb-newfeature-0-9496-lb/submission.csv')
 df2=pd.read_csv('.. /input/lightgbm-feature-engineering-eda-with-r/submission.csv')
 df3=pd.read_csv('.. /input/ieee-gb-2-make-amount-useful-again/submission.csv')
 df4=pd.read_csv('.. /input/lgb-single-model-lb-0-9419/ieee_cis_fraud_detection_v2.csv')
 df5=pd.read_csv('.. /input/feature-engineering-lightgbm/submission.csv')
 df6=pd.read_csv('.. /input/ieee-lgbm-with-groupkfold-cv/submission.csv')
 df7=pd.read_csv('.. /input/lightgbm-single-model-and-feature-engineering/submission.csv')
 df8=pd.read_csv('.. /input/is-it-really-fraud/submission.csv')
 df9=pd.read_csv('.. /input/cis-fraud-detection-visualize-feature-engineering/prediction.csv')
 df10=pd.read_csv('.. /input/ieee-simple-lgbm/submission.csv' )<define_variables>
submit.loc[mask,:]
<prepare_output><EOS> 
submit.to_csv('submit_try.csv',index = False )