Kaggle competition: IEEE-CIS Fraud Detection
for df in [train_store_1, train_store_2, test_store_1, test_store_2]:
df["visitId"] = df["Client Id"].apply(lambda x: x.split('.', 1)[1] ).astype(np.int64 )<merge>
def make_hour_feature(df, tname='TransactionDT'):

 hours = df[tname] /(3600)
 encoded_hours = np.floor(hours)% 24
return encoded_hours
def reduce_mem_usage(df):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df
train = train.merge(pd.concat([train_store_1, train_store_2], sort=False), how="left", on="visitId")
 test = test.merge(pd.concat([test_store_1, test_store_2], sort=False), how="left", on="visitId")
 for df in [train, test]:
df.drop("Client Id", 1, inplace=True )<data_type_conversions>
%%time
folder_path = '.. /input/ieee-fraud-detection/'
print('Loading data...')
 train_identity = pd.read_csv(f'{folder_path}train_identity.csv', index_col='TransactionID')
 print('\tSuccessfully loaded train_identity!')
 train_transaction = pd.read_csv(f'{folder_path}train_transaction.csv', index_col='TransactionID')
 print('\tSuccessfully loaded train_transaction!')
 test_identity = pd.read_csv(f'{folder_path}test_identity.csv', index_col='TransactionID')
 print('\tSuccessfully loaded test_identity!')
 test_transaction = pd.read_csv(f'{folder_path}test_transaction.csv', index_col='TransactionID')
 print('\tSuccessfully loaded test_transaction!')
 sub = pd.read_csv(f'{folder_path}sample_submission.csv')
 print('\tSuccessfully loaded sample_submission!')
 print('Data was successfully loaded!
')
 print('Merging data...')
 train = pd.merge(train_transaction, train_identity, on='TransactionID', how='left', left_index=True, right_index=True)
 test = pd.merge(test_transaction, test_identity, on='TransactionID', how='left', left_index=True, right_index=True)
 print('Data was successfully merged!
')
 del train_identity, train_transaction, test_identity, test_transaction
print(f'Train dataset has {train.shape[0]} rows and {train.shape[1]} columns.')
 print(f'Test dataset has {test.shape[0]} rows and {test.shape[1]} columns.
 ')
 gc.collect()
for df in [train, test]:
df["Revenue"].fillna('$', inplace=True)
 df["Revenue"] = df["Revenue"].apply(lambda x: x.replace('$', '' ).replace(',', ''))
df["Revenue"] = pd.to_numeric(df["Revenue"], errors="coerce")
 df["Revenue"].fillna(0.0, inplace=True )<drop_column>
train['TransactionAmt_to_mean_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_mean_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_std_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('std')
 train['TransactionAmt_to_std_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_mean_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_mean_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_std_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_std_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('std')
 train['id_02_to_mean_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('mean')
 train['id_02_to_mean_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('mean')
 train['id_02_to_std_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('std')
 train['id_02_to_std_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('std')
 test['id_02_to_mean_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('mean')
 test['id_02_to_mean_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('mean')
 test['id_02_to_std_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('std')
 test['id_02_to_std_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('std')
 train['D15_to_mean_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('mean')
 train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')
 train['D15_to_std_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('std')
 train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')
 test['D15_to_mean_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('mean')
 test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')
 test['D15_to_std_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('std')
 test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std')
 train['D15_to_mean_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('mean')
 train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')
 train['D15_to_std_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('std')
 train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')
 test['D15_to_mean_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('mean')
 test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')
 test['D15_to_std_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('std')
 test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std')
 train['Hours'] = make_hour_feature(train)
 test['Hours'] = make_hour_feature(test )
for df in [train_store_1, train_store_2, test_store_1, test_store_2]:
del df
gc.collect()<groupby>
emails = {'gmail': 'google', 'att.net': 'att', 'twc.com': 'spectrum', 'scranton.edu': 'other', 'optonline.net': 'other',
'hotmail.co.uk': 'microsoft', 'comcast.net': 'other', 'yahoo.com.mx': 'yahoo', 'yahoo.fr': 'yahoo', 'yahoo.es': 'yahoo',
'charter.net': 'spectrum', 'live.com': 'microsoft', 'aim.com': 'aol', 'hotmail.de': 'microsoft', 'centurylink.net': 'centurylink',
'gmail.com': 'google', 'me.com': 'apple', 'earthlink.net': 'other', 'gmx.de': 'other', 'web.de': 'other', 'cfl.rr.com': 'other',
'hotmail.com': 'microsoft', 'protonmail.com': 'other', 'hotmail.fr': 'microsoft', 'windstream.net': 'other', 'outlook.es': 'microsoft',
'yahoo.co.jp': 'yahoo', 'yahoo.de': 'yahoo', 'servicios-ta.com': 'other', 'netzero.net': 'other', 'suddenlink.net': 'other',
'roadrunner.com': 'other', 'sc.rr.com': 'other', 'live.fr': 'microsoft', 'verizon.net': 'yahoo', 'msn.com': 'microsoft', 'q.com': 'centurylink',
'prodigy.net.mx': 'att', 'frontier.com': 'yahoo', 'anonymous.com': 'other', 'rocketmail.com': 'yahoo', 'sbcglobal.net': 'att',
'frontiernet.net': 'yahoo', 'ymail.com': 'yahoo', 'outlook.com': 'microsoft', 'mail.com': 'other', 'bellsouth.net': 'other',
'embarqmail.com': 'centurylink', 'cableone.net': 'other', 'hotmail.es': 'microsoft', 'mac.com': 'apple', 'yahoo.co.uk': 'yahoo',
'netzero.com': 'other', 'yahoo.com': 'yahoo', 'live.com.mx': 'microsoft', 'ptd.net': 'other', 'cox.net': 'other', 'aol.com': 'aol',
'juno.com': 'other', 'icloud.com': 'apple'}
us_emails = ['gmail', 'net', 'edu']
target_sums = train.groupby("fullVisitorId")["totals.transactionRevenue"].sum().reset_index()<count_unique_values>
for c in ['P_emaildomain', 'R_emaildomain']:
train[c + '_bin'] = train[c].map(emails)
 test[c + '_bin'] = test[c].map(emails)
 train[c + '_suffix'] = train[c].map(lambda x: str(x ).split('.')[-1])
 test[c + '_suffix'] = test[c].map(lambda x: str(x ).split('.')[-1])
 train[c + '_suffix'] = train[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 test[c + '_suffix'] = test[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us' )
ids_train = set(train.fullVisitorId.unique())
 ids_test = set(test.fullVisitorId.unique())
 print("Unique visitor ids in train:", len(ids_train))
print("Unique visitor ids in test:", len(ids_test))
print("Common visitors in train and test:", len(ids_train & ids_test))<count_values>
drop_columns = ['TransactionDT', 'V300', 'V309', 'V111', 'C3', 'V124', 'V106', 'V125', 'V315', 'V134', 'V102', 'V123', 'V316', 'V113',
'V136', 'V305', 'V110', 'V299', 'V289', 'V286', 'V318', 'V103', 'V304', 'V116', 'V298', 'V284', 'V293', 'V137', 'V295',
'V301', 'V104', 'V311', 'V115', 'V109', 'V119', 'V321', 'V114', 'V133', 'V122', 'V319', 'V105', 'V112', 'V118', 'V117',
'V121', 'V108', 'V135', 'V320', 'V303', 'V297', 'V120']
train = train.drop(drop_columns, axis=1)
 test = test.drop(drop_columns, axis=1 )
problem = train[train.sessionId.map(train.sessionId.value_counts() == 2)].sort_values(["sessionId", 'visitStartTime'])
 problem.head(10 )<count_values>
y = train['isFraud'].copy()
X = train.drop('isFraud', axis=1)
 X_test = test.copy()
del train, test
gc.collect()
( train.visitStartTime == train.visitId ).value_counts()<data_type_conversions>
X = X.fillna(-999)
 X_test = X_test.fillna(-999 )
train.loc[pd.to_datetime(train.visitStartTime, unit='s')== "2017-04-25 18:49:35"].head(8 )<count_values>
for col in X.columns:
if X[col].dtype=='object' or X_test[col].dtype=='object':
lbl = LabelEncoder()
lbl.fit(list(X[col].values)+ list(X_test[col].values))
X[col] = lbl.transform(list(X[col].values))
X_test[col] = lbl.transform(list(X_test[col].values))
print("Train: ", np.bincount(train.visitId.value_counts()))<count_values>
%%time
X = reduce_mem_usage(X)
 X_test = reduce_mem_usage(X_test )
print("test: ", np.bincount(test.visitId.value_counts()))<data_type_conversions>
params = {'bagging_fraction': 0.8831783305477362,
'colsample_bytree': 0.5941836821402656,
'feature_fraction': 0.8359307480774029,
'learning_rate': 0.23836691517558517,
'max_depth': 13,
'min_child_samples': 105.54076348952397,
'min_child_weight': 0.005812630846994812,
'min_data_in_leaf': 24,
'num_leaves': 496,
'reg_alpha': 1.229972928485176,
'reg_lambda': 1.3080809161627962,
'subsample': 0.4071603440308888,
'metric':'auc',
'bagging_seed': 11,
'n_estimators': 5000,
}
train.visitStartTime = pd.to_datetime(train.visitStartTime, unit='s')
 test.visitStartTime = pd.to_datetime(test.visitStartTime, unit='s')
 train["date"] = train.visitStartTime
test["date"] = test.visitStartTime<sort_values>
%%time
NFOLDS = 7
folds = KFold(n_splits=NFOLDS)
 columns = X.columns
splits = folds.split(X, y)
 y_preds = np.zeros(X_test.shape[0])
 y_oof = np.zeros(X.shape[0])
 score = 0
for fold_n,(train_index, valid_index)in enumerate(splits):
clf = lgb.LGBMClassifier(**params)
 X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
clf.fit(X_train, y_train)
 y_pred_valid = clf.predict_proba(X_valid)[:,1]
y_oof[valid_index] = y_pred_valid
print(f"Fold {fold_n + 1} | AUC: {roc_auc_score(y_valid, y_pred_valid)}")
 score += roc_auc_score(y_valid, y_pred_valid)/ NFOLDS
y_preds += clf.predict_proba(X_test)[:,1] / NFOLDS
del X_train, X_valid, y_train, y_valid
gc.collect()
print(f"
Mean AUC = {score}")
 print(f"Out of folds AUC = {roc_auc_score(y, y_oof)}" )
train.set_index("visitStartTime", inplace=True)
 test.set_index("visitStartTime", inplace=True)
 train.sort_index(inplace=True)
 test.sort_index(inplace=True )<count_values>
preds = y_preds
def clearRare(columnname, limit = 1000):
vc = test[columnname].value_counts()
common = vc > limit
common = set(common.index[common].values)
 print("Set", sum(vc <= limit), columnname, "categories to 'other';", end=" ")
 train.loc[train[columnname].map(lambda x: x not in common), columnname] = 'other'
test.loc[test[columnname].map(lambda x: x not in common), columnname] = 'other'
print("now there are", train[columnname].nunique() , "categories in train" )<train_model>
sub['isFraud'] = preds
sub.to_csv('v3_noblend.csv', index=False )
train.fillna(0, inplace=True)
 test.fillna(0, inplace=True )<drop_column>
safebox = pd.read_csv('.. /input/safe-box/blend01.csv')
 safebox_preds = safebox['isFraud'].values
sub['isFraud'] =(preds + 2*safebox_preds)/3
blend01 =(preds + 2*safebox_preds)/3
sub.to_csv('v3_safebox_blend01.csv', index=False )
clearRare("device.browser")
 clearRare("device.operatingSystem")
 clearRare("geoNetwork.country")
 clearRare("geoNetwork.city")
 clearRare("geoNetwork.metro")
 clearRare("geoNetwork.networkDomain")
 clearRare("geoNetwork.region")
 clearRare("geoNetwork.subContinent")
 clearRare("trafficSource.adContent")
 clearRare("trafficSource.campaign")
 clearRare("trafficSource.keyword")
 clearRare("trafficSource.medium")
 clearRare("trafficSource.referralPath")
 clearRare("trafficSource.source" )<data_type_conversions>
safebox = pd.read_csv('.. /input/safe-box/blend02.csv')
 safebox_preds = safebox['isFraud'].values
sub['isFraud'] =(preds + 2*safebox_preds)/3
blend02 =(preds + 2*safebox_preds)/3
sub.to_csv('v3_safebox_blend02.csv', index=False )
for df in [train, test]:
df["Avg.Session Duration"][df["Avg.Session Duration"] == 0] = "00:00:00"
df["Avg.Session Duration"] = df["Avg.Session Duration"].str.split(':' ).apply(lambda x: int(x[0])* 60 + int(x[1]))
df["Bounce Rate"] = df["Bounce Rate"].astype(str ).apply(lambda x: x.replace('%', '')).astype(float)
 df["Goal Conversion Rate"] = df["Goal Conversion Rate"].astype(str ).apply(lambda x: x.replace('%', '')).astype(float )<feature_engineering>
safebox = pd.read_csv('.. /input/safe-box/submission.csv')
 safebox_preds = safebox['isFraud'].values
sub['isFraud'] =(preds + 2*safebox_preds)/3
sub.to_csv('v3_safebox_sub.csv', index=False )
for df in [train, test]:
df["id_incoherence"] = pd.to_datetime(df.visitId, unit='s')!= df.date
df["visitId_dublicates"] = df.visitId.map(df.visitId.value_counts())
 df["session_dublicates"] = df.sessionId.map(df.sessionId.value_counts() )<feature_engineering>
sub['isFraud'] =(blend01 + blend02)/2
sub.to_csv('v3_safebox_b1_and_b2.csv', index=False )
for df in [train, test]:
df['weekday'] = df['date'].dt.dayofweek.astype(object)
 df['time'] = df['date'].dt.second + df['date'].dt.minute*60 + df['date'].dt.hour*3600
df['day'] = df['date'].dt.date<data_type_conversions>
safebox = pd.read_csv('.. /input/safe-box/blend02.csv')
 safebox_preds = safebox['isFraud'].values
sub['isFraud'] =(preds + safebox_preds)/2
sub.to_csv('v3_safebox_equal_parts.csv', index=False )
<feature_engineering><EOS> 
safebox = pd.read_csv('.. /input/stackers-blend-top-4/All_Blends.csv')
 safebox_preds = safebox['isFraud'].values
sub['isFraud'] =(preds + safebox_preds)/2
sub.to_csv('v3_stacking_equal_parts.csv', index=False )