Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<filter>
warnings.filterwarnings('ignore' )
test_df[test_df.Fare.isna() ]<categorify>
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed )
def getFareType(row):
if -0.512 < row['Fare'] <= 102.466:
return '1'
elif 102.466 < row['Fare'] <= 204.932:
return '2'
elif 204.932 < row['Fare'] <= 307.398:
return '3'
elif 409.863 < row['Fare'] <= 512.329:
return '5'
else :
return '4'
test_df['FareType'] = test_df.apply(lambda row: getFareType(row), axis=1)
 test_df.head()<count_values>
SEED = 42
seed_everything(SEED)
 TARGET = 'isFraud'
START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d' )
test_df.FareType.value_counts()<count_values>
lgb_params = {
'objective':'binary',
'boosting_type':'gbdt',
'metric':'auc',
'n_jobs':-1,
'learning_rate':0.01,
'num_leaves': 2**8,
'max_depth':-1,
'tree_learner':'serial',
'colsample_bytree': 0.7,
'subsample_freq':1,
'subsample':0.7,
'n_estimators':20000,
'max_bin':255,
'verbose':-1,
'seed': SEED,
'early_stopping_rounds':100,
}
for df in dataset:
df['FamilySize'] = df.Parch + df.SibSp + 1
train_df.FamilySize.value_counts()<categorify>
print('Load Data')
 train_df = pd.read_pickle('.. /input/ieee-data-minification/train_transaction.pkl')
 train_df['DT_M'] = train_df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 train_df['DT_M'] =(train_df['DT_M'].dt.year-2017)*12 + train_df['DT_M'].dt.month
test_df = train_df[train_df['DT_M']==train_df['DT_M'].max() ].reset_index(drop=True)
 train_df = train_df[train_df['DT_M']<(train_df['DT_M'].max())].reset_index(drop=True)
 print('Shape control:', train_df.shape, test_df.shape )
for df in dataset:
df['Single'] = df.FamilySize.map(lambda size: 1 if size==1 else 0)
 df['Medium'] = df.FamilySize.map(lambda size: 1 if 2<=size<=4 else 0)
 df['Large'] = df.FamilySize.map(lambda size: 1 if size>4 else 0 )<drop_column>
for col in list(train_df):
if train_df[col].dtype=='O':
print(col)
 train_df[col] = train_df[col].fillna('unseen_before_label')
 test_df[col] = test_df[col].fillna('unseen_before_label')
 train_df[col] = train_df[col].astype(str)
 test_df[col] = test_df[col].astype(str)
 le = LabelEncoder()
le.fit(list(train_df[col])+list(test_df[col]))
train_df[col] = le.transform(train_df[col])
 test_df[col] = le.transform(test_df[col])
 train_df[col] = train_df[col].astype('category')
 test_df[col] = test_df[col].astype('category' )
for df in dataset:
df.drop(['PassengerId', 'Name', 'SibSp', 'Parch', 'Ticket', 'Cabin', 'FamilySize', 'Age', 'Fare'], inplace=True, axis=1 )<count_missing_values>
rm_cols = [
'TransactionID','TransactionDT',
TARGET,
'DT_M'
]
rm_cols += ['V'+str(i)for i in range(1,340)]
features_columns = [col for col in list(train_df)if col not in rm_cols]
train_df.isnull().any()<count_missing_values>
RESULTS = test_df[['TransactionID',TARGET]]
N_SPLITS = 3
test_df.isnull().any()<categorify>
X,y = train_df[features_columns], train_df[TARGET]
P = test_df[features_columns]
for n_rounds in [500,1000,2500,5000]:
print('
print('No Validation training...', n_rounds, 'boosting rounds')
 corrected_lgb_params = lgb_params.copy()
corrected_lgb_params['n_estimators'] = n_rounds
corrected_lgb_params['early_stopping_rounds'] = None
train_data = lgb.Dataset(X, label=y)
 estimator = lgb.train(
 corrected_lgb_params,
train_data 
)
 RESULTS['no_validation_'+str(n_rounds)] = estimator.predict(P)
 print('AUC score', metrics.roc_auc_score(RESULTS[TARGET], RESULTS['no_validation_'+str(n_rounds)]))
print('
oh_cols = ['Pclass', 'Embarked', 'Title', 'AgeType', 'FareType']
train_df = pd.get_dummies(train_df, columns=oh_cols, prefix=oh_cols)
 test_df = pd.get_dummies(test_df, columns=oh_cols, prefix=oh_cols)
 train_df.head()<drop_column>
print('
print('KFold training...')
 folds = KFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)
 X,y = train_df[features_columns], train_df[TARGET]
P = test_df[features_columns]
RESULTS['kfold'] = 0
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X, y)) :
print('Fold:',fold_+1)
 tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]
vl_x, v_y = X.iloc[val_idx,:], y[val_idx]
train_data = lgb.Dataset(tr_x, label=tr_y)
 valid_data = lgb.Dataset(vl_x, label=v_y)
 estimator = lgb.train(
 lgb_params,
train_data,
valid_sets = [train_data, valid_data],
verbose_eval = 1000, 
)
 RESULTS['kfold'] = estimator.predict(P)
 print('AUC score', metrics.roc_auc_score(RESULTS[TARGET], RESULTS['kfold']))
print('
train_df.drop(['FareType_5'], axis=1, inplace=True )<load_from_csv>
print('
print('StratifiedKFold training...')
 folds = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)
 X,y = train_df[features_columns], train_df[TARGET]
P = test_df[features_columns]
RESULTS['stratifiedkfold'] = 0
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X, y, groups=y)) :
print('Fold:',fold_+1)
 tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]
vl_x, v_y = X.iloc[val_idx,:], y[val_idx]
train_data = lgb.Dataset(tr_x, label=tr_y)
 valid_data = lgb.Dataset(vl_x, label=v_y)
 estimator = lgb.train(
 lgb_params,
train_data,
valid_sets = [train_data, valid_data],
verbose_eval = 1000, 
)
 RESULTS['stratifiedkfold'] += estimator.predict(P)/N_SPLITS
print('AUC score', metrics.roc_auc_score(RESULTS[TARGET], RESULTS['stratifiedkfold']))
print('
test_df2 = pd.read_csv('/kaggle/input/titanic/test.csv' )<prepare_x_and_y>
print('
print('LBO training...')
 train_df['DT_M'] = train_df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 train_df['DT_M'] =(train_df['DT_M'].dt.year-2017)*12 + train_df['DT_M'].dt.month
main_train_set = train_df[train_df['DT_M']<(train_df['DT_M'].max())].reset_index(drop=True)
 validation_set = train_df[train_df['DT_M']==train_df['DT_M'].max() ].reset_index(drop=True)
 folds = KFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)
 X,y = main_train_set[features_columns], main_train_set[TARGET]
v_X, v_y = validation_set[features_columns], validation_set[TARGET]
estimators_bestround = []
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X, y)) :
print('Fold:',fold_+1)
 tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]
train_data = lgb.Dataset(tr_x, label=tr_y)
 valid_data = lgb.Dataset(v_X, label=v_y)
 estimator = lgb.train(
 lgb_params,
train_data,
valid_sets = [train_data, valid_data],
verbose_eval = 1000, 
)
 estimators_bestround.append(estimator.current_iteration())
 corrected_lgb_params = lgb_params.copy()
corrected_lgb_params['n_estimators'] = int(np.mean(estimators_bestround))
corrected_lgb_params['early_stopping_rounds'] = None
print('
print('Mean Best round:', corrected_lgb_params['n_estimators'])
 X,y = train_df[features_columns], train_df[TARGET]
P = test_df[features_columns]
RESULTS['lbo'] = 0
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X, y)) :
print('Fold:',fold_+1)
 tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]
train_data = lgb.Dataset(tr_x, label=tr_y)
 estimator = lgb.train(
 corrected_lgb_params,
train_data 
)
 RESULTS['lbo'] += estimator.predict(P)/N_SPLITS
print('AUC score', metrics.roc_auc_score(RESULTS[TARGET], RESULTS['lbo']))
print('
X, y = train_df.drop(['Survived'], axis=1),train_df['Survived']<compute_train_metric>
print('
print('GroupKFold timeblocks split training...')
 folds = GroupKFold(n_splits=N_SPLITS)
 train_df['groups'] = train_df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 train_df['groups'] =(train_df['groups'].dt.year-2017)*12 + train_df['groups'].dt.month
X,y = train_df[features_columns], train_df[TARGET]
split_groups = train_df['groups']
P = test_df[features_columns]
RESULTS['groupkfold_timeblocks'] = 0
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X, y, groups=split_groups)) :
print('Fold:',fold_+1)
 tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]
vl_x, v_y = X.iloc[val_idx,:], y[val_idx]
train_data = lgb.Dataset(tr_x, label=tr_y)
 valid_data = lgb.Dataset(vl_x, label=v_y)
 estimator = lgb.train(
 lgb_params,
train_data,
valid_sets = [train_data, valid_data],
verbose_eval = 1000, 
)
 RESULTS['groupkfold_timeblocks'] += estimator.predict(P)/N_SPLITS
print('AUC score', metrics.roc_auc_score(RESULTS[TARGET], RESULTS['groupkfold_timeblocks']))
print('
def compute_score(clf, X, y, cv, scoring='accuracy'):
xval = cross_val_score(clf, X, y, cv = cv, scoring=scoring)
 return np.mean(xval )<choose_model_class>
print('
print('GroupKFold uID split training...')
 folds = GroupKFold(n_splits=N_SPLITS)
 train_df['groups'] = ''
for col in ['card1','card2','card3','card5','addr1','addr2',]:
train_df['groups'] = '_' + train_df[col].astype(str)
 X,y = train_df[features_columns], train_df[TARGET]
split_groups = train_df['groups']
P = test_df[features_columns]
RESULTS['groupkfold_uid'] = 0
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X, y, groups=split_groups)) :
print('Fold:',fold_+1)
 tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]
vl_x, v_y = X.iloc[val_idx,:], y[val_idx]
train_data = lgb.Dataset(tr_x, label=tr_y)
 valid_data = lgb.Dataset(vl_x, label=v_y)
 estimator = lgb.train(
 lgb_params,
train_data,
valid_sets = [train_data, valid_data],
verbose_eval = 1000, 
)
 RESULTS['groupkfold_uid'] += estimator.predict(P)/N_SPLITS
print('AUC score', metrics.roc_auc_score(RESULTS[TARGET], RESULTS['groupkfold_uid']))
print('
cv = StratifiedKFold(n_splits=10, shuffle=True, random_state=1)
 model_lgbm = LGBMClassifier(
 boosting='dart',
objective='binary',
metric='binary_logloss,auc',
bagging_freq=5, bagging_fraction=0.75 
)
 score_lgbm = compute_score(model_lgbm, X=X, y=y, cv=cv)
 model_lgbm.fit(X, y)
 predictions_lgbm = model_lgbm.predict(test_df)
 output_lgbm = pd.DataFrame({'PassengerId': test_df2['PassengerId'], 'Survived': predictions_lgbm})
 output_lgbm.to_csv('lgbm_faretype_new.csv', index=False)
 print(score_lgbm )<find_best_model_class>
print('
print('Intermediate results...')
 final_df = []
for current_strategy in list(RESULTS.iloc[:,2:]):
auc_score = metrics.roc_auc_score(RESULTS[TARGET], RESULTS[current_strategy])
 final_df.append([current_strategy, auc_score])
 final_df = pd.DataFrame(final_df, columns=['Stategy', 'Result'])
 final_df.sort_values(by=['Result'], ascending=False, inplace=True)
 print(final_df )
cv = StratifiedKFold(n_splits=10, shuffle=True, random_state=1)
 model_xgbc = XGBClassifier(learning_rate=0.01,
n_estimators=1800,
max_depth=7,
objective= 'binary:logistic', subsample=0.5,
min_split_loss=1)
 score_xgbc = compute_score(model_xgbc, X=X, y=y, cv=cv)
 model_xgbc.fit(X, y)
 predictions_xgbc = model_xgbc.predict(test_df)
 output_xgbc = pd.DataFrame({'PassengerId': test_df2['PassengerId'], 'Survived': predictions_xgbc})
 output_xgbc.to_csv('xgboost_faretype_new.csv', index=False)
 print(score_xgbc )<save_to_csv>
print('
print('LBO full set training...')
 train_df['DT_M'] = train_df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 train_df['DT_M'] =(train_df['DT_M'].dt.year-2017)*12 + train_df['DT_M'].dt.month
main_train_set = train_df[train_df['DT_M']<(train_df['DT_M'].max())].reset_index(drop=True)
 validation_set = train_df[train_df['DT_M']==train_df['DT_M'].max() ].reset_index(drop=True)
 X,y = main_train_set[features_columns], main_train_set[TARGET]
v_X, v_y = validation_set[features_columns], validation_set[TARGET]
estimators_bestround = []
for current_model in range(3):
print('Model:',current_model+1)
 SEED += 1
seed_everything(SEED)
 corrected_lgb_params = lgb_params.copy()
corrected_lgb_params['seed'] = SEED
train_data = lgb.Dataset(X, label=y)
 valid_data = lgb.Dataset(v_X, label=v_y)
 estimator = lgb.train(
 corrected_lgb_params,
train_data,
valid_sets = [train_data, valid_data],
verbose_eval = 1000, 
)
 estimators_bestround.append(estimator.current_iteration())
 corrected_lgb_params = lgb_params.copy()
corrected_lgb_params['n_estimators'] = int(np.mean(estimators_bestround))
corrected_lgb_params['early_stopping_rounds'] = None
print('
print('Mean Best round:', corrected_lgb_params['n_estimators'])
 X,y = train_df[features_columns], train_df[TARGET]
P = test_df[features_columns]
RESULTS['lbo_full'] = 0
NUMBER_OF_MODELS = 3
for current_model in range(NUMBER_OF_MODELS):
print('Model:',current_model+1)
 SEED += 1
seed_everything(SEED)
 train_data = lgb.Dataset(X, label=y)
 estimator = lgb.train(
 corrected_lgb_params,
train_data 
)
 RESULTS['lbo_full'] += estimator.predict(P)/NUMBER_OF_MODELS
print('AUC score', metrics.roc_auc_score(RESULTS[TARGET], RESULTS['lbo_full']))
print('
cv = StratifiedKFold(n_splits=10, shuffle=True, random_state=1)
 model_rf = RandomForestClassifier(n_estimators=1000, max_features='sqrt', max_depth=4)
 score_rf = compute_score(model_rf, X=X, y=y, cv=cv)
 model_rf.fit(X, y)
 predictions_rf = model_rf.predict(test_df)
 output_rf = pd.DataFrame({'PassengerId': test_df2['PassengerId'], 'Survived': predictions_rf})
 output_rf.to_csv('random_forest_faretype_new.csv', index=False)
 print(score_rf )<load_from_csv>
print('
print('Intermediate results...')
 final_df = []
for current_strategy in list(RESULTS.iloc[:,2:]):
auc_score = metrics.roc_auc_score(RESULTS[TARGET], RESULTS[current_strategy])
 final_df.append([current_strategy, auc_score])
 final_df = pd.DataFrame(final_df, columns=['Stategy', 'Result'])
 final_df.sort_values(by=['Result'], ascending=False, inplace=True)
 print(final_df )
data = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv' )<count_missing_values>
test_df['DT_W'] = test_df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 RESULTS['DT_W'] =(test_df['DT_W'].dt.year-2017)*52 + test_df['DT_W'].dt.weekofyear
for curent_time_block in range(RESULTS['DT_W'].min() , RESULTS['DT_W'].max() +1):
print('
print('Time Block:', curent_time_block)
 final_df = []
temp_df = RESULTS[RESULTS['DT_W']==curent_time_block]
for current_strategy in list(temp_df.iloc[:,2:]):
auc_score = metrics.roc_auc_score(temp_df[TARGET], temp_df[current_strategy])
 final_df.append([current_strategy, auc_score])
 final_df = pd.DataFrame(final_df, columns=['Stategy', 'Result'])
 final_df.sort_values(by=['Result'], ascending=False, inplace=True)
 print(final_df)
 print('
<count_missing_values><EOS> 
print('
print('Small bonus')
 test_df = pd.read_pickle('.. /input/ieee-data-minification/test_transaction.pkl')
 kernel_with_identity = pd.read_csv('.. /input/ieee-gb-2-make-amount-useful-again/submission.csv')
 kernel_no_identity = pd.read_csv('.. /input/ieee-experimental/submission.csv')
 test_df = test_df[['TransactionID']]
test_df['isFraud'] = kernel_with_identity['isFraud'] + kernel_no_identity['isFraud']
test_df[['TransactionID','isFraud']].to_csv('submission.csv', index=False )