Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<train_model>
warnings.filterwarnings('ignore' )
model_checkpoint = ModelCheckpoint("./keras.model", save_best_only=True, verbose=1)
 def step_decay(epoch):
if epoch<=60:
return 0.01
elif epoch<=90:
return 0.001
else:
return 0.0001
reduce_lr = LearningRateScheduler(step_decay)
 epochs = 120
batch_size = 32
gen = ImageDataGenerator()
gen_flow = gen_flow_for_two_inputs(x_train, y_train)
 history = model.fit_generator(gen_flow,
validation_data=[x_valid, y_valid],
epochs=epochs,
steps_per_epoch=len(x_train)/ batch_size,
callbacks=[model_checkpoint, reduce_lr] )<load_pretrained>
def seed_everything(seed=0):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
model.load_weights("./keras.model",{"bce_dice_loss":bce_dice_loss,'iouMetric':iouMetric} )<predict_on_test>
SEED = 42
seed_everything(SEED)
 LOCAL_TEST = False
TARGET = 'isFraud'
preds_valid = model.predict(x_valid ).reshape(-1, img_size_target, img_size_target)
 preds_valid = np.array([downsample(x)for x in preds_valid])
 y_valid_ori = np.array([train_df.loc[idx].masks for idx in ids_valid] )<compute_train_metric>
print('Load Data')
 train_df = pd.read_pickle('.. /input/ieee-data-minification/train_transaction.pkl')
 if LOCAL_TEST:
test_df = train_df.iloc[-100000:,].reset_index(drop=True)
 train_df = train_df.iloc[:400000,].reset_index(drop=True)
 train_identity = pd.read_pickle('.. /input/ieee-data-minification/train_identity.pkl')
 test_identity = train_identity[train_identity['TransactionID'].isin(test_df['TransactionID'])].reset_index(drop=True)
 train_identity = train_identity[train_identity['TransactionID'].isin(train_df['TransactionID'])].reset_index(drop=True)
 else:
test_df = pd.read_pickle('.. /input/ieee-data-minification/test_transaction.pkl')
 test_identity = pd.read_pickle('.. /input/ieee-data-minification/test_identity.pkl' )
thresholds = np.linspace(0, 1, 50)
 ious = np.array([iou_metric_batch(y_valid_ori, np.int32(preds_valid > threshold)) for threshold in tqdm_notebook(thresholds)] )<find_best_params>
valid_card = train_df['card1'].value_counts()
valid_card = valid_card[valid_card>10]
valid_card = list(valid_card.index)
 train_df['card1'] = np.where(train_df['card1'].isin(valid_card), train_df['card1'], np.nan)
 test_df['card1'] = np.where(test_df['card1'].isin(valid_card), test_df['card1'], np.nan )
threshold_best_index = np.argmax(ious[9:-10])+ 9
iou_best = ious[threshold_best_index]
threshold_best = thresholds[threshold_best_index]<categorify>
i_cols = ['card1','card2','card3','card5',
'C1','C2','C3','C4','C5','C6','C7','C8','C9','C10','C11','C12','C13','C14',
'D1','D2','D3','D4','D5','D6','D7','D8','D9',
'addr1','addr2',
'dist1','dist2',
'P_emaildomain', 'R_emaildomain'
]
for col in i_cols:
temp_df = pd.concat([train_df[[col]], test_df[[col]]])
 fq_encode = temp_df[col].value_counts().to_dict()
train_df[col+'_fq_enc'] = train_df[col].map(fq_encode)
 test_df[col+'_fq_enc'] = test_df[col].map(fq_encode )
def RLenc(img, order='F', format=True):

 bytes = img.reshape(img.shape[0] * img.shape[1], order=order)
 runs = []
r = 0
pos = 1
for c in bytes:
if(c == 0):
if r != 0:
runs.append(( pos, r))
pos += r
r = 0
pos += 1
else:
r += 1
if r != 0:
runs.append(( pos, r))
pos += r
r = 0
if format:
z = ''
for rr in runs:
z += '{} {} '.format(rr[0], rr[1])
 return z[:-1]
else:
return runs<predict_on_test>
for col in ['ProductCD','M4']:
temp_dict = train_df.groupby([col])[TARGET].agg(['mean'] ).reset_index().rename(
 columns={'mean': col+'_target_mean'})
 temp_dict.index = temp_dict[col].values
temp_dict = temp_dict[col+'_target_mean'].to_dict()
train_df[col+'_target_mean'] = train_df[col].map(temp_dict)
 test_df[col+'_target_mean'] = test_df[col].map(temp_dict )
preds_test = model.predict(x_test )<compute_test_metric>
for col in list(train_df):
if train_df[col].dtype=='O':
print(col)
 train_df[col] = train_df[col].fillna('unseen_before_label')
 test_df[col] = test_df[col].fillna('unseen_before_label')
 train_df[col] = train_df[col].astype(str)
 test_df[col] = test_df[col].astype(str)
 le = LabelEncoder()
le.fit(list(train_df[col])+list(test_df[col]))
train_df[col] = le.transform(train_df[col])
 test_df[col] = le.transform(test_df[col])
 train_df[col] = train_df[col].astype('category')
 test_df[col] = test_df[col].astype('category' )
pred_dict = {idx: RLenc(np.round(downsample(preds_test[i])> threshold_best)) for i, idx in enumerate(tqdm_notebook(test_df.index.values)) }<save_to_csv>
lgb_params = {
'objective':'binary',
'boosting_type':'gbdt',
'metric':'auc',
'n_jobs':-1,
'learning_rate':0.01,
'num_leaves': 2**8,
'max_depth':-1,
'tree_learner':'serial',
'colsample_bytree': 0.7,
'subsample_freq':1,
'subsample':1,
'n_estimators':800,
'max_bin':255,
'verbose':-1,
'seed': SEED,
'early_stopping_rounds':100,
}
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv('submission.csv' )<set_options>
def make_predictions(tr_df, tt_df, features_columns, target, lgb_params, NFOLDS=2):
folds = KFold(n_splits=NFOLDS, shuffle=True, random_state=SEED)
 X,y = tr_df[features_columns], tr_df[target]
P,P_y = tt_df[features_columns], tt_df[target]
tt_df = tt_df[['TransactionID',target]]
predictions = np.zeros(len(tt_df))
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X, y)) :
print('Fold:',fold_)
 tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]
vl_x, vl_y = X.iloc[val_idx,:], y[val_idx]
print(len(tr_x),len(vl_x))
tr_data = lgb.Dataset(tr_x, label=tr_y)
 if LOCAL_TEST:
vl_data = lgb.Dataset(P, label=P_y)
 else:
vl_data = lgb.Dataset(vl_x, label=vl_y)
 estimator = lgb.train(
 lgb_params,
tr_data,
valid_sets = [tr_data, vl_data],
verbose_eval = 200, 
)
 pp_p = estimator.predict(P)
 predictions += pp_p/NFOLDS
if LOCAL_TEST:
feature_imp = pd.DataFrame(sorted(zip(estimator.feature_importance() ,X.columns)) , columns=['Value','Feature'])
 print(feature_imp)
 del tr_x, tr_y, vl_x, vl_y, tr_data, vl_data
gc.collect()
tt_df['prediction'] = predictions
return tt_df
plt.style.use('seaborn-white')
 sns.set_style("white")
 %matplotlib inline
imgaug.seed(1)
 t_start = time.time()<define_variables>
if LOCAL_TEST:
test_predictions = make_predictions(train_df, test_df, features_columns, TARGET, lgb_params)
 print(metrics.roc_auc_score(test_predictions[TARGET], test_predictions['prediction']))
else:
lgb_params['learning_rate'] = 0.01
lgb_params['n_estimators'] = 800
lgb_params['early_stopping_rounds'] = 100
test_predictions = make_predictions(train_df, test_df, features_columns, TARGET, lgb_params, NFOLDS=2 )
version = 5
basic_name = f'Unet_resnet_v{version}'
save_model_name = basic_name + '.model'
submission_file = basic_name + '.csv'
print(save_model_name)
 print(submission_file )<train_model>
if not LOCAL_TEST:
test_predictions['isFraud'] = test_predictions['prediction']
test_predictions[['TransactionID','isFraud']].to_csv('submission.csv', index=False )