Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<define_search_model>
warnings.filterwarnings('ignore')
 sns.set_style('darkgrid')
 pd.options.display.float_format = '{:,.3f}'.format
print(os.listdir(".. /input"))
def BatchActivate(x):
x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 return x
def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True):
x = Conv2D(filters, size, strides=strides, padding=padding )(x)
 if activation == True:
x = BatchActivate(x)
 return x
def residual_block(blockInput, num_filters=16, batch_activate = False):
x = BatchActivate(blockInput)
 x = convolution_block(x, num_filters,(3,3))
x = convolution_block(x, num_filters,(3,3), activation=False)
 x = Add()([x, blockInput])
 if batch_activate:
x = BatchActivate(x)
 return x<choose_model_class>
%%time
train_id = pd.read_csv('.. /input/train_identity.csv')
 train_trn = pd.read_csv('.. /input/train_transaction.csv')
 test_id = pd.read_csv('.. /input/test_identity.csv')
 test_trn = pd.read_csv('.. /input/test_transaction.csv' )
size =(3, 3)
 def build_model(input_layer, start_neurons, DropoutRatio = 0.5):
conv1 = Conv2D(start_neurons * 1, size, activation="relu", padding="same" )(input_layer)
 conv1 = residual_block(conv1,start_neurons * 1)
 conv1 = residual_block(conv1,start_neurons * 1, True)
 pool1 = MaxPooling2D(( 2, 2))(conv1)
 pool1 = Dropout(DropoutRatio/2 )(pool1)
 conv2 = Conv2D(start_neurons * 2, size, activation=None, padding="same" )(pool1)
 conv2 = residual_block(conv2,start_neurons * 2)
 conv2 = residual_block(conv2,start_neurons * 2, True)
 pool2 = MaxPooling2D(( 2, 2))(conv2)
 pool2 = Dropout(DropoutRatio )(pool2)
 conv3 = Conv2D(start_neurons * 4, size, activation=None, padding="same" )(pool2)
 conv3 = residual_block(conv3,start_neurons * 4)
 conv3 = residual_block(conv3,start_neurons * 4, True)
 pool3 = MaxPooling2D(( 2, 2))(conv3)
 pool3 = Dropout(DropoutRatio )(pool3)
 conv4 = Conv2D(start_neurons * 8, size, activation=None, padding="same" )(pool3)
 conv4 = residual_block(conv4,start_neurons * 8)
 conv4 = residual_block(conv4,start_neurons * 8, True)
 pool4 = MaxPooling2D(( 2, 2))(conv4)
 pool4 = Dropout(DropoutRatio )(pool4)
 convm = Conv2D(start_neurons * 16, size, activation=None, padding="same" )(pool4)
 convm = residual_block(convm,start_neurons * 16)
 convm = residual_block(convm,start_neurons * 16, True)
 deconv4 = Conv2DTranspose(start_neurons * 8, size, strides=(2, 2), padding="same" )(convm)
 uconv4 = concatenate([deconv4, conv4])
 uconv4 = Dropout(DropoutRatio )(uconv4)
 uconv4 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(uconv4)
 uconv4 = residual_block(uconv4,start_neurons * 8)
 uconv4 = residual_block(uconv4,start_neurons * 8, True)
 deconv3 = Conv2DTranspose(start_neurons * 4, size, strides=(2, 2), padding="valid" )(uconv4)
 uconv3 = concatenate([deconv3, conv3])
 uconv3 = Dropout(DropoutRatio )(uconv3)
 uconv3 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(uconv3)
 uconv3 = residual_block(uconv3,start_neurons * 4)
 uconv3 = residual_block(uconv3,start_neurons * 4, True)
 deconv2 = Conv2DTranspose(start_neurons * 2, size, strides=(2, 2), padding="same" )(uconv3)
 uconv2 = concatenate([deconv2, conv2])
 uconv2 = Dropout(DropoutRatio )(uconv2)
 uconv2 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(uconv2)
 uconv2 = residual_block(uconv2,start_neurons * 2)
 uconv2 = residual_block(uconv2,start_neurons * 2, True)
 deconv1 = Conv2DTranspose(start_neurons * 1, size, strides=(2, 2), padding="valid" )(uconv2)
 uconv1 = concatenate([deconv1, conv1])
 uconv1 = Dropout(DropoutRatio )(uconv1)
 uconv1 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(uconv1)
 uconv1 = residual_block(uconv1,start_neurons * 1)
 uconv1 = residual_block(uconv1,start_neurons * 1, True)
 output_layer_noActi = Conv2D(1,(1,1), padding="same", activation=None )(uconv1)
 output_layer = Activation('sigmoid' )(output_layer_noActi)
 return output_layer<compute_test_metric>
train_id = reduce_mem_usage(train_id)
 train_trn = reduce_mem_usage(train_trn)
 test_id = reduce_mem_usage(test_id)
 test_trn = reduce_mem_usage(test_trn )
def get_iou_vector(A, B):
batch_size = A.shape[0]
metric = []
for batch in range(batch_size):
t, p = A[batch]>0, B[batch]>0
intersection = np.logical_and(t, p)
 union = np.logical_or(t, p)
 iou =(np.sum(intersection > 0)+ 1e-10)/(np.sum(union > 0)+ 1e-10)
 thresholds = np.arange(0.5, 1, 0.05)
 s = []
for thresh in thresholds:
s.append(iou > thresh)
 metric.append(np.mean(s))
return np.mean(metric)
 def my_iou_metric(label, pred):
return tf.py_func(get_iou_vector, [label, pred > 0.3], tf.float64)
 def my_iou_metric_2(label, pred):
return tf.py_func(get_iou_vector, [label, pred > 0.1], tf.float64)
 def my_iou_metric_3(label, pred):
return tf.py_func(get_iou_vector, [label, pred > 0.3], tf.float64 )<compute_train_metric>
[c for c in train_trn.columns if c not in test_trn.columns]
def lovasz_grad(gt_sorted):

 gts = tf.reduce_sum(gt_sorted)
 intersection = gts - tf.cumsum(gt_sorted)
 union = gts + tf.cumsum(1.- gt_sorted)
 jaccard = 1.- intersection / union
jaccard = tf.concat(( jaccard[0:1], jaccard[1:] - jaccard[:-1]), 0)
 return jaccard
def lovasz_hinge(logits, labels, per_image=True, ignore=None):

 if per_image:
def treat_image(log_lab):
log, lab = log_lab
log, lab = tf.expand_dims(log, 0), tf.expand_dims(lab, 0)
 log, lab = flatten_binary_scores(log, lab, ignore)
 return lovasz_hinge_flat(log, lab)
 losses = tf.map_fn(treat_image,(logits, labels), dtype=tf.float32)
 loss = tf.reduce_mean(losses)
 else:
loss = lovasz_hinge_flat(*flatten_binary_scores(logits, labels, ignore))
return loss
def lovasz_hinge_flat(logits, labels):

 def compute_loss() :
labelsf = tf.cast(labels, logits.dtype)
 signs = 2.* labelsf - 1.
 errors = 1.- logits * tf.stop_gradient(signs)
 errors_sorted, perm = tf.nn.top_k(errors, k=tf.shape(errors)[0], name="descending_sort")
 gt_sorted = tf.gather(labelsf, perm)
 grad = lovasz_grad(gt_sorted)
 loss = tf.tensordot(tf.nn.relu(errors_sorted), tf.stop_gradient(grad), 1, name="loss_non_void")
 return loss
loss = tf.cond(tf.equal(tf.shape(logits)[0], 0),
lambda: tf.reduce_sum(logits)* 0.,
compute_loss,
strict=True,
name="loss" 
)
 return loss
def flatten_binary_scores(scores, labels, ignore=None):

 scores = tf.reshape(scores,(-1,))
labels = tf.reshape(labels,(-1,))
if ignore is None:
return scores, labels
valid = tf.not_equal(labels, ignore)
 vscores = tf.boolean_mask(scores, valid, name='valid_scores')
 vlabels = tf.boolean_mask(labels, valid, name='valid_labels')
 return vscores, vlabels
def lovasz_loss(y_true, y_pred):
y_true, y_pred = K.cast(K.squeeze(y_true, -1), 'int32'), K.cast(K.squeeze(y_pred, -1), 'float32')
 logits = y_pred
loss = lovasz_hinge(logits, y_true, per_image = True, ignore = None)
 return loss<choose_model_class>
START_DATE = '2017-11-30'
startdate = datetime.datetime.strptime(START_DATE, "%Y-%m-%d")
 train_trn['Date'] = train_trn['TransactionDT'].apply(lambda x:(startdate + datetime.timedelta(seconds=x)))
 train_trn['_ymd'] = train_trn['Date'].dt.year.astype(str)+ '-' + train_trn['Date'].dt.month.astype(str)+ '-' + train_trn['Date'].dt.day.astype(str)
 train_trn['_year_month'] = train_trn['Date'].dt.year.astype(str)+ '-' + train_trn['Date'].dt.month.astype(str)
 train_trn['_weekday'] = train_trn['Date'].dt.dayofweek
train_trn['_hour'] = train_trn['Date'].dt.hour
train_trn['_day'] = train_trn['Date'].dt.day
fig,ax = plt.subplots(4, 1, figsize=(16,12))
train_trn.groupby('_weekday')['isFraud'].mean().to_frame().plot.bar(ax=ax[0])
 train_trn.groupby('_hour')['isFraud'].mean().to_frame().plot.bar(ax=ax[1])
 train_trn.groupby('_day')['isFraud'].mean().to_frame().plot.bar(ax=ax[2])
 train_trn.groupby('_year_month')['isFraud'].mean().to_frame().plot.bar(ax=ax[3] )
def _bn_relu(input):

 norm = BatchNormalization(axis=CHANNEL_AXIS )(input)
 return Activation("relu" )(norm)
 def _conv_bn_relu(**conv_params):

 filters = conv_params["filters"]
kernel_size = conv_params["kernel_size"]
strides = conv_params.setdefault("strides",(1, 1))
kernel_initializer = conv_params.setdefault("kernel_initializer", "he_normal")
 padding = conv_params.setdefault("padding", "same")
 kernel_regularizer = conv_params.setdefault("kernel_regularizer", l2(1.e-4))
def f(input):
conv = Conv2D(filters=filters, kernel_size=kernel_size,
strides=strides, padding=padding,
kernel_initializer=kernel_initializer,
kernel_regularizer=kernel_regularizer )(input)
 return _bn_relu(conv)
 return f
def _bn_relu_conv(**conv_params):

 filters = conv_params["filters"]
kernel_size = conv_params["kernel_size"]
strides = conv_params.setdefault("strides",(1, 1))
kernel_initializer = conv_params.setdefault("kernel_initializer", "he_normal")
 padding = conv_params.setdefault("padding", "same")
 kernel_regularizer = conv_params.setdefault("kernel_regularizer", l2(1.e-4))
def f(input):
activation = _bn_relu(input)
 return Conv2D(filters=filters, kernel_size=kernel_size,
strides=strides, padding=padding,
kernel_initializer=kernel_initializer,
kernel_regularizer=kernel_regularizer )(activation)
 return f
def _shortcut(input, residual):

 input_shape = K.int_shape(input)
 residual_shape = K.int_shape(residual)
 stride_width = int(round(input_shape[ROW_AXIS] / residual_shape[ROW_AXIS]))
stride_height = int(round(input_shape[COL_AXIS] / residual_shape[COL_AXIS]))
equal_channels = input_shape[CHANNEL_AXIS] == residual_shape[CHANNEL_AXIS]
shortcut = input
if stride_width > 1 or stride_height > 1 or not equal_channels:
shortcut = Conv2D(filters=residual_shape[CHANNEL_AXIS],
kernel_size=(1, 1),
strides=(stride_width, stride_height),
padding="valid",
kernel_initializer="he_normal",
kernel_regularizer=l2(0.0001))(input)
 return add([shortcut, residual])
 def basic_block(filters, init_strides=(1, 1), is_first_block_of_first_layer=False):

 def f(input):
if is_first_block_of_first_layer:
conv1 = Conv2D(filters=filters, kernel_size=(3, 3),
strides=init_strides,
padding="same",
kernel_initializer="he_normal",
kernel_regularizer=l2(1e-4))(input)
 else:
conv1 = _bn_relu_conv(filters=filters, kernel_size=(3, 3),
strides=init_strides )(input)
 residual = _bn_relu_conv(filters=filters, kernel_size=(3, 3))(conv1)
 return _shortcut(input, residual)
 return f
def _residual_block(block_function, filters, repetitions, is_first_layer=False):

 def f(input):
for i in range(repetitions):
init_strides =(1, 1)
 if i == 0 and not is_first_layer:
init_strides =(2, 2)
 input = block_function(filters=filters, init_strides=init_strides,
is_first_block_of_first_layer=(is_first_layer and i == 0))(input)
 return input
return f
def _handle_dim_ordering() :
global ROW_AXIS
global COL_AXIS
global CHANNEL_AXIS
if K.image_dim_ordering() == 'tf':
ROW_AXIS = 1
COL_AXIS = 2
CHANNEL_AXIS = 3
else:
CHANNEL_AXIS = 1
ROW_AXIS = 2
COL_AXIS = 3
def _get_block(identifier):
if isinstance(identifier, six.string_types):
res = globals().get(identifier)
 if not res:
raise ValueError('Invalid {}'.format(identifier))
return res
return identifier
class ResnetBuilder(object):
@staticmethod
def build(input_shape, block_fn, repetitions,input_tensor):
_handle_dim_ordering()
if len(input_shape)!= 3:
raise Exception("Input shape should be a tuple(nb_channels, nb_rows, nb_cols)")
 if K.image_dim_ordering() == 'tf':
input_shape =(input_shape[1], input_shape[2], input_shape[0])
 block_fn = _get_block(block_fn)
 if input_tensor is None:
img_input = Input(shape=input_shape)
 else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
 else:
img_input = input_tensor
conv1 = _conv_bn_relu(filters=64, kernel_size=(7, 7), strides=(2, 2))(img_input)
 pool1 = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding="same" )(conv1)
 block = pool1
filters = 64
for i, r in enumerate(repetitions):
block = _residual_block(block_fn, filters=filters, repetitions=r, is_first_layer=(i == 0))(block)
 filters *= 2
block = _bn_relu(block)
 model = Model(inputs=img_input, outputs=block)
 return model
@staticmethod
def build_resnet_34(input_shape,input_tensor):
return ResnetBuilder.build(input_shape, basic_block, [3, 4, 6, 3],input_tensor )<choose_model_class>
df = train_trn.groupby(['_ymd'])['isFraud'].agg(['count','mean','sum'])
 df.sort_values(by='mean',ascending=False)[:10].T
def UResNet34(input_shape=(128, 128, 1), classes=1, decoder_filters=16, decoder_block_type='upsampling',
encoder_weights="imagenet", input_tensor=None, activation='sigmoid', **kwargs):
backbone = ResnetBuilder.build_resnet_34(input_shape=input_shape,input_tensor=input_tensor)
 input_layer = backbone.input
output_layer = build_model(input_layer, 16,0.5)
 model = Model(input_layer, output_layer)
 c = optimizers.adam(lr = 0.01)
 model.compile(loss="binary_crossentropy", optimizer=c, metrics=[my_iou_metric])
 model.name = 'u-resnet34'
return model<choose_model_class>
df.sort_values(by='count',ascending=False)[:10].T
model1 = UResNet34(input_shape =(1,img_size_target,img_size_target)) 
<concatenate>
train_trn['_weekday_hour'] = train_trn['_weekday'].astype(str)+ '_' + train_trn['_hour'].astype(str)
 train_trn.groupby('_weekday_hour')['isFraud'].mean().to_frame().plot.line(figsize=(16,3))
x_train = np.append(x_train, [np.fliplr(x)for x in x_train], axis=0)
 y_train = np.append(y_train, [np.fliplr(x)for x in y_train], axis=0 )<train_model>
df = train_trn.groupby('_weekday')['isFraud'].mean().to_frame()
df.sort_values(by='isFraud', ascending=False )
model_checkpoint = ModelCheckpoint(save_model_name,monitor='my_iou_metric',
mode = 'max', save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='my_iou_metric', mode = 'max',factor=0.5, patience=5, min_lr=0.0001, verbose=1)
 history = model1.fit(x_train, y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[ model_checkpoint,reduce_lr],
verbose=1 )<choose_model_class>
df = train_trn.groupby('_hour')['isFraud'].mean().to_frame()
df.sort_values(by='isFraud', ascending=False ).head(10 )
model1 = load_model(save_model_name,custom_objects={'my_iou_metric': my_iou_metric})
 input_x = model1.layers[0].input
output_layer = model1.layers[-1].input
model = Model(input_x, output_layer)
 c = optimizers.adam(lr = 0.01)
 model.compile(loss=lovasz_loss, optimizer=c, metrics=[my_iou_metric_2])
<train_model>
fraud_id = train_trn[train_trn['isFraud'] == 1]['TransactionID']
fraud_id_in_trn = [i for i in fraud_id if i in train_id['TransactionID'].values]
print(f'fraud data count:{len(fraud_id)}, and in trn:{len(fraud_id_in_trn)}' )
early_stopping = EarlyStopping(monitor='val_my_iou_metric_2', mode = 'max',patience=20, verbose=1)
 model_checkpoint = ModelCheckpoint(save_model_name,monitor='val_my_iou_metric_2',
mode = 'max', save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='val_my_iou_metric_2', mode = 'max',factor=0.5, patience=5, min_lr=0.0001, verbose=1)
 history = model.fit(x_train, y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[model_checkpoint,reduce_lr,early_stopping],
verbose=1 )<load_pretrained>
numid_cols = [f'id_{str(i ).zfill(2)}' for i in range(1,12)]
train_id_trn[numid_cols].isna().sum().to_frame().T / len(train_id )
model = load_model(save_model_name,custom_objects={'my_iou_metric_2': my_iou_metric_2,
'lovasz_loss': lovasz_loss} )<predict_on_test>
plotCategoryRateBar('id_19', 20)
 plotHistByFraud('id_19')
 print('unique count:', train_id['id_19'].nunique() )
def predict_result(model,x_test,img_size_target):
x_test_reflect = np.array([np.fliplr(x)for x in x_test])
 preds_test = model.predict(x_test ).reshape(-1, img_size_target, img_size_target)
 preds_test2_refect = model.predict(x_test_reflect ).reshape(-1, img_size_target, img_size_target)
 preds_test += np.array([ np.fliplr(x)for x in preds_test2_refect])
 return preds_test/2<predict_on_test>
plotCategoryRateBar('id_20', 20)
 plotHistByFraud('id_20')
 print('unique count:', train_id['id_20'].nunique() )
preds_valid = predict_result(model,x_valid,img_size_target )<compute_train_metric>
ccols = [f'C{i}' for i in range(1,15)]
dcols = [f'D{i}' for i in range(1,16)]
mcols = [f'M{i}' for i in range(1,10)]
vcols = [f'V{i}' for i in range(1,340)]
thresholds_ori = np.linspace(0.3, 0.7, 31)
 thresholds = np.log(thresholds_ori/(1-thresholds_ori))
ious = np.array([iou_metric_batch(y_valid, preds_valid > threshold)for threshold in tqdm_notebook(thresholds)])
 print(ious )<categorify>
START_DATE = '2017-12-01'
startdate = datetime.datetime.strptime(START_DATE, "%Y-%m-%d")
 train_date = train_trn['TransactionDT'].apply(lambda x:(startdate + datetime.timedelta(seconds=x)))
 test_date = test_trn['TransactionDT'].apply(lambda x:(startdate + datetime.timedelta(seconds=x)))
 print('train date:', train_date.min() , '-', train_date.max())
 print('test date:', test_date.min() , '-', test_date.max() )
def rle_encode(im):

 pixels = im.flatten(order = 'F')
 pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<predict_on_test>
def appendLagDT(df):
df = df.assign(_date_lag = df['TransactionDT'] - df.groupby(['card1','card2'])['TransactionDT'].shift(1))
return df
train_trn = appendLagDT(train_trn)
 train_trn_f0 = train_trn[train_trn['isFraud'] == 0]
train_trn_f1 = train_trn[train_trn['isFraud'] == 1]
preds_test = predict_result(model,x_test,img_size_target )<compute_test_metric>
def appendLagAmt(df):
df = df.assign(_amt_lag = df['TransactionAmt'] - df.groupby(['card1','card2'])['TransactionAmt'].shift(1))
df['_amt_lag_sig'] = df['_amt_lag'].apply(lambda x: '0' if np.isnan(x)else '+' if x >=0 else '-')
 return df
train_trn = appendLagAmt(train_trn)
 train_trn_f0 = train_trn[train_trn['isFraud'] == 0]
train_trn_f1 = train_trn[train_trn['isFraud'] == 1]
t1 = time.time()
pred_dict = {idx: rle_encode(np.round(downsample(preds_test[i])> threshold_best)) for i, idx in enumerate(tqdm_notebook(test_df.index.values)) }
t2 = time.time()
print(f"Usedtime = {t2-t1} s" )<save_to_csv>
cols = ['addr1','addr2','dist1','dist2']
for f in cols:
train_trn[f + '_isna'] = train_trn[f].isna()
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv(submission_file )<train_model>
train_trn[(train_trn['dist1_isna'] == False)&(train_trn['dist2_isna'] == False)][cols]
t_finish = time.time()
print(f"Kernel run time = {(t_finish-t_start)/3600} hours" )<define_search_space>
train_trn = pd.concat([train_trn, pd.get_dummies(train_trn[['ProductCD']])], axis=1)
 train_trn.head(5 )
split_num=3
split_index=[0,1,3]<categorify>
train_trn['_amount_max_ProductCD'] = train_trn.groupby(['ProductCD'])['TransactionAmt'].transform('max')
 train_trn[['ProductCD','_amount_max_ProductCD']].drop_duplicates().sort_values(by='_amount_max_ProductCD', ascending=False )
def rle_decode(rle_mask):

 if type(rle_mask)==float:
return np.zeros([101,101])
 s = rle_mask.split()
starts, lengths = [np.asarray(x, dtype=int)for x in(s[0:][::2], s[1:][::2])]
starts -= 1
ends = starts + lengths
img = np.zeros(101*101, dtype=np.uint8)
 for lo, hi in zip(starts, ends):
img[lo:hi] = 1
return img.reshape(101,101)

def rle_encode(im):

 pixels = im.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<load_from_csv>
cols = [f'card{n}' for n in range(1,7)]
train_trn[cols].isnull().sum()
test_df=pd.read_csv(".. /input/tgs-salt-identification-challenge/sample_submission.csv",index_col='id' )<load_from_csv>
train_trn[cols].nunique()
test_img=np.zeros([18000,101,101])
 for fold in split_index:
test_fold=pd.read_csv(".. /input/fork-of-result-visualization-split-"+str(fold)+"/submission.csv",index_col='id')
 for index,ids in enumerate(test_fold.index):
img=rle_decode(test_fold.loc[ids,'rle_mask'])
 test_img[index]+=img/split_num
test_img=np.round(test_img )<load_from_csv>
train_trn_f0['_card1_card2'] = train_trn_f0['card1'].astype(str)+ '_' + train_trn_f0['card2'].astype(str)
 train_trn_f1['_card1_card2'] = train_trn_f1['card1'].astype(str)+ '_' + train_trn_f1['card2'].astype(str)
 plotTrnCategoryRateBar('_card1_card2', 50, figsize=(15,3))
test_fold=pd.read_csv(".. /input/fork-of-result-visualization-split-0/submission.csv",index_col='id' )<categorify>
print(len(train_trn))
print(train_trn['card1'].nunique() , train_trn['card2'].nunique() , train_trn['card3'].nunique() , train_trn['card5'].nunique())
 train_trn['card_n'] =(train_trn['card1'].astype(str)+ '_' + train_trn['card2'].astype(str)\
+ '_' + train_trn['card3'].astype(str)+ '_' + train_trn['card5'].astype(str))
print('unique cards:', train_trn['card_n'].nunique() )
for index,ids in enumerate(test_df.index):
test_df.loc[ids,'rle_mask']=rle_encode(test_img[index])
<save_to_csv>
train_trn.groupby(['card_n'])['isFraud'].mean().sort_values(ascending=False )
test_df.to_csv('submission.csv' )<set_options>
train_trn['addr1'].nunique() , train_trn['addr2'].nunique()
plt.style.use('seaborn-white')
 sns.set_style("white")
<normalization>
train_trn['addr1'].value_counts(dropna=False ).to_frame().iloc[:10]
img_size_ori = 101
img_size_target = 128
def upsample_reflect(img):
if img_size_ori == img_size_target:
return img
return np.pad(img[:,:,0],(( 13,14),(13,14)) ,'reflect')
 def upsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_target, img_size_target), mode='constant', preserve_range=True)
 def downsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_ori, img_size_ori), mode='constant', preserve_range=True)
 def iouMetric(true, pred):
true = K.batch_flatten(true)
 pred = K.batch_flatten(pred)
 pred = K.cast(K.greater(pred, 0.5), K.floatx())
 intersec = true * pred
iou = K.sum(intersec)/(K.sum(true + pred - intersec)+ K.epsilon())
 return iou<compute_test_metric>
plotTrnCategoryRateBar('addr2', 10)
 print('addr2 nunique:', train_trn['addr2'].nunique() )
def dice_coef(y_true, y_pred):
y_true_f = K.flatten(y_true)
 y_pred = K.cast(y_pred, 'float32')
 y_pred_f = K.cast(K.greater(K.flatten(y_pred), 0.5), 'float32')
 intersection = y_true_f * y_pred_f
score = 2.* K.sum(intersection)/(K.sum(y_true_f)+ K.sum(y_pred_f))
return score
def dice_loss(y_true, y_pred):
smooth = 1.
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = y_true_f * y_pred_f
score =(2.* K.sum(intersection)+ smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 return 1.- score
def bce_dice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)+ dice_loss(y_true, y_pred)
 def bce_logdice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)- K.log(1.- dice_loss(y_true, y_pred))
def weighted_bce_loss(y_true, y_pred, weight):
epsilon = 1e-7
y_pred = K.clip(y_pred, epsilon, 1.- epsilon)
 logit_y_pred = K.log(y_pred /(1.- y_pred))
loss = weight *(logit_y_pred *(1.- y_true)+
K.log(1.+ K.exp(-K.abs(logit_y_pred)))+ K.maximum(-logit_y_pred, 0.))
 return K.sum(loss)/ K.sum(weight)
 def weighted_dice_loss(y_true, y_pred, weight):
smooth = 1.
 w, m1, m2 = weight, y_true, y_pred
intersection =(m1 * m2)
 score =(2.* K.sum(w * intersection)+ smooth)/(K.sum(w * m1)+ K.sum(w * m2)+ smooth)
 loss = 1.- K.sum(score)
 return loss
def weighted_bce_dice_loss(y_true, y_pred):
y_true = K.cast(y_true, 'float32')
 y_pred = K.cast(y_pred, 'float32')
 averaged_mask = K.pool2d(
 y_true, pool_size=(50, 50), strides=(1, 1), padding='same', pool_mode='avg')
 weight = K.ones_like(averaged_mask)
 w0 = K.sum(weight)
 weight = 5.* K.exp(-5.* K.abs(averaged_mask - 0.5))
w1 = K.sum(weight)
 weight *=(w0 / w1)
 loss = weighted_bce_loss(y_true, y_pred, weight)+ dice_loss(y_true, y_pred)
 return loss<normalization>
train_trn['addr2'].value_counts(dropna=False ).to_frame().iloc[:10]
def do_resize2(image, mask, H, W):
image = cv2.resize(image,dsize=(W,H))
mask = cv2.resize(mask,dsize=(W,H))
mask =(mask>0.5 ).astype(np.float32)
 return image,mask
def compute_center_pad(H,W, factor=32):
if H%factor==0:
dy0,dy1=0,0
else:
dy = factor - H%factor
dy0 = dy//2
dy1 = dy - dy0
if W%factor==0:
dx0,dx1=0,0
else:
dx = factor - W%factor
dx0 = dx//2
dx1 = dx - dx0
return dy0, dy1, dx0, dx1
def do_center_pad_to_factor(image, factor=32):
H,W = image.shape[:2]
dy0, dy1, dx0, dx1 = compute_center_pad(H,W, factor)
 image = cv2.copyMakeBorder(image, dy0, dy1, dx0, dx1, cv2.BORDER_REFLECT_101)
 return image
def do_center_pad_to_factor2(image, mask, factor=32):
image = do_center_pad_to_factor(image, factor)
 mask = do_center_pad_to_factor(mask, factor)
 return image, mask
def do_horizontal_flip(image):
image = cv2.flip(image,1)
 return image
def do_horizontal_flip2(image,mask):
image = do_horizontal_flip(image)
 mask = do_horizontal_flip(mask)
 return image, mask
def compute_random_pad(H,W, limit=(-4,4), factor=32):
if H%factor==0:
dy0,dy1=0,0
else:
dy = factor - H%factor
dy0 = dy//2 + np.random.randint(limit[0],limit[1])
 dy1 = dy - dy0
if W%factor==0:
dx0,dx1=0,0
else:
dx = factor - W%factor
dx0 = dx//2 + np.random.randint(limit[0],limit[1])
 dx1 = dx - dx0
return dy0, dy1, dx0, dx1
def do_random_pad_to_factor2(image, mask, limit=(-4,4), factor=32):
H,W = image.shape[:2]
dy0, dy1, dx0, dx1 = compute_random_pad(H,W, limit, factor)
 image = cv2.copyMakeBorder(image, dy0, dy1, dx0, dx1, cv2.BORDER_REFLECT_101)
 mask = cv2.copyMakeBorder(mask, dy0, dy1, dx0, dx1, cv2.BORDER_REFLECT_101)
 return image, mask
def do_invert_intensity(image):
image = np.clip(1-image,0,1)
 return image
def do_brightness_shift(image, alpha=0.125):
image = image + alpha
image = np.clip(image, 0, 1)
 return image
def do_brightness_multiply(image, alpha=1):
image = alpha*image
image = np.clip(image, 0, 1)
 return image
def do_gamma(image, gamma=1.0):
image = image **(1.0 / gamma)
 image = np.clip(image, 0, 1)
 return image
def do_flip_transpose2(image, mask, type=0):
if type==1:
image = image.transpose(1,0)
 image = cv2.flip(image,1)
 mask = mask.transpose(1,0)
 mask = cv2.flip(mask,1)
 if type==2:
image = cv2.flip(image,-1)
 mask = cv2.flip(mask,-1)
 if type==3:
image = image.transpose(1,0)
 image = cv2.flip(image,0)
 mask = mask.transpose(1,0)
 mask = cv2.flip(mask,0)
 if type==4:
image = cv2.flip(image,1)
 mask = cv2.flip(mask,1)
 if type==5:
image = cv2.flip(image,0)
 mask = cv2.flip(mask,0)
 if type==6:
image = cv2.flip(image,1)
 image = image.transpose(1,0)
 image = cv2.flip(image,1)
 mask = cv2.flip(mask,1)
 mask = mask.transpose(1,0)
 mask = cv2.flip(mask,1)
 if type==7:
image = cv2.flip(image,0)
 image = image.transpose(1,0)
 image = cv2.flip(image,1)
 mask = cv2.flip(mask,0)
 mask = mask.transpose(1,0)
 mask = cv2.flip(mask,1)
 return image, mask
def do_shift_scale_crop(image, mask, x0=0, y0=0, x1=1, y1=1):
height, width = image.shape[:2]
image = image[y0:y1,x0:x1]
mask = mask [y0:y1,x0:x1]
image = cv2.resize(image,dsize=(width,height))
mask = cv2.resize(mask,dsize=(width,height))
mask =(mask>0.5 ).astype(np.float32)
 return image, mask
def do_random_shift_scale_crop_pad2(image, mask, limit=0.10):
H, W = image.shape[:2]
dy = int(H*limit)
 y0 = np.random.randint(0,dy)
 y1 = H-np.random.randint(0,dy)
 dx = int(W*limit)
 x0 = np.random.randint(0,dx)
 x1 = W-np.random.randint(0,dx)
 image, mask = do_shift_scale_crop(image, mask, x0, y0, x1, y1)
 return image, mask
def do_shift_scale_rotate2(image, mask, dx=0, dy=0, scale=1, angle=0):
borderMode=cv2.BORDER_REFLECT_101
height, width = image.shape[:2]
sx = scale
sy = scale
cc = math.cos(angle/180*math.pi)*(sx)
 ss = math.sin(angle/180*math.pi)*(sy)
 rotate_matrix = np.array([ [cc,-ss], [ss,cc] ])
 box0 = np.array([ [0,0], [width,0], [width,height], [0,height], ],np.float32)
 box1 = box0 - np.array([width/2,height/2])
 box1 = np.dot(box1,rotate_matrix.T)+ np.array([width/2+dx,height/2+dy])
 box0 = box0.astype(np.float32)
 box1 = box1.astype(np.float32)
 mat = cv2.getPerspectiveTransform(box0,box1)
 image = cv2.warpPerspective(image, mat,(width,height),flags=cv2.INTER_LINEAR,
borderMode=borderMode,borderValue=(0,0,0,))
mask = cv2.warpPerspective(mask, mat,(width,height),flags=cv2.INTER_NEAREST,
borderMode=borderMode,borderValue=(0,0,0,))
mask =(mask>0.5 ).astype(np.float32)
 return image, mask
def do_elastic_transform2(image, mask, grid=32, distort=0.2):
borderMode=cv2.BORDER_REFLECT_101
height, width = image.shape[:2]
x_step = int(grid)
 xx = np.zeros(width,np.float32)
 prev = 0
for x in range(0, width, x_step):
start = x
end = x + x_step
if end > width:
end = width
cur = width
else:
cur = prev + x_step*(1+random.uniform(-distort,distort))
xx[start:end] = np.linspace(prev,cur,end-start)
 prev=cur
y_step = int(grid)
 yy = np.zeros(height,np.float32)
 prev = 0
for y in range(0, height, y_step):
start = y
end = y + y_step
if end > height:
end = height
cur = height
else:
cur = prev + y_step*(1+random.uniform(-distort,distort))
yy[start:end] = np.linspace(prev,cur,end-start)
 prev=cur
map_x,map_y = np.meshgrid(xx, yy)
 map_x = map_x.astype(np.float32)
 map_y = map_y.astype(np.float32)
 image = cv2.remap(image, map_x, map_y, interpolation=cv2.INTER_LINEAR, borderMode=borderMode,borderValue=(0,0,0,))
mask = cv2.remap(mask, map_x, map_y, interpolation=cv2.INTER_NEAREST, borderMode=borderMode,borderValue=(0,0,0,))
mask =(mask>0.5 ).astype(np.float32)
 return image, mask
def do_horizontal_shear2(image, mask, dx=0):
borderMode=cv2.BORDER_REFLECT_101
height, width = image.shape[:2]
dx = int(dx*width)
 box0 = np.array([ [0,0], [width,0], [width,height], [0,height], ],np.float32)
 box1 = np.array([ [+dx,0], [width+dx,0], [width-dx,height], [-dx,height], ],np.float32)
 box0 = box0.astype(np.float32)
 box1 = box1.astype(np.float32)
 mat = cv2.getPerspectiveTransform(box0,box1)
 image = cv2.warpPerspective(image, mat,(width,height),flags=cv2.INTER_LINEAR,
borderMode=borderMode,borderValue=(0,0,0,))
mask = cv2.warpPerspective(mask, mat,(width,height),flags=cv2.INTER_NEAREST,
borderMode=borderMode,borderValue=(0,0,0,))
mask =(mask>0.5 ).astype(np.float32)
 return image, mask<normalization>
train_trn[vcols].isnull().sum() / len(train_trn )
def train_augment(image,mask,index):
if np.random.rand() < 0.5:
image, mask = do_horizontal_flip2(image, mask)
 pass
if np.random.rand() < 0.5:
c = np.random.choice(3)
 if c==0:
image, mask = do_random_shift_scale_crop_pad2(image, mask, 0.125)
 if c==1:
image, mask = do_elastic_transform2(image, mask, grid=10,
distort=np.random.uniform(0,0.1))
if c==2:
image, mask = do_shift_scale_rotate2(image, mask, dx=0, dy=0, scale=1,
angle=np.random.uniform(0,10))
if np.random.rand() < 0.5:
c = np.random.choice(3)
 if c==0:
image = do_brightness_shift(image,np.random.uniform(-0.05,+0.05))
if c==1:
image = do_brightness_multiply(image,np.random.uniform(1-0.05,1+0.05))
if c==2:
image = do_gamma(image,np.random.uniform(1-0.05,1+0.05))
image, mask = do_center_pad_to_factor2(image, mask, factor=32)
 return image,mask,index<load_from_csv>
del train_trn_f0,train_trn_f1,train_id_f0,train_id_f1
print(pd.DataFrame([[val for val in dir() ], [sys.getsizeof(eval(val)) for val in dir() ]],
index=['name','size'] ).T.sort_values('size', ascending=False ).reset_index(drop=True)[:10] )
train_df = pd.read_csv(".. /input/train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(".. /input/depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]<feature_engineering>
train_id = pd.read_csv('.. /input/train_identity.csv')
 train_trn = pd.read_csv('.. /input/train_transaction.csv')
 test_id = pd.read_csv('.. /input/test_identity.csv')
 test_trn = pd.read_csv('.. /input/test_transaction.csv')
 id_cols = list(train_id.columns.values)
 trn_cols = list(train_trn.drop('isFraud', axis=1 ).columns.values)
 X_train = pd.merge(train_trn[trn_cols + ['isFraud']], train_id[id_cols], how='left')
 X_train = reduce_mem_usage(X_train)
 X_test = pd.merge(test_trn[trn_cols], test_id[id_cols], how='left')
 X_test = reduce_mem_usage(X_test)
 X_train_id = X_train.pop('TransactionID')
 X_test_id = X_test.pop('TransactionID')
 del train_id,train_trn,test_id,test_trn
all_data = X_train.append(X_test, sort=False ).reset_index(drop=True )
train_df["images"] = [np.array(load_img(".. /input/train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]<feature_engineering>
vcols = [f'V{i}' for i in range(1,340)]
sc = preprocessing.MinMaxScaler()
pca = PCA(n_components=2)
 vcol_pca = pca.fit_transform(sc.fit_transform(all_data[vcols].fillna(-1)))
 all_data['_vcol_pca0'] = vcol_pca[:,0]
all_data['_vcol_pca1'] = vcol_pca[:,1]
all_data['_vcol_nulls'] = all_data[vcols].isnull().sum(axis=1)
 all_data.drop(vcols, axis=1, inplace=True )
train_df["masks"] = [np.array(load_img(".. /input/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]<feature_engineering>
START_DATE = '2017-12-01'
startdate = datetime.datetime.strptime(START_DATE, "%Y-%m-%d")
 all_data['Date'] = all_data['TransactionDT'].apply(lambda x:(startdate + datetime.timedelta(seconds=x)))
 all_data['_weekday'] = all_data['Date'].dt.dayofweek
all_data['_hour'] = all_data['Date'].dt.hour
all_data['_day'] = all_data['Date'].dt.day
all_data['_weekday'] = all_data['_weekday'].astype(str)
 all_data['_hour'] = all_data['_hour'].astype(str)
 all_data['_weekday__hour'] = all_data['_weekday'] + all_data['_hour']
cnt_day = all_data['_day'].value_counts()
cnt_day = cnt_day / cnt_day.mean()
all_data['_count_rate'] = all_data['_day'].map(cnt_day.to_dict())
 all_data.drop(['TransactionDT','Date','_day'], axis=1, inplace=True )
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size_ori, 2 )<categorify>
all_data['_P_emaildomain__addr1'] = all_data['P_emaildomain'] + '__' + all_data['addr1'].astype(str)
 all_data['_card1__card2'] = all_data['card1'].astype(str)+ '__' + all_data['card2'].astype(str)
 all_data['_card1__addr1'] = all_data['card1'].astype(str)+ '__' + all_data['addr1'].astype(str)
 all_data['_card2__addr1'] = all_data['card2'].astype(str)+ '__' + all_data['addr1'].astype(str)
 all_data['_card12__addr1'] = all_data['_card1__card2'] + '__' + all_data['addr1'].astype(str)
 all_data['_card_all__addr1'] = all_data['_card1__card2'] + '__' + all_data['addr1'].astype(str )
def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class )<split>
all_data['_amount_decimal'] =(( all_data['TransactionAmt'] - all_data['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 all_data['_amount_decimal_len'] = all_data['TransactionAmt'].apply(lambda x: len(re.sub('0+$', '', str(x)).split('.')[1]))
all_data['_amount_fraction'] = all_data['TransactionAmt'].apply(lambda x: float('0.'+re.sub('^[0-9]|\.|0+$', '', str(x))))
all_data[['TransactionAmt','_amount_decimal','_amount_decimal_len','_amount_fraction']].head(10 )
ids_train, ids_valid, x_train, x_valid, y_train, y_valid, cov_train, cov_test, depth_train, depth_test = train_test_split(
 train_df.index.values,
np.array(train_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
np.array(train_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
train_df.coverage.values,
train_df.z.values,
test_size=0.2, stratify=train_df.coverage_class, random_state=1337 )<concatenate>
cols = ['ProductCD','card1','card2','card5','card6','P_emaildomain','_card_all__addr1']
for f in cols:
all_data[f'_amount_mean_{f}'] = all_data['TransactionAmt'] / all_data.groupby([f])['TransactionAmt'].transform('mean')
 all_data[f'_amount_std_{f}'] = all_data['TransactionAmt'] / all_data.groupby([f])['TransactionAmt'].transform('std')
 all_data[f'_amount_pct_{f}'] =(all_data['TransactionAmt'] - all_data[f'_amount_mean_{f}'])/ all_data[f'_amount_std_{f}']
for f in cols:
vc = all_data[f].value_counts(dropna=False)
 all_data[f'_count_{f}'] = all_data[f].map(vc )
def ConvBn2D(x,in_dim,out_dim):
x=Conv2D() 
<create_dataframe>
_=
cat_cols = [f'id_{i}' for i in range(12,39)]
for i in cat_cols:
if i in all_data.columns:
all_data[i] = all_data[i].astype(str)
 all_data[i].fillna('unknown', inplace=True)
 enc_cols = []
for i, t in all_data.loc[:, all_data.columns != 'isFraud'].dtypes.iteritems() :
if t == object:
enc_cols.append(i)
 all_data[i] = pd.factorize(all_data[i])[0]
print(enc_cols )
model = UNet(( img_size_target,img_size_target,1),start_ch=16,depth=5,batchnorm=True )<choose_model_class>
X_train = all_data[all_data['isFraud'].notnull() ]
X_test = all_data[all_data['isFraud'].isnull() ].drop('isFraud', axis=1)
 Y_train = X_train.pop('isFraud')
 del all_data
sgd = SGD(lr=0.01, decay=1e-4, momentum=0.9, nesterov=True)
 model.compile(loss=bce_dice_loss, optimizer="adam", metrics=["accuracy",iouMetric] )<choose_model_class>
%%time
params={'learning_rate': 0.01,
'objective': 'binary',
'metric': 'auc',
'num_threads': -1,
'num_leaves': 256,
'verbose': 1,
'random_state': 42,
'bagging_fraction': 1,
'feature_fraction': 0.85
}
oof_preds = np.zeros(X_train.shape[0])
 sub_preds = np.zeros(X_test.shape[0])
 clf = lgb.LGBMClassifier(**params, n_estimators=3000)
 clf.fit(X_train, Y_train)
 oof_preds = clf.predict_proba(X_train, num_iteration=clf.best_iteration_)[:,1]
sub_preds = clf.predict_proba(X_test, num_iteration=clf.best_iteration_)[:,1]
<normalization><EOS> 
submission = pd.DataFrame()
submission['TransactionID'] = X_test_id
submission['isFraud'] = sub_preds
submission.to_csv('submission.csv', index=False )