Kaggle competition: IEEE-CIS Fraud Detection
ids_train, ids_valid, x_train, x_valid, y_train, y_valid, cov_train, cov_test, depth_train, depth_test = train_test_split(
 train_df.index.values,
np.array(train_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
np.array(train_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
train_df.coverage.values,
train_df.z.values,
test_size=0.2, stratify=train_df.coverage_class, random_state= None )<define_search_model>
sns.set()
%matplotlib inline
warnings.filterwarnings('ignore' )
def BatchActivate(x):
x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 return x
def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True):
x = Conv2D(filters, size, strides=strides, padding=padding )(x)
 if activation == True:
x = BatchActivate(x)
 return x
def residual_block(blockInput, num_filters=16, batch_activate = False):
x = BatchActivate(blockInput)
 x = convolution_block(x, num_filters,(3,3))
x = convolution_block(x, num_filters,(3,3), activation=False)
 x = Add()([x, blockInput])
 if batch_activate:
x = BatchActivate(x)
 return x 
<choose_model_class>
folder_path = '.. /input/ieee-fraud-detection/'
train_identity = pd.read_csv(f'{folder_path}train_identity.csv', index_col='TransactionID')
 train_transaction = pd.read_csv(f'{folder_path}train_transaction.csv', index_col='TransactionID')
 test_identity = pd.read_csv(f'{folder_path}test_identity.csv', index_col='TransactionID')
 test_transaction = pd.read_csv(f'{folder_path}test_transaction.csv', index_col='TransactionID')
 sub = pd.read_csv(f'{folder_path}sample_submission.csv' )
def build_model(input_layer, start_neurons, DropoutRatio = 0.5):
conv1 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(input_layer)
 conv1 = residual_block(conv1,start_neurons * 1)
 conv1 = residual_block(conv1,start_neurons * 1, True)
 pool1 = MaxPooling2D(( 2, 2))(conv1)
 pool1 = Dropout(DropoutRatio/2 )(pool1)
 conv2 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(pool1)
 conv2 = residual_block(conv2,start_neurons * 2)
 conv2 = residual_block(conv2,start_neurons * 2, True)
 pool2 = MaxPooling2D(( 2, 2))(conv2)
 pool2 = Dropout(DropoutRatio )(pool2)
 conv3 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(pool2)
 conv3 = residual_block(conv3,start_neurons * 4)
 conv3 = residual_block(conv3,start_neurons * 4, True)
 pool3 = MaxPooling2D(( 2, 2))(conv3)
 pool3 = Dropout(DropoutRatio )(pool3)
 conv4 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(pool3)
 conv4 = residual_block(conv4,start_neurons * 8)
 conv4 = residual_block(conv4,start_neurons * 8, True)
 pool4 = MaxPooling2D(( 2, 2))(conv4)
 pool4 = Dropout(DropoutRatio )(pool4)
 convm = Conv2D(start_neurons * 16,(3, 3), activation=None, padding="same" )(pool4)
 convm = residual_block(convm,start_neurons * 16)
 convm = residual_block(convm,start_neurons * 16, True)
 deconv4 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(convm)
 uconv4 = concatenate([deconv4, conv4])
 uconv4 = Dropout(DropoutRatio )(uconv4)
 uconv4 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(uconv4)
 uconv4 = residual_block(uconv4,start_neurons * 8)
 uconv4 = residual_block(uconv4,start_neurons * 8, True)
 deconv3 = Conv2DTranspose(start_neurons * 4,(3, 3), strides=(2, 2), padding="valid" )(uconv4)
 uconv3 = concatenate([deconv3, conv3])
 uconv3 = Dropout(DropoutRatio )(uconv3)
 uconv3 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(uconv3)
 uconv3 = residual_block(uconv3,start_neurons * 4)
 uconv3 = residual_block(uconv3,start_neurons * 4, True)
 deconv2 = Conv2DTranspose(start_neurons * 2,(3, 3), strides=(2, 2), padding="same" )(uconv3)
 uconv2 = concatenate([deconv2, conv2])
 uconv2 = Dropout(DropoutRatio )(uconv2)
 uconv2 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(uconv2)
 uconv2 = residual_block(uconv2,start_neurons * 2)
 uconv2 = residual_block(uconv2,start_neurons * 2, True)
 deconv1 = Conv2DTranspose(start_neurons * 1,(3, 3), strides=(2, 2), padding="valid" )(uconv2)
 uconv1 = concatenate([deconv1, conv1])
 uconv1 = Dropout(DropoutRatio )(uconv1)
 uconv1 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(uconv1)
 uconv1 = residual_block(uconv1,start_neurons * 1)
 uconv1 = residual_block(uconv1,start_neurons * 1, True)
 output_layer_noActi = Conv2D(1,(1,1), padding="same", activation=None )(uconv1)
 output_layer = Activation('sigmoid' )(output_layer_noActi)
 return output_layer<compute_test_metric>
train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)
 del train_identity, train_transaction, test_identity, test_transaction
gc.collect()
def get_iou_vector(A, B):
batch_size = A.shape[0]
metric = []
for batch in range(batch_size):
t, p = A[batch]>0, B[batch]>0
intersection = np.logical_and(t, p)
 union = np.logical_or(t, p)
 iou =(np.sum(intersection > 0)+ 1e-10)/(np.sum(union > 0)+ 1e-10)
 thresholds = np.arange(0.5, 1, 0.05)
 s = []
for thresh in thresholds:
s.append(iou > thresh)
 metric.append(np.mean(s))
return np.mean(metric)
 def my_iou_metric(label, pred):
return tf.py_func(get_iou_vector, [label, pred>0.5], tf.float64)
 def my_iou_metric_2(label, pred):
return tf.py_func(get_iou_vector, [label, pred >0], tf.float64 )<compute_train_metric>
def id_split(dataframe):
dataframe['device_name'] = dataframe['DeviceInfo'].str.split('/', expand=True)[0]
dataframe['device_version'] = dataframe['DeviceInfo'].str.split('/', expand=True)[1]
dataframe['OS_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[0]
dataframe['version_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[1]
dataframe['browser_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[0]
dataframe['version_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[1]
dataframe['screen_width'] = dataframe['id_33'].str.split('x', expand=True)[0]
dataframe['screen_height'] = dataframe['id_33'].str.split('x', expand=True)[1]
dataframe['id_34'] = dataframe['id_34'].str.split(':', expand=True)[1]
dataframe['id_23'] = dataframe['id_23'].str.split(':', expand=True)[1]
dataframe.loc[dataframe['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
dataframe.loc[dataframe['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
dataframe.loc[dataframe['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
dataframe.loc[dataframe['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
dataframe.loc[dataframe['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
dataframe.loc[dataframe['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
dataframe.loc[dataframe['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
dataframe.loc[dataframe['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
dataframe.loc[dataframe.device_name.isin(dataframe.device_name.value_counts() [dataframe.device_name.value_counts() < 200].index), 'device_name'] = "Others"
dataframe['had_id'] = 1
gc.collect()
return dataframe
def lovasz_grad(gt_sorted):

 gts = tf.reduce_sum(gt_sorted)
 intersection = gts - tf.cumsum(gt_sorted)
 union = gts + tf.cumsum(1.- gt_sorted)
 jaccard = 1.- intersection / union
jaccard = tf.concat(( jaccard[0:1], jaccard[1:] - jaccard[:-1]), 0)
 return jaccard
def lovasz_hinge(logits, labels, per_image=True, ignore=None):

 if per_image:
def treat_image(log_lab):
log, lab = log_lab
log, lab = tf.expand_dims(log, 0), tf.expand_dims(lab, 0)
 log, lab = flatten_binary_scores(log, lab, ignore)
 return lovasz_hinge_flat(log, lab)
 losses = tf.map_fn(treat_image,(logits, labels), dtype=tf.float32)
 loss = tf.reduce_mean(losses)
 else:
loss = lovasz_hinge_flat(*flatten_binary_scores(logits, labels, ignore))
return loss
def lovasz_hinge_flat(logits, labels):

 def compute_loss() :
labelsf = tf.cast(labels, logits.dtype)
 signs = 2.* labelsf - 1.
 errors = 1.- logits * tf.stop_gradient(signs)
 errors_sorted, perm = tf.nn.top_k(errors, k=tf.shape(errors)[0], name="descending_sort")
 gt_sorted = tf.gather(labelsf, perm)
 grad = lovasz_grad(gt_sorted)
 loss = tf.tensordot(tf.nn.relu(errors_sorted), tf.stop_gradient(grad), 1, name="loss_non_void")
 return loss
loss = tf.cond(tf.equal(tf.shape(logits)[0], 0),
lambda: tf.reduce_sum(logits)* 0.,
compute_loss,
strict=True,
name="loss" 
)
 return loss
def flatten_binary_scores(scores, labels, ignore=None):

 scores = tf.reshape(scores,(-1,))
labels = tf.reshape(labels,(-1,))
if ignore is None:
return scores, labels
valid = tf.not_equal(labels, ignore)
 vscores = tf.boolean_mask(scores, valid, name='valid_scores')
 vlabels = tf.boolean_mask(labels, valid, name='valid_labels')
 return vscores, vlabels
def lovasz_loss(y_true, y_pred):
y_true, y_pred = K.cast(K.squeeze(y_true, -1), 'int32'), K.cast(K.squeeze(y_pred, -1), 'float32')
 logits = y_pred
loss = lovasz_hinge(logits, y_true, per_image = True, ignore = None)
 return loss<concatenate>
useful_features = ['TransactionAmt', 'ProductCD', 'card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'addr1', 'addr2', 'dist1',
'P_emaildomain', 'R_emaildomain', 'C1', 'C2', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11', 'C12', 'C13',
'C14', 'D1', 'D2', 'D3', 'D4', 'D5', 'D6', 'D8', 'D9', 'D10', 'D11', 'D12', 'D13', 'D14', 'D15', 'M2', 'M3',
'M4', 'M5', 'M6', 'M7', 'M8', 'M9', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V17',
'V19', 'V20', 'V29', 'V30', 'V33', 'V34', 'V35', 'V36', 'V37', 'V38', 'V40', 'V44', 'V45', 'V46', 'V47', 'V48',
'V49', 'V51', 'V52', 'V53', 'V54', 'V56', 'V58', 'V59', 'V60', 'V61', 'V62', 'V63', 'V64', 'V69', 'V70', 'V71',
'V72', 'V73', 'V74', 'V75', 'V76', 'V78', 'V80', 'V81', 'V82', 'V83', 'V84', 'V85', 'V87', 'V90', 'V91', 'V92',
'V93', 'V94', 'V95', 'V96', 'V97', 'V99', 'V100', 'V126', 'V127', 'V128', 'V130', 'V131', 'V138', 'V139', 'V140',
'V143', 'V145', 'V146', 'V147', 'V149', 'V150', 'V151', 'V152', 'V154', 'V156', 'V158', 'V159', 'V160', 'V161',
'V162', 'V163', 'V164', 'V165', 'V166', 'V167', 'V169', 'V170', 'V171', 'V172', 'V173', 'V175', 'V176', 'V177',
'V178', 'V180', 'V182', 'V184', 'V187', 'V188', 'V189', 'V195', 'V197', 'V200', 'V201', 'V202', 'V203', 'V204',
'V205', 'V206', 'V207', 'V208', 'V209', 'V210', 'V212', 'V213', 'V214', 'V215', 'V216', 'V217', 'V219', 'V220',
'V221', 'V222', 'V223', 'V224', 'V225', 'V226', 'V227', 'V228', 'V229', 'V231', 'V233', 'V234', 'V238', 'V239',
'V242', 'V243', 'V244', 'V245', 'V246', 'V247', 'V249', 'V251', 'V253', 'V256', 'V257', 'V258', 'V259', 'V261',
'V262', 'V263', 'V264', 'V265', 'V266', 'V267', 'V268', 'V270', 'V271', 'V272', 'V273', 'V274', 'V275', 'V276',
'V277', 'V278', 'V279', 'V280', 'V282', 'V283', 'V285', 'V287', 'V288', 'V289', 'V291', 'V292', 'V294', 'V303',
'V304', 'V306', 'V307', 'V308', 'V310', 'V312', 'V313', 'V314', 'V315', 'V317', 'V322', 'V323', 'V324', 'V326',
'V329', 'V331', 'V332', 'V333', 'V335', 'V336', 'V338', 'id_01', 'id_02', 'id_03', 'id_05', 'id_06', 'id_09',
'id_11', 'id_12', 'id_13', 'id_14', 'id_15', 'id_17', 'id_19', 'id_20', 'id_30', 'id_31', 'id_32', 'id_33',
'id_36', 'id_37', 'id_38', 'DeviceType', 'DeviceInfo', 'device_name', 'device_version', 'OS_id_30', 'version_id_30',
'browser_id_31', 'version_id_31', 'screen_width', 'screen_height', 'had_id']
x_train = np.append(x_train, [np.fliplr(x)for x in x_train], axis=0)
 y_train = np.append(y_train, [np.fliplr(x)for x in y_train], axis=0)
 print(x_train.shape)
 print(y_valid.shape )<load_pretrained>
train = id_split(train)
 test = id_split(test )
if Path(previous_model_name ).is_file() :
print("Using previous sucessful run's model")
 model2 = load_model(previous_model_name,custom_objects={'my_iou_metric_2': my_iou_metric,
'lovasz_loss': lovasz_loss,
'my_iou_metric': my_iou_metric})
 else:
print("Using stored trained model")
 model2 = load_model(stored_trained_model,custom_objects={'my_iou_metric_2': my_iou_metric,
'lovasz_loss': lovasz_loss,
'my_iou_metric': my_iou_metric} )<choose_model_class>
cols_to_drop = [col for col in train.columns if col not in useful_features]
cols_to_drop.remove('isFraud')
 cols_to_drop.remove('TransactionDT')
 train = train.drop(cols_to_drop, axis=1)
 test = test.drop(cols_to_drop, axis=1 )
input_x = model2.layers[0].input
output_layer = Activation('sigmoid',name='output_activaton' )(model2.layers[-1].output)
 model1 = Model(input_x, output_layer)
 c = optimizers.adam(lr = 0.01)
 model1.compile(loss="binary_crossentropy", optimizer=c, metrics=[my_iou_metric])
<train_model>
columns_a = ['TransactionAmt', 'id_02', 'D15']
columns_b = ['card1', 'card4', 'addr1']
for col_a in columns_a:
for col_b in columns_b:
for df in [train, test]:
df[f'{col_a}_to_mean_{col_b}'] = df[col_a] / df.groupby([col_b])[col_a].transform('mean')
 df[f'{col_a}_to_std_{col_b}'] = df[col_a] / df.groupby([col_b])[col_a].transform('std' )
model_checkpoint = ModelCheckpoint(save_model_name,monitor='my_iou_metric',
mode = 'max', save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='my_iou_metric', mode = 'max',factor=0.5, patience=5, min_lr=0.0001, verbose=1)
 epochs = 10
batch_size = 32
history = model1.fit(x_train, y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[ model_checkpoint,reduce_lr],
verbose=2 )<choose_model_class>
train['TransactionAmt_Log'] = np.log(train['TransactionAmt'])
 test['TransactionAmt_Log'] = np.log(test['TransactionAmt'])
 train['TransactionAmt_decimal'] =(( train['TransactionAmt'] - train['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 test['TransactionAmt_decimal'] =(( test['TransactionAmt'] - test['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 train['Transaction_day_of_week'] = np.floor(( train['TransactionDT'] /(3600 * 24)- 1)% 7)
 test['Transaction_day_of_week'] = np.floor(( test['TransactionDT'] /(3600 * 24)- 1)% 7)
 train['Transaction_hour'] = np.floor(train['TransactionDT'] / 3600)% 24
test['Transaction_hour'] = np.floor(test['TransactionDT'] / 3600)% 24
model1 = load_model(save_model_name,custom_objects={'my_iou_metric': my_iou_metric})
 input_x = model1.layers[0].input
output_layer = model1.layers[-1].input
model = Model(input_x, output_layer)
 c = optimizers.adam(lr = 0.01)
 model.compile(loss=lovasz_loss, optimizer=c, metrics=[my_iou_metric_2])
<train_model>
for feature in ['id_02__id_20', 'id_02__D8', 'D11__DeviceInfo', 'DeviceInfo__P_emaildomain', 'P_emaildomain__C2',
'card2__dist1', 'card1__card5', 'card2__id_20', 'card5__P_emaildomain', 'addr1__card1']:
f1, f2 = feature.split('__')
 train[feature] = train[f1].astype(str)+ '_' + train[f2].astype(str)
 test[feature] = test[f1].astype(str)+ '_' + test[f2].astype(str)
 le = LabelEncoder()
le.fit(list(train[feature].astype(str ).values)+ list(test[feature].astype(str ).values))
train[feature] = le.transform(list(train[feature].astype(str ).values))
test[feature] = le.transform(list(test[feature].astype(str ).values))
early_stopping = EarlyStopping(monitor='val_my_iou_metric_2', mode = 'max',patience=30, verbose=1)
 model_checkpoint = ModelCheckpoint(save_model_name,monitor='val_my_iou_metric_2',
mode = 'max', save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='val_my_iou_metric_2', mode = 'max',factor=0.5, patience=5, min_lr=0.0001, verbose=1)
 epochs = 50
batch_size = 32
history = model.fit(x_train, y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[ model_checkpoint,reduce_lr,early_stopping],
verbose=2 )<choose_model_class>
for feature in ['card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'id_36']:
train[feature + '_count_full'] = train[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
test[feature + '_count_full'] = test[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
for feature in ['id_01', 'id_31', 'id_33', 'id_36']:
train[feature + '_count_dist'] = train[feature].map(train[feature].value_counts(dropna=False))
test[feature + '_count_dist'] = test[feature].map(test[feature].value_counts(dropna=False))
model = model2<load_pretrained>
for col in train.columns:
if train[col].dtype == 'object':
le = LabelEncoder()
le.fit(list(train[col].astype(str ).values)+ list(test[col].astype(str ).values))
train[col] = le.transform(list(train[col].astype(str ).values))
test[col] = le.transform(list(test[col].astype(str ).values))
model = load_model(save_model_name,custom_objects={'my_iou_metric_2': my_iou_metric_2,
'lovasz_loss': lovasz_loss})
<predict_on_test>
X = train.sort_values('TransactionDT' ).drop(['isFraud', 'TransactionDT'], axis=1)
 y = train.sort_values('TransactionDT')['isFraud']
X_test = test.drop(['TransactionDT'], axis=1)
 del train, test
gc.collect()
def predict_result(model,x_test,img_size_target):
x_test_reflect = np.array([np.fliplr(x)for x in x_test])
 preds_test = model.predict(x_test ).reshape(-1, img_size_target, img_size_target)
 preds_test2_refect = model.predict(x_test_reflect ).reshape(-1, img_size_target, img_size_target)
 preds_test += np.array([ np.fliplr(x)for x in preds_test2_refect])
 return preds_test/2<predict_on_test>
X = reduce_mem_usage(X)
 X_test = reduce_mem_usage(X_test )
preds_train = predict_result(model,x_train,img_size_target)
 preds_valid = predict_result(model,x_valid,img_size_target )<compute_train_metric>
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=9,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.9,
random_state=51,
tree_method='gpu_hist' 
)
thresholds_ori = np.linspace(0.3, 0.7, 31)
 thresholds = np.log(thresholds_ori/(1-thresholds_ori))
ious = np.array([iou_metric_batch(y_valid, preds_valid > threshold)for threshold in tqdm_notebook(thresholds)])
 print(ious )<categorify>
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=9,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.9,
random_state=51,
tree_method='gpu_hist' 
)
def rle_encode(im):

 pixels = im.flatten(order = 'F')
 pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<predict_on_test>
clf.fit(X, y )
x_all_train = np.array(train_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1)
 preds_train = predict_result(model,x_all_train,img_size_target)
 preds_test = predict_result(model,x_test,img_size_target )<merge>
y_train_pred = clf.predict_proba(X)[:,1]
print('AUC score: ',roc_auc_score(y, y_train_pred))
all_preds = list(np.concatenate(( preds_train,preds_test)))
 preds_df = pd.DataFrame(pd.Series(all_preds), columns = ['pred_masks'])
 preds_df.insert(0, 'orig_id', range(1,len(preds_df)+1))
all_arr = pd.merge(all_arr,preds_df,how = 'left',left_on = 'orig_id', right_on = 'orig_id')
<create_dataframe>
y_pred = clf.predict_proba(X_test)[:,1]
sample_submission = pd.read_csv('.. /input/ieee-fraud-detection/sample_submission.csv', index_col='TransactionID')
 sample_submission['isFraud'] = y_pred
sample_submission.to_csv('submission1.csv' )
all_arr_saved = all_arr.copy()<prepare_x_and_y>
def create_eval_metric_df(y,y_pred,model_name='model'):
total = np.shape(y)[0]
tn, fp, fn, tp = metrics.confusion_matrix(y, y_pred ).ravel()
tp /= float(total)
 tn /= float(total)
 fp /= float(total)
 fn /= float(total)
 roc_auc = metrics.roc_auc_score(y, y_pred)
 columns_list = ['model_name',
"total_number",
"cond_pos", "cond_neg",
"true_pos", "true_neg", "false_pos", "false_neg",
"recall", "specificity", "precision", "neg_pred_val", "miss_rate", "false_pos_rate", "false_disc_rate", "false_omission_rate",
"accuracy", "f1_score", "matthews_corr_coef", "informedness", "markedness",
"roc_auc_score"
]
eval_metrics = [model_name,
total,
tp+fn, tn+fp,
tp, tn, fp, fn, 
(tp)/(tp + fn),(tn)/(tn + fp),(tp)/(tp + fp),(tn)/(tn + fn),(fn)/(fn + tp),(fp)/(fp + tn),(fp)/(fp + tp),(fn)/(fn + tn), 
(tp + tn),(2*tp)/(2*tp + fp + fn),(tp*tn - fp*fn)/(np.sqrt(( tp + fp)*(tp + fn)*(tn + fp)*(tn + fn))),(tp)/(tp + fn)+(tn)/(tn + fp)- 1,(tp)/(tp + fp)+(tn)/(tn + fn)- 1,
roc_auc
]
df_eval = pd.DataFrame(eval_metrics)
 df = df_eval.T
df.columns = columns_list
return df
preds_test = np.array(all_arr.loc[all_arr['img_id']>3999,'pred_masks'].tolist())
 preds_train = np.array(all_arr.loc[all_arr['img_id']<=3999,'pred_masks'].tolist())
 y_train = np.array(train_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1)
<compute_test_metric>
y_train_pred = clf.predict(X )
t1 = time.time()
pred_dict = {idx: rle_encode(np.round(downsample(preds_test[i])> threshold_best)) for i, idx in enumerate(tqdm_notebook(test_df.index.values)) }
t2 = time.time()
print(f"Usedtime = {t2-t1} s" )<save_to_csv>
df_eval = create_eval_metric_df(y,y_train_pred,model_name='xgboost1')
 print(df_eval )
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv(submission_file )<train_model>
X_train, X_val, y_train, y_val = train_test_split(X, y, random_state=0 )
t_finish = time.time()
print(f"Kernel run time = {(t_finish-t_start)/3600} hours" )<install_modules>
clf2 = xgb.XGBClassifier(
 n_estimators=500,
max_depth=9,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.9,
random_state=51,
tree_method='gpu_hist' 
)
!pip3 install pycocotools<set_options>
clf2.fit(X_train,y_train )
%matplotlib inline
%reload_ext autoreload
%autoreload 2
print(torch.__version__)
 torch.cuda.is_available()
torch.backends.cudnn.benchmark=True<load_from_csv>
y_train_pred = clf2.predict(X_train)
 y_val_pred = clf2.predict(X_val)
 print('AUC score train: ',roc_auc_score(y_val, y_val_pred))
print('AUC score val: ',roc_auc_score(y_val, y_val_pred))
MASKS_FN = 'train.csv'
TRAIN_DN = Path('train/images/')
 MASKS_DN = Path('train/masks/')
 TEST = Path('test/images/')
 PATH = Path('/kaggle/input/tgs-salt-identification-challenge/')
 PATH128 = Path('/tmp/128/')
 TMP = Path('/tmp/')
 MODEL = Path('/tmp/model/')
 PRETRAINED = Path('/kaggle/input/is-there-salt-resnet34/model/resnet34_issalt.h5')
 seg = pd.read_csv(PATH/MASKS_FN ).set_index('id')
 seg.head()
sz = 128
bs = 64
nw = 4<define_variables>
df_eval_train = create_eval_metric_df(y_train,y_train_pred,model_name='xgboost2')
 df_eval_val = create_eval_metric_df(y_val,y_val_pred,model_name='xgboost2' )
train_names_png = [TRAIN_DN/f for f in os.listdir(PATH/TRAIN_DN)]
train_names = list(seg.index.values)
 masks_names_png = [MASKS_DN/f for f in os.listdir(PATH/MASKS_DN)]
test_names_png = [TEST/f for f in os.listdir(PATH/TEST)]<categorify>
clfFiveFold = xgb.XGBClassifier(
 n_estimators=500,
max_depth=9,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.9,
random_state=51,
tree_method='gpu_hist' 
)
with ThreadPoolExecutor(4)as e: e.map(resize_mask, train_names_png )<categorify>
%%time
NFOLDS = 5
folds = KFold(n_splits = NFOLDS)
 splits = folds.split(X,y)
 y_preds = np.zeros(X_test.shape[0])
 y_oof = np.zeros(X.shape[0])
 score = 0
feature_importances = pd.DataFrame()
feature_importances['feature'] = X.columns
for fold_n,(train_index, valid_index)in enumerate(splits):
X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
clfFiveFold.fit(X_train, y_train)
 y_pred_train = clfFiveFold.predict(X_train)
 print(f"Fold {fold_n + 1} | Training AUC: {roc_auc_score(y_train, y_pred_train)}")
 y_pred_valid = clfFiveFold.predict(X_valid)
 y_oof[valid_index] = y_pred_valid
print(f"Fold {fold_n + 1} | Validation AUC: {roc_auc_score(y_valid, y_pred_valid)}")
 score += roc_auc_score(y_valid, y_pred_valid)/ NFOLDS
y_preds += clfFiveFold.predict(X_test)/ NFOLDS
del X_train, X_valid, y_train, y_valid
gc.collect()
print(f"
Mean AUC = {score}")
 print(f"Out of folds AUC = {roc_auc_score(y, y_oof)}" )
with ThreadPoolExecutor(4)as e: e.map(resize_mask, masks_names_png )<categorify>
y_train_pred = clfFiveFold.predict(X)
 print('AUC score train: ',roc_auc_score(y, y_train_pred))
print('AUC score OOF: ',roc_auc_score(y, y_oof))
df_eval_train = create_eval_metric_df(y,y_train_pred,model_name='xgboost2')
 df_eval_oof = create_eval_metric_df(y,y_oof,model_name='xgboost2' )
with ThreadPoolExecutor(4)as e: e.map(resize_mask, test_names_png )<define_variables>
X_train, X_val, y_train, y_val = train_test_split(X, y, random_state=0 )
PATH = PATH128<import_modules>
import eli5
from datasets import CustomDataset<compute_test_metric>
eli5.show_weights(clf )
def dice(pred, targs):
pred =(pred>0 ).float()
return 2.*(pred*targs ).sum() /(pred+targs ).sum()
def IoU_np(pred, targs, thres=0):
pred =(pred>thres)
 intersection =(pred*targs ).sum()
return intersection /(( pred+targs ).sum() - intersection + 1.0)
 def IoU(pred, targs, thres=0):
pred =(pred>thres ).float()
intersection =(pred*targs ).sum()
return intersection /(( pred+targs ).sum() - intersection + 1.0 )<load_pretrained>
eli5.show_weights(clf2 )
def get_base() :
layers = cut_model(f(True), cut)
 return nn.Sequential(*layers)
 def load_pretrained(model, path):
weights = torch.load(PRETRAINED, map_location=lambda storage, loc: storage)
 model.load_state_dict(weights, strict=False)
 return model<categorify>
eli5.show_weights(clfFiveFold )
class SaveFeatures() :
features=None
def __init__(self, m): self.hook = m.register_forward_hook(self.hook_fn)
 def hook_fn(self, module, input, output): self.features = output
def remove(self): self.hook.remove()<concatenate>
def _check_booster_args(xgb, is_regression=None):
if isinstance(xgb, eli5.xgboost.Booster):
booster = xgb
else:
booster = xgb.get_booster()
_is_regression = isinstance(xgb, XGBRegressor)
 if is_regression is not None and is_regression != _is_regression:
raise ValueError(
 'Inconsistent is_regression={} passed.'
'You don't have to pass it when using scikit-learn API' 
.format(is_regression))
is_regression = _is_regression
return booster, is_regression
eli5.xgboost._check_booster_args = _check_booster_args
class UnetBlock(nn.Module):
def __init__(self, up_in, x_in, n_out):
super().__init__()
up_out = x_out = n_out//2
self.x_conv = nn.Conv2d(x_in, x_out, 1)
 self.tr_conv = nn.ConvTranspose2d(up_in, up_out, 2, stride=2)
 self.bn = nn.BatchNorm2d(n_out)
 def forward(self, up_p, x_p):
up_p = self.tr_conv(up_p)
 x_p = self.x_conv(x_p)
 cat_p = torch.cat([up_p,x_p], dim=1)
 return self.bn(F.relu(cat_p))<choose_model_class>
colnames = list(X_val.columns)
 elist = []
for i in range(10):
e = eli5.explain_prediction(clf2, X_val.iloc[i], top=10, feature_names=colnames)
 elist.append(e )
class UnetModel() :
def __init__(self,model,name='unet'):
self.model,self.name = model,name
def get_layer_groups(self, precompute):
lgs = list(split_by_idxs(children(self.model.rn), [lr_cut]))
return lgs + [children(self.model)[1:]]<prepare_x_and_y>
for i in range(10):
etext = eli5.formatters.text.format_as_text(elist[i])
 print(etext)
 print('------')
x_names = [f'{x}.png' for x in train_names]
x_names_path = np.array([str(TRAIN_DN/x)for x in x_names])
 y_names = [x for x in x_names]
y_names_path = np.array([str(MASKS_DN/x)for x in x_names] )<set_options>
df_eli = eli5.formatters.as_dataframe.explain_prediction_df(clf2, X_val.iloc[1], top=10, feature_names=colnames )
<split><EOS> 
x = eli5.explain_prediction_xgboost(clfFiveFold, X_val.iloc[1])
 print(x )