Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<feature_engineering>
import numpy as np
import pandas as pd
from sklearn.preprocessing import LabelEncoder
import xgboost as xgb
sub = pd.read_csv('.. /input/SampleSubmissionStage2.csv')
 sub['Season'], sub['Team1'], sub['Team2'] = sub['ID'].str.split('_' ).str
sub[['Season', 'Team1', 'Team2']] = sub[['Season', 'Team1', 'Team2']].apply(pd.to_numeric)
 sub['deltaSeed'] = sub.apply(delta_seed,axis=1)
 sub['deltaMO'] = sub.apply(delta_ord,axis=1)
 sub['deltaWinPct'] = sub.apply(delta_winPct,axis=1)
 cols = ['PointsFor','PointsAgainst','FGM','FGA','FGM3','FGA3','FTM','FTA','OR','DR','Ast','TO','Stl','Blk','PF']
for col in cols:
print("Processing",col)
 sub['delta' + col] = sub.apply(delta_stat,args=(col,),axis=1)
 cols = ['deltaMO', 'deltaWinPct', 'deltaPointsAgainst']
biases = {
'1163':1.2,
'1181':1.2,
'1437':1.2,
'1314':1.2,
}
Xtrain = training_set[cols]
ytrain = training_set['Result']
logit_model=sm.Logit(ytrain,Xtrain)
 result=logit_model.fit()
Xtest = sub[cols]
pred = result.predict(Xtest)
 for bias in biases:
pred.loc[sub['Team1'] == int(bias)] = pred.loc[sub['Team1'] == int(bias)] * biases[bias]
pred.loc[sub['Team2'] == int(bias)] = pred.loc[sub['Team2'] == int(bias)] / biases[bias]
pred.loc[(pred >= 0.9999)] = 0.9999
pred.loc[(pred <= 0.0001)] = 0.0001
pred.loc[(sub['Team1'] == 1163)&(sub['Team2'] == 1181)] = 0.8
sub['Pred'] = pred
sub[['ID', 'Pred']].to_csv('submission.csv', index=False)
<import_modules>
train_transaction = pd.read_csv('.. /input/ieee-fraud-detection/train_transaction.csv', index_col='TransactionID')
 test_transaction = pd.read_csv('.. /input/ieee-fraud-detection/test_transaction.csv', index_col='TransactionID')
 train_identity = pd.read_csv('.. /input/ieee-fraud-detection/train_identity.csv', index_col='TransactionID')
 test_identity = pd.read_csv('.. /input/ieee-fraud-detection/test_identity.csv', index_col='TransactionID')
 sample_submission = pd.read_csv('.. /input/ieee-fraud-detection/sample_submission.csv', index_col='TransactionID' )
!pip install lap
Lambda, MaxPooling2D, Reshape 
<load_from_csv>
train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True )
TRAIN_DF = '.. /input/humpback-whale-identification/train.csv'
SUB_Df = '.. /input/humpback-whale-identification/sample_submission.csv'
TRAIN = '.. /input/humpback-whale-identification/train/'
TEST = '.. /input/humpback-whale-identification/test/'
P2H = '.. /input/metadata/p2h.pickle'
P2SIZE = '.. /input/metadata/p2size.pickle'
BB_DF = ".. /input/metadata/bounding_boxes.csv"
tagged = dict([(p, w)for _, p, w in read_csv(TRAIN_DF ).to_records() ])
 submit = [p for _, p, _ in read_csv(SUB_Df ).to_records() ]
join = list(tagged.keys())+ submit<train_model>
y_train = train['isFraud'].copy()
if isfile(P2SIZE):
print("P2SIZE exists.")
 with open(P2SIZE, 'rb')as f:
p2size = pickle.load(f)
 else:
p2size = {}
for p in tqdm(join):
size = pil_image.open(expand_path(p)).size
p2size[p] = size<compute_test_metric>
del train_transaction, train_identity, test_transaction, test_identity
def match(h1, h2):
for p1 in h2ps[h1]:
for p2 in h2ps[h2]:
i1 = pil_image.open(expand_path(p1))
i2 = pil_image.open(expand_path(p2))
if i1.mode != i2.mode or i1.size != i2.size: return False
a1 = np.array(i1)
 a1 = a1 - a1.mean()
a1 = a1 / sqrt(( a1 ** 2 ).mean())
 a2 = np.array(i2)
 a2 = a2 - a2.mean()
a2 = a2 / sqrt(( a2 ** 2 ).mean())
 a =(( a1 - a2)** 2 ).mean()
if a > 0.1: return False
return True
if isfile(P2H):
print("P2H exists.")
 with open(P2H, 'rb')as f:
p2h = pickle.load(f)
 else:
p2h = {}
for p in tqdm(join):
img = pil_image.open(expand_path(p))
h = phash(img)
 p2h[p] = h
h2ps = {}
for p, h in p2h.items() :
if h not in h2ps: h2ps[h] = []
if p not in h2ps[h]: h2ps[h].append(p)
 hs = list(h2ps.keys())
 h2h = {}
for i, h1 in enumerate(tqdm(hs)) :
for h2 in hs[:i]:
if h1 - h2 <= 6 and match(h1, h2):
s1 = str(h1)
 s2 = str(h2)
 if s1 < s2: s1, s2 = s2, s1
h2h[s1] = s2
for p, h in p2h.items() :
h = str(h)
 if h in h2h: h = h2h[h]
p2h[p] = h
h2ps = {}
for p, h in p2h.items() :
if h not in h2ps: h2ps[h] = []
if p not in h2ps[h]: h2ps[h].append(p )<categorify>
X_train = train.drop('isFraud', axis=1)
 X_test = test.copy()
def prefer(ps):
if len(ps)== 1: return ps[0]
best_p = ps[0]
best_s = p2size[best_p]
for i in range(1, len(ps)) :
p = ps[i]
s = p2size[p]
if s[0] * s[1] > best_s[0] * best_s[1]:
best_p = p
best_s = s
return best_p
h2p = {}
for h, ps in h2ps.items() :
h2p[h] = prefer(ps)
 len(h2p), list(h2p.items())[:5]<set_options>
del train, test
p2bb = pd.read_csv(BB_DF ).set_index("Image")
 old_stderr = sys.stderr
sys.stderr = open('/dev/null' if platform.system() != 'Windows' else 'nul', 'w')
 sys.stderr = old_stderr
img_shape =(384, 384, 1)
 anisotropy = 2.15
crop_margin = 0.05<normalization>
X_train = X_train.fillna(-999)
 X_test = X_test.fillna(-999 )
def build_transform(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):

 rotation = np.deg2rad(rotation)
 shear = np.deg2rad(shear)
 rotation_matrix = np.array(
 [[np.cos(rotation), np.sin(rotation), 0], [-np.sin(rotation), np.cos(rotation), 0], [0, 0, 1]])
 shift_matrix = np.array([[1, 0, height_shift], [0, 1, width_shift], [0, 0, 1]])
 shear_matrix = np.array([[1, np.sin(shear), 0], [0, np.cos(shear), 0], [0, 0, 1]])
 zoom_matrix = np.array([[1.0 / height_zoom, 0, 0], [0, 1.0 / width_zoom, 0], [0, 0, 1]])
 shift_matrix = np.array([[1, 0, -height_shift], [0, 1, -width_shift], [0, 0, 1]])
 return np.dot(np.dot(rotation_matrix, shear_matrix), np.dot(zoom_matrix, shift_matrix))<prepare_x_and_y>
for f in X_train.columns:
if X_train[f].dtype=='object' or X_test[f].dtype=='object':
lbl = LabelEncoder()
lbl.fit(list(X_train[f].values)+ list(X_test[f].values))
X_train[f] = lbl.transform(list(X_train[f].values))
X_test[f] = lbl.transform(list(X_test[f].values))
def read_cropped_image(p, augment):

 if p in h2p:
p = h2p[p]
size_x, size_y = p2size[p]
row = p2bb.loc[p]
x0, y0, x1, y1 = row['x0'], row['y0'], row['x1'], row['y1']
dx = x1 - x0
dy = y1 - y0
x0 -= dx * crop_margin
x1 += dx * crop_margin + 1
y0 -= dy * crop_margin
y1 += dy * crop_margin + 1
if x0 < 0:
x0 = 0
if x1 > size_x:
x1 = size_x
if y0 < 0:
y0 = 0
if y1 > size_y:
y1 = size_y
dx = x1 - x0
dy = y1 - y0
if dx > dy * anisotropy:
dy = 0.5 *(dx / anisotropy - dy)
 y0 -= dy
y1 += dy
else:
dx = 0.5 *(dy * anisotropy - dx)
 x0 -= dx
x1 += dx
trans = np.array([[1, 0, -0.5 * img_shape[0]], [0, 1, -0.5 * img_shape[1]], [0, 0, 1]])
 trans = np.dot(np.array([[(y1 - y0)/ img_shape[0], 0, 0], [0,(x1 - x0)/ img_shape[1], 0], [0, 0, 1]]), trans)
 if augment:
trans = np.dot(build_transform(
 random.uniform(-5, 5),
random.uniform(-5, 5),
random.uniform(0.8, 1.0),
random.uniform(0.8, 1.0),
random.uniform(-0.05 *(y1 - y0), 0.05 *(y1 - y0)) ,
random.uniform(-0.05 *(x1 - x0), 0.05 *(x1 - x0)) 
), trans)
 trans = np.dot(np.array([[1, 0, 0.5 *(y1 + y0)], [0, 1, 0.5 *(x1 + x0)], [0, 0, 1]]), trans)
 img = read_raw_image(p ).convert('L')
 img = img_to_array(img)
 matrix = trans[:2, :2]
offset = trans[:2, 2]
img = img.reshape(img.shape[:-1])
 img = affine_transform(img, matrix, offset, output_shape=img_shape[:-1], order=1, mode='constant',
cval=np.average(img))
img = img.reshape(img_shape)
 img -= np.mean(img, keepdims=True)
 img /= np.std(img, keepdims=True)+ K.epsilon()
return img
def read_for_training(p):

 return read_cropped_image(p, True)
 def read_for_validation(p):

 return read_cropped_image(p, False)
 p = list(tagged.keys())[312]<categorify>
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=9,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.9,
missing=-999,
random_state=2019,
tree_method='gpu_hist'
)
def subblock(x, filter, **kwargs):
x = BatchNormalization()(x)
 y = x
y = Conv2D(filter,(1, 1), activation='relu', **kwargs )(y)
 y = BatchNormalization()(y)
 y = Conv2D(filter,(3, 3), activation='relu', **kwargs )(y)
 y = BatchNormalization()(y)
 y = Conv2D(K.int_shape(x)[-1],(1, 1), **kwargs )(y)
 y = Add()([x, y])
 y = Activation('relu' )(y)
 return y
def build_model(lr, l2, activation='sigmoid'):
regul = regularizers.l2(l2)
 optim = Adam(lr=lr)
 kwargs = {'padding': 'same', 'kernel_regularizer': regul}
inp = Input(shape=img_shape)
 x = Conv2D(64,(9, 9), strides=2, activation='relu', **kwargs )(inp)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 for _ in range(2):
x = BatchNormalization()(x)
 x = Conv2D(64,(3, 3), activation='relu', **kwargs )(x)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(128,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = subblock(x, 64, **kwargs)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(256,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = subblock(x, 64, **kwargs)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(384,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = subblock(x, 96, **kwargs)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(512,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = subblock(x, 128, **kwargs)
 x = GlobalMaxPooling2D()(x)
 branch_model = Model(inp, x)
 mid = 32
xa_inp = Input(shape=branch_model.output_shape[1:])
 xb_inp = Input(shape=branch_model.output_shape[1:])
 x1 = Lambda(lambda x: x[0] * x[1] )([xa_inp, xb_inp])
 x2 = Lambda(lambda x: x[0] + x[1] )([xa_inp, xb_inp])
 x3 = Lambda(lambda x: K.abs(x[0] - x[1]))([xa_inp, xb_inp])
 x4 = Lambda(lambda x: K.square(x))(x3)
 x = Concatenate()([x1, x2, x3, x4])
 x = Reshape(( 4, branch_model.output_shape[1], 1), name='reshape1' )(x)
 x = Conv2D(mid,(4, 1), activation='relu', padding='valid' )(x)
 x = Reshape(( branch_model.output_shape[1], mid, 1))(x)
 x = Conv2D(1,(1, mid), activation='linear', padding='valid' )(x)
 x = Flatten(name='flatten' )(x)
 x = Dense(1, use_bias=True, activation=activation, name='weighted-average' )(x)
 head_model = Model([xa_inp, xb_inp], x, name='head')
 img_a = Input(shape=img_shape)
 img_b = Input(shape=img_shape)
 xa = branch_model(img_a)
 xb = branch_model(img_b)
 x = head_model([xa, xb])
 model = Model([img_a, img_b], x)
 model.compile(optim, loss='binary_crossentropy', metrics=['binary_crossentropy', 'acc'])
 return model, branch_model, head_model
model, branch_model, head_model = build_model(64e-5, 0 )<feature_engineering>
%time clf.fit(X_train, y_train )
<categorify><EOS> 
sample_submission['isFraud'] = clf.predict_proba(X_test)[:,1]
sample_submission.to_csv('simple_xgboost.csv' )