Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<choose_model_class>
%matplotlib inline
class TransformerBlock(layers.Layer):
def __init__(self, embed_dim, num_heads, ff_dim, rate=0.2):
super(TransformerBlock, self ).__init__()
self.att = MultiHeadSelfAttention(embed_dim, num_heads)
 self.ffn = keras.Sequential(
 [layers.Dense(ff_dim, activation="relu"), layers.Dense(embed_dim),] 
)
 self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)
 self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)
 self.dropout1 = layers.Dropout(rate)
 self.dropout2 = layers.Dropout(rate)
 def call(self, inputs, training):
attn_output = self.att(inputs)
 attn_output = self.dropout1(attn_output, training=training)
 out1 = self.layernorm1(inputs + attn_output)
 ffn_output = self.ffn(out1)
 ffn_output = self.dropout2(ffn_output, training=training)
 return self.layernorm2(out1 + ffn_output )<choose_model_class>
train = pd.read_csv('.. /input/train.csv', index_col=0)
 test = pd.read_csv('.. /input/test.csv', index_col=0)
 target = train.target.values
train.drop('target', axis=1, inplace=True)
 train.shape, target.shape, test.shape,
class TokenAndPositionEmbedding(layers.Layer):
def __init__(self, maxlen, vocab_size, embed_dim):
super(TokenAndPositionEmbedding, self ).__init__()
self.token_emb = layers.Embedding(input_dim=vocab_size, output_dim=embed_dim)
 self.pos_emb = layers.Embedding(input_dim=maxlen, output_dim=embed_dim)
 def call(self, x):
maxlen = tf.shape(x)[-1]
positions = tf.range(start=0, limit=maxlen, delta=1)
 positions = self.pos_emb(positions)
 x = self.token_emb(x)
 return x + positions<categorify>
zval1 =(train.values - stats_df.neg_mean.values)/ stats_df.neg_sd.values
zval2 =(train.values - stats_df.pos_mean.values)/ stats_df.pos_sd.values
tr_zval = np.column_stack([zval1, zval2])
 tr_zval.shape
maxlen = 32
X_train = keras.preprocessing.sequence.pad_sequences(X_train, maxlen=maxlen)
 X_test = keras.preprocessing.sequence.pad_sequences(X_test, maxlen=maxlen )<choose_model_class>
zval1 =(test.values - stats_df.neg_mean.values)/ stats_df.neg_sd.values
zval2 =(test.values - stats_df.pos_mean.values)/ stats_df.pos_sd.values
te_zval = np.column_stack([zval1, zval2])
 te_zval.shape
embed_dim = 32
num_heads = 8
ff_dim = 64
vocab_size = 50000
inputs = layers.Input(shape=(maxlen,))
embedding_layer = TokenAndPositionEmbedding(maxlen, vocab_size, embed_dim)
 x = embedding_layer(inputs)
 transformer_block = TransformerBlock(embed_dim, num_heads, ff_dim)
 x = transformer_block(x)
 x = layers.GlobalAveragePooling1D()(x)
 x = layers.Dropout(0.2 )(x)
 x = layers.Dense(20, activation="relu" )(x)
 x = layers.Dropout(0.2 )(x)
 outputs = layers.Dense(1, activation="sigmoid" )(x)
 model = keras.Model(inputs=inputs, outputs=outputs )<choose_model_class>
def augment(X, y, times=8):
pos_idx =(y == 1)
 nsample = times * pos_idx.sum()
X_sample = []
for i in range(X.shape[1]):
X_sample.append(np.random.choice(X[pos_idx, i], size=nsample))
X_sample = np.column_stack(X_sample)
 idx = np.arange(X.shape[0] + nsample)
 np.random.shuffle(idx)
 return np.vstack([X, X_sample])[idx], np.hstack([y, np.ones(nsample)])[idx]
es = EarlyStopping(monitor='val_auc', mode='max',patience=10 )<train_model>
nfold = 5
kfold = StratifiedKFold(n_splits=nfold, shuffle=True)
 nn_oof_tr, nn_oof_te = np.zeros(tr_zval.shape[0]), 0
lgb_oof_tr, lgb_oof_te = np.zeros(tr_zval.shape[0]), 0
nb_oof_tr, nb_oof_te = np.zeros(tr_zval.shape[0]), 0
for tr_idx, va_idx in kfold.split(tr_zval, target):
X_tr, y_tr = tr_zval[tr_idx], target[tr_idx]
X_va, y_va = tr_zval[va_idx], target[va_idx]
X_tr, y_tr = augment(X_tr, y_tr)
 model = Sequential([
layers.Dense(16, input_shape=(400, 1), activation='relu'),
layers.Flatten() ,
layers.Dense(1, activation='sigmoid')
 ])
 model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['binary_crossentropy'])
 earlystop = callbacks.EarlyStopping(patience=10)
 X_tr_re = X_tr[:, :, np.newaxis]
X_va_re = X_va[:, :, np.newaxis]
model.fit(X_tr_re, y_tr, validation_data=(X_va_re, y_va), epochs=100, verbose=2, batch_size=256, callbacks=[earlystop])
 nn_oof_tr[va_idx] = model.predict(X_va_re ).flatten()
X_te_re = te_zval[:, :, np.newaxis]
nn_oof_te += model.predict(X_te_re ).flatten() / nfold
param = {
'objective': 'binary',
'boost': 'gbdt',
'metric': 'auc',
'learning_rate': 0.01,
'num_leaves': 13,
'max_depth': -1,
'feature_fraction': 0.05,
'bagging_freq': 5,
'bagging_fraction': 0.4,
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10,
'num_threads': 4
}
trn_data = lgb.Dataset(X_tr, y_tr)
 val_data = lgb.Dataset(X_va, y_va)
 clf = lgb.train(param, trn_data, 100000, valid_sets=(val_data), early_stopping_rounds=600, verbose_eval=600)
 lgb_oof_tr[va_idx] = clf.predict(X_va)
 lgb_oof_te += clf.predict(te_zval)/ nfold
bayes = GaussianNB()
bayes.fit(X_tr, y_tr)
 nb_oof_tr[va_idx] = bayes.predict_proba(X_va)[:, 1]
nb_oof_te += bayes.predict_proba(te_zval)[:, 1] / nfold
model.compile("adam", "binary_crossentropy", metrics=[tf.keras.metrics.AUC() ])
 history = model.fit(
 X_train, y, epochs=25,verbose=1,validation_split=0.1,batch_size=65536,callbacks=[es]
)<compute_train_metric>
lr = LogisticRegression()
lr.fit(np.column_stack([nn_oof_tr, lgb_oof_tr, nb_oof_tr]), target)
 pred = lr.predict_proba(np.column_stack([nn_oof_te, lgb_oof_te, nb_oof_te])) [:, 1]
y_pred = model.predict(X_test)
 y_true = np.array(y_val)
 roc_auc_score(y_true, y_pred )<split>
pd.DataFrame({
'ID_code': test.index,
'target': pred
} ).to_csv('sub.csv', index=False )