Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<compute_test_metric>
warnings.filterwarnings('ignore' )
if VERBOSE:
TEST_GENERATOR = RetinaGenerator(test_df, BATCH_SIZE)
 STEPS = TEST_GENERATOR.step_per_pred
Y_pred =(model.predict_generator(TEST_GENERATOR.getInputGenerator() , steps=STEPS)> 0.5 ).sum(axis=1)- 1
Y_test = np.array(test_df.diagnosis)
 print("Average absolute distance is: {:.2f}".format(np.abs(Y_pred - Y_test ).mean()))
display(confusion_matrix(Y_test, Y_pred))
print("Accuracy Score:" + str(accuracy_score(Y_test, Y_pred)))
 print("Cohen Kappa Score:" + str(cohen_kappa_score(Y_test, Y_pred, weights='quadratic')))
<predict_on_test>
import keras
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers.normalization import BatchNormalization
from keras.callbacks import EarlyStopping
from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, LearningRateScheduler
from keras.models import load_model
from keras.initializers import glorot_normal, Zeros, Ones
import keras.backend as K
from keras.optimizers import RMSprop
import tensorflow as tf
STEPS = ALL_GENERATOR.step_per_pred
Y_pred = model.predict_generator(ALL_GENERATOR.getInputGenerator() , steps=STEPS)
 Y_true = to_categorical(np.array(df.diagnosis), 5 )<train_model>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
post_model = Sequential()
post_model.add(Dense(5, input_dim=5, activation='relu'))
post_model.add(Dense(5, activation='softmax'))
post_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 opt = optimizers.SGD(lr=LEARNING_RATE, decay=1e-6, momentum=0.9, nesterov=True)
 Callbacks=[EarlyStopping(patience=PATIENCE, restore_best_weights=True),
ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=LR_PATIENCE, verbose=VERBOSE)]
with tf.device('/device:GPU:0'):
H = post_model.fit(Y_pred, Y_true, batch_size=10, epochs=10, verbose=VERBOSE)
<drop_column>
features = [c for c in train.columns if c not in ['ID_code', 'target']]
test_id_code = test['ID_code']
train_id_code = train['ID_code']
target = train['target']
if DATA_AUGM:
shutil.rmtree(PATH_AUGM, ignore_errors=True, onerror=None )<load_from_csv>
df_train = train[features].copy()
df_test = test[features].copy()
train_target = target.values
ntrain = df_train.shape[0]
ntest = df_test.shape[0]
sub_df = pd.read_csv(PATH_APTOS + 'test.csv')
 if VERBOSE:
print(sub_df.shape)
 display(sub_df.head() )<predict_on_test>
def hot_encoder(df, columns):
one_hot = {c: list(df[c].unique())for c in columns}
for c in one_hot:
for val in one_hot[c]:
df[c+'_oh_' + str(val)] =(df[c].values == val ).astype(np.int)
 return df
def scale_feat(df):
scaler = StandardScaler()
return pd.DataFrame(scaler.fit_transform(df), columns=df.columns)
 def rank_gauss(x):
N = x.shape[0]
temp = x.argsort()
rank_x = temp.argsort() / N
rank_x -= rank_x.mean()
rank_x *= 2
efi_x = erfinv(rank_x)
 efi_x -= efi_x.mean()
return efi_x
def df_inputSwapNoise(df, p):
n = df.shape[0]
idx = list(range(n))
swap_n = round(n*p)
 for col in df.columns:
arr = df[col].values
col_vals = np.random.permutation(arr)
 swap_idx = np.random.choice(idx, size= swap_n)
 arr[swap_idx] = np.random.choice(col_vals, size = swap_n)
 df[col] = arr
return df
SUB_GENERATOR = RetinaGenerator(sub_df, BATCH_SIZE, False)
 STEPS = SUB_GENERATOR.step_per_pred
Y_sub_preview = model.predict_generator(SUB_GENERATOR.getInputGenerator() , steps=STEPS)
 Y_sub_cat = post_model.predict(Y_sub_preview)
 Y_sub = np.argmax(Y_sub_cat, axis=1)
 if VERBOSE:
print(Y_sub.shape )<save_to_csv>
print('Transforming data')
 feature_cols = [c for c in df_train.columns if c not in ['ID_code','target']]
keep_cols = [c for c in feature_cols]
df_all = pd.concat([df_train[keep_cols], df_test[keep_cols]])
 df_all_org = df_all.copy()
df_all_noise = df_inputSwapNoise(df_all, 0.15)
 data_all_org = df_all_org.values
data_all_noise = df_all_noise.values
cols = data_all_org.shape[1]
print('Final data with {} columns'.format(cols))
sub_df.to_csv('submission.csv', index=False )<load_from_csv>
for i in range(cols):
u = np.unique(data_all_org[:,i])
 if u.shape[0] > 3:
data_all_org[:,i] = rank_gauss(data_all_org[:,i])
 for i in range(cols):
u = np.unique(data_all_noise[:,i])
 if u.shape[0] > 3:
data_all_noise[:,i] = rank_gauss(data_all_noise[:,i])
 train_data_orig = data_all_org[0:ntrain,:]
test_data_orig = data_all_org[ntrain:,:]
train_data_noise = data_all_noise[0:ntrain,:]
test_data_noise = data_all_noise[ntrain:,:]
print(train_data_orig.shape)
 print(test_data_orig.shape)
 print(train_data_noise.shape)
 print(test_data_noise.shape )
train = pd.read_csv('.. /input/train_V2.csv')
 train.head(10 )<feature_engineering>
print('Original data')
 all_data = np.vstack(( train_data_orig, test_data_orig))
print('Noise data')
 all_data_noise = np.vstack(( train_data_noise, test_data_noise))
<load_from_csv>
print('Creating neural net')
 model = Sequential()
model.add(Dense(units=1500, input_dim = all_data.shape[1], kernel_initializer=glorot_normal()))
model.add(Activation('relu'))
model.add(BatchNormalization())
 model.add(Dense(units=1500, kernel_initializer=glorot_normal()))
model.add(Activation('relu'))
model.add(BatchNormalization())
 model.add(Dense(units=1500, kernel_initializer=glorot_normal()))
model.add(Activation('relu'))
model.add(BatchNormalization())
 model.add(Dense(all_data.shape[1]))
model.add(Activation('linear'))
opt = keras.optimizers.Adam(lr=0.0009)
 model.compile(loss='mse', optimizer=opt )
test = pd.read_csv('.. /input/test_V2.csv')
<categorify>
print('Training neural net')
 epochs = 15
chck = ModelCheckpoint('keras_dae.h5', monitor='loss', save_best_only=True)
 cb = [ EarlyStopping(monitor='loss', patience=100, verbose=2, min_delta=0), chck ]
model.fit(all_data_noise, all_data, batch_size=128, verbose=1, epochs=epochs, callbacks=cb)
 print('Applying neural net')
 train_data_transform = model.predict(train_data_orig)
 test_data_transform = model.predict(test_data_orig)
 print(train_data_transform.shape)
 print(test_data_transform.shape )
train_matchtype = pd.get_dummies(train['matchType'] )<categorify>
train_data = pd.DataFrame(data=train_data_transform, columns=features)
 test_data = pd.DataFrame(data=test_data_transform, columns=features )
test_matchtype = pd.get_dummies(test['matchType'] )<categorify>
train_data.to_csv('train_dae.csv',index=False)
 test_data.to_csv('test_dae.csv',index=False )
for i in ['assists', 'boosts', 'damageDealt', 'DBNOs',
'headshotKills', 'heals', 'killPlace', 'killPoints', 'kills',
'killStreaks', 'longestKill', 'matchDuration', 'maxPlace',
'numGroups', 'rankPoints', 'revives', 'rideDistance', 'roadKills',
'swimDistance', 'teamKills', 'vehicleDestroys', 'walkDistance',
'weaponsAcquired', 'winPoints']:
a='std'+i
print(a)
 test[a] = test.groupby(['groupId'])[i].transform('std')
 train[a] = train.groupby(['groupId'])[i].transform('std')
 test[a] = test[a].fillna(0)
 train[a] = train[a].fillna(0)
<categorify>
param = {
'bagging_freq': 5,
'bagging_fraction': 0.335,
'boost_from_average':'true',
'boost': 'gbdt',
'feature_fraction': 0.041,
'learning_rate': 0.0083,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1
}
for i in ['assists', 'boosts', 'damageDealt', 'DBNOs',
'headshotKills', 'heals', 'killPlace', 'killPoints', 'kills',
'killStreaks', 'longestKill', 'matchDuration', 'maxPlace',
'numGroups', 'rankPoints', 'revives', 'rideDistance', 'roadKills',
'swimDistance', 'teamKills', 'vehicleDestroys', 'walkDistance',
'weaponsAcquired', 'winPoints']:
a='Mean'+i
print(a)
 test[a] = test.groupby(['groupId'])[i].transform('mean')
 train[a] = train.groupby(['groupId'])[i].transform('mean')
 test[a] = test[a].fillna(0)
 train[a] = train[a].fillna(0)
<data_type_conversions>
folds = StratifiedKFold(n_splits=15, shuffle=False, random_state=2319)
 oof = np.zeros(len(train_data))
predictions = np.zeros(len(test_data))
feature_importance_df = pd.DataFrame()
print(train_data.shape)
 for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_data.values, target.values)) :
print("Fold {}".format(fold_))
trn_data = lgb.Dataset(train_data.iloc[trn_idx][features], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(train_data.iloc[val_idx][features], label=target.iloc[val_idx])
 num_round = 1000000
clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 4000)
 oof[val_idx] = clf.predict(train_data.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test_data[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)) )
<drop_column><EOS> 
sub_df = pd.DataFrame({"ID_code":test_id_code})
 sub_df["target"] = predictions
sub_df.to_csv("submission1.csv", index=False)