Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<define_search_space>
warnings.filterwarnings('ignore' )
lr = 3e-2<train_model>
train_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/train.csv')
 test_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/test.csv' )
learner.fit_one_cycle(6, max_lr = slice(lr))<predict_on_test>
preds,_ = learner.get_preds(ds_type=DatasetType.Test )<filter>
train = train_df.copy()
test = test_df.copy()
test_df.has_cactus = preds.numpy() [:, 0]<save_to_csv>
del train_df
del test_df
test_df.to_csv('submission.csv', index=False )<import_modules>
def augment(x,y,t=2):
xs,xn = [],[]
for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xs.append(x1)
 for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xn.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x,xs,xn])
 y = np.concatenate([y,ys,yn])
 return x,y
from keras.models import Sequential
from keras.layers import Conv2D, Dense, Flatten, BatchNormalization, Dropout, LeakyReLU, DepthwiseConv2D, Flatten
from keras.layers.pooling import GlobalAveragePooling2D
from keras.optimizers import Adam
from keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping
from sklearn.metrics import confusion_matrix, roc_auc_score, classification_report
from sklearn.model_selection import train_test_split
from tqdm import tqdm, tqdm_notebook
import numpy as np
import pandas as pd
import cv2 as cv
import seaborn as sns
import matplotlib.pyplot as plt
import os<load_from_csv>
features = [c for c in train.columns if c not in ['ID_code', 'target']]
target = train['target']
X_test = test[features].values
train_data = pd.read_csv('.. /input/train.csv' )<choose_model_class>
param = {
'bagging_freq': 5,
'bagging_fraction': 0.335,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.041,
'learning_rate': 0.0083,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1}
def create_model() :
model = Sequential()
model.add(Conv2D(3, kernel_size = 3, activation = 'relu', input_shape =(32, 32, 3)))
 model.add(Conv2D(filters = 16, kernel_size = 3, activation = 'relu'))
model.add(Conv2D(filters = 16, kernel_size = 3, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(DepthwiseConv2D(kernel_size = 3, strides = 1, padding = 'Same', use_bias = True))
model.add(Conv2D(filters = 32, kernel_size = 1, activation = 'relu'))
model.add(Conv2D(filters = 64, kernel_size = 1, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(DepthwiseConv2D(kernel_size = 3, strides = 2, padding = 'Same', use_bias = True))
model.add(Conv2D(filters = 128, kernel_size = 1, activation = 'relu'))
model.add(Conv2D(filters = 256, kernel_size = 1, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(DepthwiseConv2D(kernel_size = 3, strides = 1, padding = 'Same', use_bias = True))
model.add(Conv2D(filters = 256, kernel_size = 1, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 512, kernel_size = 1, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(DepthwiseConv2D(kernel_size = 3, strides = 2, padding = 'Same', use_bias = True))
model.add(Conv2D(filters = 512, kernel_size = 1, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 1024, kernel_size = 1, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(DepthwiseConv2D(kernel_size = 3, strides = 1, padding = 'Same', use_bias = True))
model.add(Conv2D(filters = 1024, kernel_size = 1, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 2048, kernel_size = 1, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(470, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(256, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(128, activation = 'tanh'))
model.add(Dense(1, activation = 'sigmoid'))
model.compile(optimizer = 'rmsprop', loss = 'mean_squared_error', metrics = ['accuracy'])
 return model<choose_model_class>
num_folds = 9
features = [c for c in train.columns if c not in ['ID_code', 'target']]
folds = KFold(n_splits=num_folds, random_state=2319)
 oof = np.zeros(len(train))
getVal = np.zeros(len(train))
predictions = np.zeros(len(target))
feature_importance_df = pd.DataFrame()
print('Light GBM Model')
 for fold_,(trn_idx, val_idx)in enumerate(folds.split(train.values, target.values)) :
X_train, y_train = train.iloc[trn_idx][features], target.iloc[trn_idx]
X_valid, y_valid = train.iloc[val_idx][features], target.iloc[val_idx]
X_tr, y_tr = augment(X_train.values, y_train.values)
 X_tr = pd.DataFrame(X_tr)
 print("Fold:{}".format(fold_ + 1))
trn_data = lgb.Dataset(X_tr, label=y_tr)
 val_data = lgb.Dataset(X_valid, label=y_valid)
 val_data = lgb.Dataset(train.iloc[val_idx][features], label=target.iloc[val_idx])
 clf = lgb.train(param, trn_data, 1000000, valid_sets = [trn_data, val_data], verbose_eval=5000, early_stopping_rounds = 4000)
 oof[val_idx] = clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 getVal[val_idx]+= clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)/ folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)) )
file_path = 'weights-aerial-cactus.h5'
callbacks = [
ModelCheckpoint(file_path, monitor = 'val_acc', verbose = 1, save_best_only = True, mode = 'max'),
ReduceLROnPlateau(monitor = 'val_loss', factor = 0.2, patience = 3, verbose = 1, mode = 'min', min_lr = 0.00001),
EarlyStopping(monitor = 'val_loss', min_delta = 1e-10, patience = 15, verbose = 1, restore_best_weights = True)
 ]
training_path = '.. /input/train/train/'
test_path = '.. /input/test/test/'<concatenate>
predictions['target'] = np.mean(predictions[[col for col in predictions.columns if col not in ['ID_code', 'target']]].values, axis=1)
 sub = pd.DataFrame({"ID_code": test.ID_code.values})
 sub["target"] = predictions['target']
sub.to_csv('submission_oof.csv', index=False )