Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<define_variables>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from pathlib import Path
data_folder = Path(".. /input")
<load_from_csv>
DATA_PATH = ".. /input/santander-customer-transaction-prediction/"
train = pd.read_csv(str(Path(DATA_PATH)/ "train.csv"))
test = pd.read_csv(str(Path(DATA_PATH)/ "test.csv"))
print("Train and test shapes", train.shape, test.shape )
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/sample_submission.csv" )<choose_model_class>
train.target.value_counts()
def md(f=None):
mdl = ptcv_get_model('condensenet74_c4_g4', pretrained=True)
 mdl.features.final_pool = nn.AvgPool2d(kernel_size=7, stride=1, padding=3)
 return mdl<train_model>
def woe(X, y):
tmp = pd.DataFrame()
tmp["variable"] = X
tmp["target"] = y
var_counts = tmp.groupby("variable")["target"].count()
var_events = tmp.groupby("variable")["target"].sum()
var_nonevents = var_counts - var_events
tmp["var_counts"] = tmp.variable.map(var_counts)
 tmp["var_events"] = tmp.variable.map(var_events)
 tmp["var_nonevents"] = tmp.variable.map(var_nonevents)
 events = sum(tmp["target"] == 1)
 nonevents = sum(tmp["target"] == 0)
 tmp["woe"] = np.log(((tmp["var_nonevents"])/nonevents)/(( tmp["var_events"])/events))
tmp["woe"] = tmp["woe"].replace(np.inf, 0 ).replace(-np.inf, 0)
 tmp["iv"] =(tmp["var_nonevents"]/nonevents - tmp["var_events"]/events)* tmp["woe"]
iv = tmp.groupby("variable")["iv"].last().sum()
return tmp["woe"], tmp["iv"], iv
lr = 3.5e-02
learn.fit_one_cycle(5, slice(lr))<predict_on_test>
iv_values = []
feats = ["var_{}".format(i)for i in range(200)]
y = train["target"]
for f in feats:
X = pd.qcut(train[f], 10, duplicates='drop')
 _, _, iv = woe(X, y)
 iv_values.append(iv)
 iv_inds = np.argsort(iv_values)[::-1][:50]
iv_values = np.array(iv_values)[iv_inds]
feats = np.array(feats)[iv_inds]
preds,_ = learn.get_preds(ds_type=DatasetType.Test )<filter>
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import StratifiedKFold, cross_val_predict
from sklearn.metrics import roc_auc_score
from sklearn.preprocessing import StandardScaler
test_df.has_cactus = preds.numpy() [:, 0]<save_to_csv>
feats = ["var_{}".format(i)for i in range(200)]
X = train[feats]
X_test = test[feats]
y = train["target"]
cvlist = list(StratifiedKFold(5, random_state=12345786 ).split(X, y))
scaler = StandardScaler()
X_sc = scaler.fit_transform(X)
 X_test_sc = scaler.fit_transform(X_test)
 lr = LogisticRegression()
y_preds_lr = cross_val_predict(lr, X_sc, y, cv=cvlist, method="predict_proba")[:, 1]
lr.fit(X_sc, y)
 y_test_preds_lr = lr.predict_proba(X_test_sc)[:, 1]
roc_auc_score(y, y_preds_lr )
test_df.to_csv('simple_fastai_submission.csv', index=False )<install_modules>
import lightgbm as lgb
!pip install pytorchcv<install_modules>
from scipy.stats import gmean
!pip install fastai==1.0.47<feature_engineering>
np.mean([0.9, 0.9, 0.9, 0.98, 0.9] )
start = time.time()<import_modules>
gmean([0.9, 0.9, 0.9, 0.98, 0.9] )
import numpy as np
import pandas as pd
from pytorchcv.model_provider import get_model as ptcv_get_model<import_modules>
!pip install -U lightgbm
from pathlib import Path
from fastai import *
from fastai.vision import *
import torch<define_variables>
model = lgb.LGBMClassifier(boosting_type='gbdt', n_estimators=200000, learning_rate=0.02, num_leaves=2, subsample=0.4, colsample_bytree=0.4, seed=1)
 y_preds_lgb = np.zeros(( len(y)))
 test_preds_allfolds = []
for i,(tr_idx, val_idx)in enumerate(cvlist):
X_dev, y_dev = X.iloc[tr_idx], y.iloc[tr_idx]
X_val, y_val = X.iloc[val_idx], y.iloc[val_idx]
model.fit(X_dev, y_dev, eval_set=[(X_val, y_val)], eval_metric="auc", verbose=5000, early_stopping_rounds=1000)
 val_preds = model.predict_proba(X_val)[:, 1]
test_preds = model.predict_proba(X_test)[:, 1]
test_preds_allfolds.append(test_preds)
 y_preds_lgb[val_idx] = val_preds
print("Score for fold {} is {}".format(i, roc_auc_score(y_val, val_preds)))
 print("Overall Score for oof predictions ", roc_auc_score(y, y_preds_lgb))
data_folder = Path(".. /input")
<load_from_csv>
sub = test[["ID_code"]]
sub["target"] = y_test_preds_lgb
sub.to_csv("submission_lgbm2_v1.csv", index=False )
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/sample_submission.csv" )<choose_model_class>
weighted_preds = y_preds_lr* 0.05 + y_preds_lgb * 0.95
weighted_test_preds = y_test_preds_lr* 0.05 + y_test_preds_lgb * 0.95
roc_auc_score(y, weighted_preds )
def md(f=None):
mdl = ptcv_get_model('condensenet74_c4_g4', pretrained=True)
 mdl.features.final_pool = nn.AvgPool2d(kernel_size=7, stride=1, padding=3)
 return mdl<choose_model_class>
public_sub = pd.read_csv(".. /input/santander-lgb-new-features-rank-mean-10-folds/submission_LGBM.csv")
 public_sub.head()
learn = cnn_learner(train_img, md, metrics=[error_rate, accuracy] )<train_model>
sub["target"] = weighted_test_preds
lr = 3.5e-02
learn.fit_one_cycle(5, slice(lr))<predict_on_test>
sub["target"] = 0.2*sub["target"].rank() + 0.8*public_sub["target"]
sub.to_csv("submission_blend.csv", index=False )
preds,_ = learn.get_preds(ds_type=DatasetType.Test )<filter>
concatenate, GaussianNoise, Reshape, TimeDistributed, LeakyReLU, PReLU, Embedding)
 class ROC_AUC(Callback):
def __init__(self, validation_data):
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
print("ROC AUC for this fold, is ", roc_auc_score(self.y_val, self.model.predict(X_val)))
 class NNv1(BaseEstimator, ClassifierMixin):
def __init__(self,
inp_shape=200,
gaussian_noise=0.01,
dense1_dim=32,
dense2_dim=32,
dense1_kwargs=None,
dense2_kwargs=None,
classifier_kwargs=None,
optimizer=SGD,
opt_kwargs=None, 
):
self.inp_shape = inp_shape
self.gaussian_noise = gaussian_noise
self.dense1_dim = dense1_dim
self.dense2_dim = dense2_dim
self.dense1_kwargs = dense1_kwargs
self.dense2_kwargs = dense2_kwargs
self.classifier_kwargs = classifier_kwargs
self.optimizer = optimizer
self.opt_kwargs = opt_kwargs
self._default_initiaization()
def _default_initiaization(self):
if self.dense1_kwargs is None:
self.dense1_kwargs = {"kernel_initializer": "glorot_uniform"}
if self.dense2_kwargs is None:
self.dense2_kwargs = {"kernel_initializer": "he_uniform"}
if self.classifier_kwargs is None:
self.classifier_kwargs = {"kernel_initializer": "he_uniform"}
if self.opt_kwargs is None:
self.opt_kwargs = {}
def _build_model(self):
inp = Input(shape=(self.inp_shape,))
x = Reshape(( self.inp_shape, 1))(inp)
 d1 = Dense(self.dense1_dim, activation='tanh', )(x)
 d2 = Dense(self.dense1_dim, activation='relu', )(x)
 x = concatenate([d1, d2])
 x = Flatten()(x)
 out = Dense(1, activation='sigmoid', **self.classifier_kwargs )(x)
 model = Model(inputs=inp, outputs=out)
 opt = self.optimizer(**self.opt_kwargs)
 model.compile(loss='binary_crossentropy', optimizer=opt)
 return model
def fit(self, X, y, *args, **kwargs):
self.model = self._build_model()
print(self.model.summary())
 self.model.fit(X, y, *args, **kwargs)
 return self
def predict(self, X, y=None, weight_path=None, **kwargs):
if self.model:
if weight_path is not None:
self.model.load_weights(weight_path)
 y_hat = self.model.predict(X, **kwargs)
 else:
raise ValueError("Model not fit yet")
 return y_hat
test_df.has_cactus = preds.numpy() [:, 0]<save_to_csv>
model = NNv1(opt_kwargs = {"lr": 0.01, "momentum": 0.9, "nesterov": True, "clipnorm": 1})
 y_preds_nn = np.zeros(( len(y)))
 for tr_idx, val_idx in cvlist:
X_dev, y_dev = X_sc[tr_idx], y.iloc[tr_idx]
X_val, y_val = X_sc[val_idx], y.iloc[val_idx]
roc_auc = ROC_AUC(( X_val, y_val))
model.fit(X_dev, y_dev, validation_data=(X_val, y_val), epochs=40, batch_size=256, verbose=0, callbacks=[roc_auc])
 val_preds = model.predict(X_val, batch_size=5000)
 y_preds_nn[val_idx] = val_preds.flatten()
<import_modules><EOS> 
roc_auc_score(y, y_preds_nn )