Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<find_best_params>
warnings.filterwarnings('ignore')
 %matplotlib inline
plt.style.use('seaborn')
 random_state = 42
np.random.seed(random_state )
learn.lr_find() 
<train_model>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 features = [c for c in train.columns if c not in ['ID_code', 'target']]
lr = 1e-02
learn.fit_one_cycle(3, slice(lr)) 
<predict_on_test>
obs = train.isnull().sum().sort_values(ascending = False)
 percent = round(train.isnull().sum().sort_values(ascending = False)/len(train)*100, 2)
 pd.concat([obs, percent], axis = 1,keys= ['Number of Observations', 'Percent'] )
preds,_ = learn.get_preds(ds_type=DatasetType.Test )<filter>
lin_pca = KernelPCA(n_components = 2, kernel="linear", fit_inverse_transform=True)
 rbf_pca = KernelPCA(n_components = 2, kernel="rbf", gamma=0.0433, fit_inverse_transform=True)
 sig_pca = KernelPCA(n_components = 2, kernel="sigmoid", gamma=0.001, coef0=1, fit_inverse_transform=True)
 plt.figure(figsize=(11, 4))
for subplot, pca, title in(( 131, lin_pca, "Linear kernel"),(132, rbf_pca, "RBF kernel, $\gamma=0.04$"), 
(133, sig_pca, "Sigmoid kernel, $\gamma=10^{-3}, r=1$")) :
PCA_train_x = PCA(2 ).fit_transform(train_scaled)
 plt.subplot(subplot)
 plt.title(title, fontsize=14)
 plt.scatter(PCA_train_x[:, 0], PCA_train_x[:, 1], c=target, cmap="nipy_spectral_r")
 plt.xlabel("$z_1$", fontsize=18)
 if subplot == 131:
plt.ylabel("$z_2$", fontsize=18, rotation=0)
 plt.grid(True)
 plt.show()
test.has_cactus = preds.numpy() [:, 0]<save_to_csv>
def augment(x,y,t=2):
xs,xn = [],[]
for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xs.append(x1)
 for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xn.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x,xs,xn])
 y = np.concatenate([y,ys,yn])
 return x,y
test.to_csv('submission.csv', index=False )<feature_engineering>
param = {
"objective" : "binary",
"metric" : "auc",
"boosting": 'gbdt',
"max_depth" : -1,
"num_leaves" : 31,
"learning_rate" : 0.01,
"bagging_freq": 5,
"bagging_fraction" : 0.4,
"feature_fraction" : 0.05,
"min_data_in_leaf": 150,
"min_sum_heassian_in_leaf": 10,
"tree_learner": "serial",
"boost_from_average": "false",
"bagging_seed" : random_state,
"verbosity" : 1,
"seed": random_state}
start = time.time() 
<define_variables>
num_folds = 11
features = [c for c in train.columns if c not in ['ID_code', 'target']]
folds = StratifiedKFold(n_splits=num_folds, shuffle=False, random_state=2319)
 oof = np.zeros(len(train))
getVal = np.zeros(len(train))
predictions = np.zeros(len(target))
feature_importance_df = pd.DataFrame()
print('Light GBM Model')
 for fold_,(trn_idx, val_idx)in enumerate(folds.split(train.values, target.values)) :
X_train, y_train = train.iloc[trn_idx][features], target.iloc[trn_idx]
X_valid, y_valid = train.iloc[val_idx][features], target.iloc[val_idx]
X_tr, y_tr = augment(X_train.values, y_train.values)
 X_tr = pd.DataFrame(X_tr)
 print("Fold idx:{}".format(fold_ + 1))
trn_data = lgb.Dataset(X_tr, label=y_tr)
 val_data = lgb.Dataset(X_valid, label=y_valid)
 clf = lgb.train(param, trn_data, 1000000, valid_sets = [trn_data, val_data], verbose_eval=5000, early_stopping_rounds = 4000)
 oof[val_idx] = clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 getVal[val_idx]+= clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)/ folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)) )
data_folder = Path(".. /input" )<load_from_csv>
num_sub = 34
print('Saving the Submission File')
 sub = pd.DataFrame({"ID_code": test.ID_code.values})
 sub["target"] = predictions
sub.to_csv('submission{}.csv'.format(num_sub), index=False)
 getValue = pd.DataFrame(getVal)
 getValue.to_csv("Validation.csv" )