Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<train_model>
import os
import gc
import sys
import time
import shutil
import feather
import numpy as np
import pandas as pd
from scipy.stats import norm, rankdata
from sklearn.pipeline import Pipeline
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import roc_curve, auc, roc_auc_score
def fit_predict_step(estimator, x_train, y_train, train_idx, valid_idx, x_test, oof):
x_train_train = x_train[train_idx]
y_train_train = y_train[train_idx]
x_train_valid = x_train[valid_idx]
y_train_valid = y_train[valid_idx]
estimator.fit(x_train_train, y_train_train, x_train_valid, y_train_valid)
 oof_part = estimator.predict(x_train_valid)
 print('MAE:', mean_absolute_error(y_train_valid, oof_part))
oof[valid_idx] = oof_part
y_part = estimator.predict(x_test)
 return y_part<predict_on_test>
path_train = '.. /input/train.feather'
path_test = '.. /input/test.feather'
print("Reading train data...")
 start = time.time()
train = pd.read_csv('.. /input/train.csv')
 end = time.time()
print("It takes {0:.2f} seconds to read 'train.csv'.".format(end - start))
def fit_predict(estimator, x_train, y_train, x_test):
oof = np.zeros(x_train.shape[0])
 y = np.zeros(x_test.shape[0])
 kf = KFold(n_splits=5, random_state=42)
 for train_idx, valid_idx in kf.split(x_train):
y_part = fit_predict_step(estimator, x_train, y_train, train_idx, valid_idx, x_test, oof)
 y += y_part / kf.n_splits
print('Final MAE:', mean_absolute_error(y_train, oof))
return oof, y<train_model>
start = time.time()
print("Reading test data...")
 test = pd.read_csv('.. /input/test.csv')
 end = time.time()
print("It takes {0:.2f} seconds to read 'test.csv'.".format(end - start))
def fit_step(estimator, x_train, y_train, train_idx, valid_idx, oof):
x_train_train = x_train[train_idx]
y_train_train = y_train[train_idx]
x_train_valid = x_train[valid_idx]
y_train_valid = y_train[valid_idx]
estimator.fit(x_train_train, y_train_train, x_train_valid, y_train_valid)
 oof_part = estimator.predict(x_train_valid)
 mae = mean_absolute_error(y_train_valid, oof_part)
 print('MAE:', mae)
 oof[valid_idx] = oof_part
return estimator, mae<train_model>
target = train.pop('target')
 train_ids = train.pop('ID_code')
 test_ids = test.pop('ID_code' )
def fit(estimator, x_train, y_train):
oof = np.zeros(x_train.shape[0])
 kf = KFold(n_splits=5, random_state=42)
 trained_estimators = []
for train_idx, valid_idx in kf.split(x_train):
e, mae = fit_step(estimator, x_train, y_train, train_idx, valid_idx, oof)
 trained_estimators.append(deepcopy(e))
print('Final MAE:', mean_absolute_error(y_train, oof))
return oof, trained_estimators<predict_on_test>
len_train = len(train )
def predict(trained_estimators, x_test):
y = np.zeros(x_test.shape[0])
 for estimator in trained_estimators:
y_part = estimator.predict(x_test)
 y += y_part / len(trained_estimators)
 return y<train_model>
merged = pd.concat([train, test] )
def pipeline_fit(estimator, df_train, scaler=None):
add_new_features_4(df_train)
 x_train, y_train = feature_engineering(df_train, is_train=True)
 x_train = reduce_mem_usage(x_train)
 gc.collect()
if not(scaler is None):
scaler.fit(x_train)
 scaled_x_train = scaler.transform(x_train)
 else:
scaled_x_train = x_train.values
del x_train
gc.collect()
oof, trained_estimators = fit(estimator, scaled_x_train, y_train.values)
 del scaled_x_train
del y_train
gc.collect()
return oof, trained_estimators<find_best_model_class>
del test, train
gc.collect()
def pipeline_predict(trained_estimators, df_test, scaler=None):
add_new_features_4(df_test)
 x_test, _ = feature_engineering(df_test, is_train=False)
 x_test = reduce_mem_usage(x_test)
 gc.collect()
if not(scaler is None):
scaled_x_test = scaler.transform(x_test)
 else:
scaled_x_test = x_test.values
del x_test
gc.collect()
y = predict(trained_estimators, scaled_x_test)
 del scaled_x_test
gc.collect()
return y<load_from_csv>
for col in merged.columns:
merged[col] =(( merged[col] - merged[col].mean())
 / merged[col].std() ).astype('float32')
 merged[col+'^2'] = merged[col] * merged[col]
merged[col+'^3'] = merged[col] * merged[col] * merged[col]
merged[col+'^4'] = merged[col] * merged[col] * merged[col] * merged[col]
merged[col+'_cp'] = rankdata(merged[col] ).astype('float32')
 merged[col+'_cnp'] = norm.cdf(merged[col] ).astype('float32' )
df_train = pd.read_csv('.. /input/train_V2.csv', index_col='Id')
 df_train.shape<feature_engineering>
new_features = set(merged.columns)- set(original_features )
df_train = reduce_mem_usage(df_train )<set_options>
for col in new_features:
merged[col] =(( merged[col] - merged[col].mean())
 / merged[col].std() ).astype('float32' )
gc.collect()<drop_column>
del target, train_ids, test_ids, merged
gc.collect()
df_train.drop(df_train[df_train['winPlacePerc'].isnull() ].index, inplace=True )<import_modules>
path_target = 'target.feather'
path_train_ids = 'train_ids_extra_features.feather'
path_test_ids = 'test_ids_extra_features.feather'
path_train = 'train_extra_features.feather'
path_test = 'test_extra_features.feather'
print("Reading target")
 start = time.time()
y = feather.read_dataframe(path_target ).values.ravel()
end = time.time()
print("{0:5f} sec".format(end - start))
import lightgbm as lgb<train_model>
print("Reading train_ids")
 start = time.time()
train_ids = feather.read_dataframe(path_train_ids ).values.ravel()
end = time.time()
print("{0:5f} sec".format(end - start))
class LightGBM(Estimator):
def __init__(self, params):
self.params = params
def fit(self, x_train, y_train, x_valid, y_valid):
lgb_train = lgb.Dataset(data=x_train.astype('float32'), label=y_train.astype('float32'))
lgb_valid = lgb.Dataset(data=x_valid.astype('float32'), label=y_valid.astype('float32'))
self.lgb_model = lgb.train(self.params, lgb_train, valid_sets=lgb_valid, verbose_eval=1000)
 def predict(self, x):
return self.lgb_model.predict(x.astype('float32'), num_iteration=self.lgb_model.best_iteration )<init_hyperparams>
print("Reading test_ids")
 start = time.time()
test_ids = feather.read_dataframe(path_test_ids ).values.ravel()
end = time.time()
print("{0:5f} sec".format(end - start))
params = {'objective': 'regression',
'metric': 'mae',
'n_estimators': 10000,
'early_stopping_rounds': 100,
'num_leaves': 300,
'max_depth': 14,
'bagging_fraction': 0.9,
'learning_rate': 0.03,
'bagging_seed': 0,
'num_threads': 4,
'colsample_bytree': 0.7}<train_model>
print("Reading training data")
 start = time.time()
train = feather.read_dataframe(path_train)
 end = time.time()
print("{0:5f} sec".format(end - start))
%%time
oof, trained_estimators = pipeline_fit(LightGBM(params), df_train )<set_options>
print("Reading testing data")
 start = time.time()
test = feather.read_dataframe(path_test)
 end = time.time()
print("{0:5f} sec".format(end - start))
del df_train
gc.collect()<load_from_csv>
NFOLDS = 5
RANDOM_STATE = 871972
feature_list = train.columns
test = test[feature_list]
X = train.values.astype('float32')
 X_test = test.values.astype('float32')
 folds = StratifiedKFold(n_splits=NFOLDS, shuffle=True,
random_state=RANDOM_STATE)
 oof_preds = np.zeros(( len(train), 1))
test_preds = np.zeros(( len(test), 1))
roc_cv =[]
del train, test
gc.collect()
df_test = pd.read_csv('.. /input/test_V2.csv', index_col = 'Id')
 df_test.shape<feature_engineering>
def augment(x,y,t=2):
if t==0:
return x, y
xs,xn = [],[]
for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xs.append(x1)
 del x1
gc.collect()
for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xn.append(x1)
 del x1
gc.collect()
print("The sizes of x, xn, and xs are {}, {}, {}, respectively.".format(sys.getsizeof(x),
sys.getsizeof(xn),
sys.getsizeof(xs)
)
)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 print("The sizes of x, xn, and xs are {}, {}, {}, respectively.".format(sys.getsizeof(x)/1024**3,
sys.getsizeof(xn),
sys.getsizeof(xs)
)
)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 y = np.concatenate([y,ys,yn])
 print("The sizes of y, yn, and ys are {}, {}, {}, respectively.".format(sys.getsizeof(y),
sys.getsizeof(yn),
sys.getsizeof(ys)
)
)
 gc.collect()
return np.vstack([x,xs, xn]), y
df_test = reduce_mem_usage(df_test )<create_dataframe>
for fold_,(trn_, val_)in enumerate(folds.split(y, y)) :
print("Current Fold: {}".format(fold_))
trn_x, trn_y = X[trn_, :], y[trn_]
val_x, val_y = X[val_, :], y[val_]
NAUGMENTATIONS=1
NSHUFFLES=0
val_pred, test_fold_pred = 0, 0
for i in range(NAUGMENTATIONS):
print("
Fold {}, Augmentation {}".format(fold_, i+1))
trn_aug_x, trn_aug_y = augment(trn_x, trn_y, NSHUFFLES)
 trn_aug_x = pd.DataFrame(trn_aug_x)
 trn_aug_x = trn_aug_x.add_prefix('var_')
 clf = Pipeline([ 
('lr_clf', LogisticRegression(solver='lbfgs', max_iter=1500, C=10))
])
 clf.fit(trn_aug_x, trn_aug_y)
 print("Making predictions for the validation data")
 val_pred += clf.predict_proba(val_x)[:,1]
print("Making predictions for the test data")
 test_fold_pred += clf.predict_proba(X_test)[:,1]
val_pred /= NAUGMENTATIONS
test_fold_pred /= NAUGMENTATIONS
roc_cv.append(roc_auc_score(val_y, val_pred))
print("AUC = {}".format(roc_auc_score(val_y, val_pred)))
 oof_preds[val_, :] = val_pred.reshape(( -1, 1))
test_preds += test_fold_pred.reshape(( -1, 1))
df_test_id = pd.DataFrame(index=df_test.index )<set_options>
test_preds /= NFOLDS
gc.collect()<predict_on_test>
roc_score_1 = round(roc_auc_score(y, oof_preds.ravel()), 5)
 roc_score = round(sum(roc_cv)/len(roc_cv), 5)
 st_dev = round(np.array(roc_cv ).std() , 5)
 print("Average of the folds' AUCs = {}".format(roc_score))
print("Combined folds' AUC = {}".format(roc_score_1))
print("The standard deviation = {}".format(st_dev))
<set_options><EOS> 
print("Saving submission file")
 sample = pd.read_csv('.. /input/sample_submission.csv')
 sample.target = test_preds.astype(float)
 sample.ID_code = test_ids
sample.to_csv('submission.csv', index=False )