Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<choose_model_class>
!pip install snapml
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1)
<choose_model_class>
import numpy as np
import pandas as pd
from sklearn import decomposition
from sklearn.metrics import roc_auc_score, roc_curve
from sklearn.model_selection import StratifiedKFold
from sklearn.utils.class_weight import compute_sample_weight
import sklearn.ensemble as skl
from os import path
import time
from scipy.stats import normaltest
from scipy.special import logit
import seaborn as sns
from matplotlib import pyplot as plt
import snapml
import multiprocessing
from typing import List, Dict, Union
from tqdm import tqdm
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )<compute_test_metric>
np.random.seed(130720 )
score = rmsle_cv(lasso)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
path_to_folder = '/kaggle/input/santander-customer-transaction-prediction'
df = pd.read_csv(path.join(path_to_folder, 'train.csv'))
df_test = pd.read_csv(path.join(path_to_folder, 'test.csv'))
id_codes = df_test['ID_code']
df_test.drop('ID_code', axis=1, inplace=True)
 df.drop('ID_code', axis=1, inplace=True)
 features = df.drop('target', axis=1)
 columns = features.columns
target = df.target
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
def time_decorator(function):

 def timed(*args, **kwargs):
ts = time.time_ns()
result = function(*args, **kwargs)
 te = time.time_ns()
return result,(te - ts)* 1e-9
return timed
score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
def get_real_synthetic_testing_samples(testing_set: pd.DataFrame)-> tuple:

 counts = np.zeros(shape=testing_set.shape)
 for i, col in enumerate(testing_set.columns):
counts[:, i] = testing_set[col].map(testing_set[col].value_counts())
 is_row_real_sample = np.any(counts == 1, axis=1)
 fake_samples_indices = np.argwhere(~is_row_real_sample ).flatten()
real_samples_indices = np.argwhere(is_row_real_sample ).flatten()
return real_samples_indices, fake_samples_indices
score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
def add_features_in_batch(data_set_for_calculation: pd.DataFrame, all_dataframes: List, columns: np.array, decimal_places: Dict)-> List:

 for col in tqdm(columns):
for feature in decimal_places.keys() :
x = get_aggregated_features(data_set_for_calculation[col], col, decimal_places[feature], feature)
 for dataframe in all_dataframes:
dataframe[f'{col}_{feature}'] = dataframe[col].round(decimal_places[feature] ).map(x)
 if feature == 'std':
dataframe.fillna(value=0, inplace=True)
 return all_dataframes
score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
def estimate_counts_based_on_real_testing_samples(training_and_validation_sets: List[pd.DataFrame], testing_set: pd.DataFrame, columns: List)-> List:

 real_samples_indices, _ = get_real_synthetic_testing_samples(testing_set)
 data_set_for_estimation = pd.concat([*training_and_validation_sets, testing_set.loc[real_samples_indices]])
 all_dataframes = [*[d.copy(deep=True)for d in training_and_validation_sets], testing_set.copy(deep=True)]
decimals = {
'count': 4,
'mean': 4,
'std': 4,
'sum': 4,
'min': 2,
'max': 3
}
args = []
n_processes = 10
columns = np.array(columns)
 batch_size = int(np.ceil(len(columns)/ n_processes))
for i in range(n_processes):
indices = list(range(batch_size * i, batch_size *(i + 1)))
 args.append(( data_set_for_estimation.iloc[:, indices], [dataframe.iloc[:, indices] for dataframe in all_dataframes], columns[indices], decimals))
with multiprocessing.Pool(processes=n_processes)as p:
chunks = p.starmap(add_features_in_batch, args)
 all_dataframes = [pd.concat(d, axis=1)for d in zip(*chunks)]
return all_dataframes
score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<predict_on_test>
def get_column_indices(df: pd.DataFrame, query_cols: List)-> np.array:

 cols = df.columns.values
sidx = np.argsort(cols)
 return sidx[np.searchsorted(cols, query_cols, sorter=sidx)]
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )<compute_train_metric>
@time_decorator
def train(model: Union[snapml.RandomForestClassifier, skl.RandomForestClassifier], X_train: np.ndarray, y_train: np.array, weights: np.array):

 model.fit(X_train, y_train, weights )
averaged_models = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<train_model>
def create_model(library: str, params: Dict, r_seed: int, n_cpus: int)-> Union[snapml.RandomForestClassifier, skl.RandomForestClassifier]:

 if library == 'snapml':
return snapml.RandomForestClassifier(**params, use_gpu=False, random_state=r_seed, n_jobs=n_cpus)
 elif library == 'sklearn':
return skl.RandomForestClassifier(**params, random_state=r_seed, n_jobs=n_cpus )
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )<compute_train_metric>
print(f'Number of NaN entries in train: {sum(df.isnull().sum())}, test: {sum(df_test.isnull().sum())}.' )
stacked_averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR),
meta_model = lasso)
 score = rmsle_cv(stacked_averaged_models)
 print("Stacking Averaged models score: {:.4f}({:.4f})".format(score.mean() , score.std()))<compute_test_metric>
corr = np.tril(features.corr() , k=-1)
 print(f'Maximal correlation: {corr.max() :.4f}, minimal correlation {corr.min() :.4f}.' )
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))<predict_on_test>
X, X_test = estimate_counts_based_on_real_testing_samples([features], df_test, columns )
stacked_averaged_models.fit(train.values, y_train)
 stacked_train_pred = stacked_averaged_models.predict(train.values)
 stacked_pred = np.expm1(stacked_averaged_models.predict(test.values))
print(rmsle(y_train, stacked_train_pred))<predict_on_test>
optimal_hyperparams = {
'max_depth': 5,
'min_samples_leaf': 386,
'n_estimators': 88
}
model_xgb.fit(train, y_train)
 xgb_train_pred = model_xgb.predict(train)
 xgb_pred = np.expm1(model_xgb.predict(test))
print(rmsle(y_train, xgb_train_pred))<predict_on_test>
features = [get_column_indices(X, X.filter(regex=fr'{col}(?!\d)' ).columns)for col in columns]
X = X.to_numpy()
X_test = X_test.to_numpy()
y = target.to_numpy()
model_lgb.fit(train, y_train)
 lgb_train_pred = model_lgb.predict(train)
 lgb_pred = np.expm1(model_lgb.predict(test.values))
print(rmsle(y_train, lgb_train_pred))<compute_test_metric>
n_cpus = multiprocessing.cpu_count()
print('RMSLE score on train data:')
 print(rmsle(y_train,stacked_train_pred*0.70 +
xgb_train_pred*0.15 + lgb_train_pred*0.15))<define_variables>
verbose = True
k = 5
y_pred_logit_snapml = np.zeros(X_test.shape[0])
 validation_auc = 0
snapml_fit_time = 0
cv = StratifiedKFold(n_splits=k, shuffle=True, random_state=1)
 for n_fold,(train_indices, val_indices)in enumerate(cv.split(X, y), start=1):
print(f'{n_fold}.fold(out of {k})is running.')
 X_train = X[train_indices, :]
X_val = X[val_indices, :]
y_train = y[train_indices]
y_val = y[val_indices]
w_train = compute_sample_weight('balanced', y_train)
 tmp_test = np.zeros(shape=X_test.shape[0])
 tmp_val = np.zeros(shape=X_val.shape[0])
 for idx in tqdm(range(len(columns))):
rf = create_model('snapml', optimal_hyperparams, idx, n_cpus)
 _, t_fit = train(rf, X_train[:, features[idx]], y_train, w_train)
 snapml_fit_time += t_fit
tmp_val += logit(rf.predict_proba(X_val[:, features[idx]])[:, 1])
 tmp_test += logit(rf.predict_proba(X_test[:, features[idx]])[:, 1])
 y_pred_logit_snapml += tmp_test
validation_auc += roc_auc_score(y_val, tmp_val)
 if verbose:
plot_roc_curve(y_val, tmp_val)
 print(f'The average AUC estimated from {k} validation splits is: {validation_auc / k:.4f}.' )
ensemble = stacked_pred*0.70 + xgb_pred*0.15 + lgb_pred*0.15<save_to_csv>
verbose = True
k = 5
y_pred_logit_skl = np.zeros(X_test.shape[0])
 validation_auc = 0
sklearn_fit_time = 0
cv = StratifiedKFold(n_splits=k, shuffle=True, random_state=1)
 for n_fold,(train_indices, val_indices)in enumerate(cv.split(X, y), start=1):
print(f'{n_fold}.fold(out of {k})is running.')
 X_train = X[train_indices, :]
X_val = X[val_indices, :]
y_train = y[train_indices]
y_val = y[val_indices]
w_train = compute_sample_weight('balanced', y_train)
 tmp_test = np.zeros(shape=X_test.shape[0])
 tmp_val = np.zeros(shape=X_val.shape[0])
 for idx in tqdm(range(len(columns))):
rf = create_model('sklearn', optimal_hyperparams, idx, n_cpus)
 _, t_fit = train(rf, X_train[:, features[idx]], y_train, w_train)
 sklearn_fit_time += t_fit
tmp_val += logit(rf.predict_proba(X_val[:, features[idx]])[:, 1])
 tmp_test += logit(rf.predict_proba(X_test[:, features[idx]])[:, 1])
 y_pred_logit_skl += tmp_test
validation_auc += roc_auc_score(y_val, tmp_val)
 if verbose:
plot_roc_curve(y_val, tmp_val)
 print(f'The average AUC estimated from {k} validation splits is: {validation_auc / k:.4f}.' )
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = ensemble
sub.to_csv('submission.csv',index=False )<set_options>
print(f'[scikit-learn] training time: {sklearn_fit_time:.2f}(s)')
 print(f'[Snap ML] training time: {snapml_fit_time:.2f}(s)')
 print(f'Speed-up: {sklearn_fit_time / snapml_fit_time:.2f}x' )
%matplotlib inline<load_from_csv>
filepath = 'submission.csv'
<count_values><EOS> 
submission = pd.DataFrame({
"ID_code": id_codes,
"target": y_pred_logit_snapml
})
 submission.to_csv(filepath, index=False )