Kaggle competition: Santander Customer Transaction Prediction
ts=time.time()
xgbreg=XGBRegressor(**xgb_params_0)
 xgbreg.fit(X_train,Y_train,eval_set=eval_set,eval_metric='rmse',verbose=False)
 print(time.time() -ts )<compute_test_metric>
import pandas as pd
import numpy as np
from os import listdir
from os.path import isfile, join
from IPython.display import display
from sklearn.ensemble import RandomForestClassifier
from xgboost import XGBClassifier
import warnings
import random as rnd
from sklearn import preprocessing
from sklearn.cluster import MiniBatchKMeans
from sklearn.decomposition import PCA
from sklearn.model_selection import KFold
import xgboost as xgb
import lightgbm as lgb
from sklearn import metrics
from sklearn.model_selection import train_test_split
import scipy
import matplotlib.pyplot as plt
from scipy.cluster import hierarchy as hc
performance_analysis_test(xgbreg,rmse_guess_train,rmse_base_train,rmse_guess_evalset,rmse_base_evalset )<load_pretrained>
path =".. /input"
create_directory(os.path.join(DATA_FOLDER, 'predictions/models/xgbreg_seniority0'))
pickle.dump(xgbreg, open(os.path.join(DATA_FOLDER,'predictions/models/xgbreg_seniority0/model.pickle'), 'wb'))
X_test.to_pickle(os.path.join(DATA_FOLDER,'predictions/models/xgbreg_seniority0/X_test.pkl'))<drop_column>
allfiles = [f for f in listdir(path)if isfile(join(path, f)) if(not f.endswith('.ipynb')) ]
allfiles
del X_train, Y_train, X_test
del eval_set
del rmse_guess_train, rmse_base_train, rmse_guess_evalset, rmse_base_evalset
del xgbreg
del features_to_keep_0, features_to_discard_0
del xgb_params_0
loaded.remove('features_to_keep_0')
 loaded.remove('features_to_discard_0')
 loaded.remove('xgb_params_0')
 gc.collect()<define_variables>
def print_size(file):
statinfo=os.stat(join(path,file))
print('Size of file {} in Bytes is {} bytes'.format(str(file),str(statinfo.st_size)) )
reset_variable_space<load_pretrained>
def load_df(df,**kwds):
if str(df ).endswith('.csv'):
ndf =pd.read_csv(str(df),**kwds)
 else:
print('Could not find the df')
 print('Loaded file {} with shape{}'.format(df,ndf.shape))
return ndf
dataset_name='train_1_pred'
month_id_first=18
train_X=pd.read_pickle(os.path.join(DATA_FOLDER,'training/'+dataset_name+'.pkl'))
print(train_X.info(null_counts=True,verbose=True))<split>
test =load_df(join(path,allfiles[0]))
train =load_df(join(path,allfiles[1]))
submission =load_df(join(path,allfiles[2]))
rmse_guess_train, rmse_base_train, rmse_guess_evalset, rmse_base_evalset = target_analysis_test(train_X,month_id_first=18 )<train_model>
train.target.value_counts()
ts=time.time()
xgbreg=XGBRegressor(**xgb_params_1)
 xgbreg.fit(X_train,Y_train,eval_set=eval_set,eval_metric='rmse',verbose=False)
 print(time.time() -ts )<compute_test_metric>
train_df =train.drop(['ID_code','target'],axis =1 )
performance_analysis_test(xgbreg,rmse_guess_train,rmse_base_train,rmse_guess_evalset,rmse_base_evalset )<load_pretrained>
train_df.isnull().sum().sort_index() /len(train_df )
create_directory(os.path.join(DATA_FOLDER, 'predictions/models/xgbreg_seniority1'))
pickle.dump(xgbreg, open(os.path.join(DATA_FOLDER,'predictions/models/xgbreg_seniority1/model.pickle'), 'wb'))
X_test.to_pickle(os.path.join(DATA_FOLDER,'predictions/models/xgbreg_seniority1/X_test.pkl'))<drop_column>
def auc(x,y): return metrics.roc_auc_score(x,y)
 def print_score(m):
res = [auc(y_train,m.predict_proba(X_train)[:,1]),auc(y_valid,m.predict_proba(X_valid)[:,1]),
m.score(X_train, y_train), m.score(X_valid, y_valid)]
print(res )
del X_train, Y_train, X_test
del eval_set
del rmse_guess_train, rmse_base_train, rmse_guess_evalset, rmse_base_evalset
del xgbreg
del features_to_keep_1, features_to_discard_1
del xgb_params_1
loaded.remove('features_to_keep_1')
 loaded.remove('features_to_discard_1')
 loaded.remove('xgb_params_1')
 gc.collect()<define_variables>
df_trn, y_trn =train_df, train['target']
df_tst = test.drop(['ID_code'], axis=1)
 X_train, X_valid,y_train, y_valid = train_test_split(df_trn,y_trn, test_size =.20, stratify =y_trn,random_state =2019)
 X_train.shape, y_train.shape, X_valid.shape
reset_variable_space<load_pretrained>
m = RandomForestClassifier(n_estimators=10, n_jobs=8,min_samples_leaf=3, max_features=0.5)
 m.fit(X_train, y_train)
 print_score(m )
dataset_name='train_2_pred'
month_id_first=18
train_X=pd.read_pickle(os.path.join(DATA_FOLDER,'training/'+dataset_name+'.pkl'))
print(train_X.info(null_counts=True,verbose=True))<split>
fi = rf_feat_importance(m, df_trn); fi[:30]
rmse_guess_train, rmse_base_train, rmse_guess_evalset, rmse_base_evalset = target_analysis_test(train_X,month_id_first=18 )<train_model>
to_keep = fi[fi.imp>0.005].cols; len(to_keep )
train_X.drop(features_to_discard_2,axis=1,inplace=True)
 train_X['item_quantity_extralin2'].fillna(train_X['item_quantity_lag_1'],inplace=True)
 train_X['item_quantity_extralin3'].fillna(train_X['item_quantity_lag_1'],inplace=True)
 train_X.fillna(-1,inplace=True)
(X_train,Y_train,X_test)= datasplit_train_test(train_X,month_id_first)
 eval_set = [(X_train,Y_train)]+datasplit_evalset(train_X,month_id_first)
 print('number of features to keep : '+str(len(features_to_keep_2)))
 print('number of features kept : '+str(len(X_train.columns)))
 print(X_train.info(null_counts=True,verbose=True))
del train_X
gc.collect()<train_model>
df_keep = df_trn[to_keep].copy()
X_train, X_valid,y_train, y_valid = train_test_split(df_trn,y_trn, test_size =.20, stratify =y_trn,random_state =2019 )
ts=time.time()
xgbreg=XGBRegressor(**xgb_params_2)
 xgbreg.fit(X_train,Y_train,eval_set=eval_set,eval_metric='rmse',verbose=False)
 print(time.time() -ts )<compute_test_metric>
class Model(object):
def __init__(self,trn,tst,tget,subm):
self.trn = trn
self.tst = tst
self.tget = tget
self.subm =subm
def predict(self, path, model):
X_train, X_val, y_train, y_val = self.kfold_cv()
y_pred = model(X_train, y_train, X_val, y_val)
 self.save_pred(y_pred, path)
 def kfold_cv(self):
X = self.trn.values
y = self.tget.values
kf = KFold(n_splits=5, random_state=2019)
 kf.get_n_splits(X)
 for train_index, val_index in kf.split(X):
X_train, X_val = X[train_index], X[val_index]
y_train, y_val = y[train_index], y[val_index]
return X_train, X_val, y_train, y_val
def xgb(self, X_train, y_train, X_val, y_val):
dtrain = xgb.DMatrix(X_train, label = y_train)
 dvalid = xgb.DMatrix(X_val, label = y_val)
 dtest = xgb.DMatrix(self.tst.values)
 dlist = [(dtrain, 'train'),(dvalid, 'valid')]
parameters = {'eta': 0.025,
'colsample_bytree': 0.7,
'max_depth': 4,
'subsample': 0.7,
'nthread': -1,
'booster' : 'gbtree',
'colsample_bytree': 0.7,
'min_child_weight': 50,
'colsample_bylevel': 0.7,
'lambda' :1,
'alpha':0,
'eval_metric' : "auc",
'silent': 1,
'objective': 'binary:logistic'}
model = xgb.train(parameters, dtrain, 20000, dlist, early_stopping_rounds=300,
verbose_eval=100)
 print("Start prediction...")
 y_pred = model.predict(dtest)
 return y_pred
def lgb(self, X_train, y_train, X_val, y_val):
dtrain = lgb.Dataset(X_train, label=y_train)
 dvalid = lgb.Dataset(X_val, label=y_val)
 dtest = self.tst.values
dlist = [dtrain, dvalid]
parameters = {'learning_rate': 0.025,
'colsample_bytree': 0.3,
'max_depth': 6,
'subsample': 0.9,
'num_threads': -1,
'boosting_type' : 'gbdt',
'metric' : 'auc',
'objective': 'binary',
'num_leaves' : 20,
'sub_feature':0.7,
'sub_row' :0.7,
'bagging_freq':1,
'lambda_l1':5,
'lambda_l2': 5}
model = lgb.train(parameters, dtrain, 20000, dlist,early_stopping_rounds=300, verbose_eval=100)
 print("Start prediction...")
 y_pred = model.predict(dtest)
 return y_pred
def save_pred(self, y_pred, path):
self.subm['target'] = y_pred
self.subm[['ID_code','target']].to_csv(path, index=False )
performance_analysis_test(xgbreg,rmse_guess_train,rmse_base_train,rmse_guess_evalset,rmse_base_evalset )<load_pretrained>
print("Training model...
 ")
 model = Model(df_trn,df_tst,y_trn,submission)
 y_pred = model.predict(path ='submission_lgb.csv', model=model.lgb )
create_directory(os.path.join(DATA_FOLDER, 'predictions/models/xgbreg_seniority2'))
pickle.dump(xgbreg, open(os.path.join(DATA_FOLDER,'predictions/models/xgbreg_seniority2/model.pickle'), 'wb'))
X_test.to_pickle(os.path.join(DATA_FOLDER,'predictions/models/xgbreg_seniority2/X_test.pkl'))<drop_column>
print("Training model...
 ")
 model = Model(df_trn,df_tst,y_trn,submission)
 y_pred = model.predict(path ='submission_xgb.csv', model=model.xgb )