Kaggle competition: Santander Customer Transaction Prediction
use_feature = False
if use_feature == True:
train['euclidean_distance'] = calculate_euclidean_distance(train.pickup_latitude ,train.pickup_longitude,
train.dropoff_latitude ,train.dropoff_longitude)
 test['euclidean_distance'] = calculate_euclidean_distance(test.pickup_latitude ,test.pickup_longitude,
test.dropoff_latitude ,test.dropoff_longitude )<data_type_conversions>
df_train=pd.read_csv('.. /input/train.csv' )
def get_timefeatures(df):
pickup_datetime = df.pickup_datetime.copy()
pickup_datetime = pickup_datetime.str.slice(0, 16)
 pickup_datetime = pd.to_datetime(pickup_datetime, utc=True, format='%Y-%m-%d %H:%M')
 df['year'] = pickup_datetime.dt.year.astype('uint16')
 df['month'] = pickup_datetime.dt.month.astype('uint8')
 df['day'] = pickup_datetime.dt.day.astype('uint8')
 df['dayofweek'] = pickup_datetime.dt.dayofweek.astype('uint8')
 df['hour'] = pickup_datetime.dt.hour.astype('uint8')
 df['minute'] = pickup_datetime.dt.minute.astype('uint8')
 del pickup_datetime
return df<feature_engineering>
df_test=pd.read_csv('.. /input/test.csv' )
train = get_timefeatures(train)
 test = get_timefeatures(test )<define_variables>
def memory_usage(df):
numerics = ['int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64']
for col in df.columns:
if str(df[col].dtype)in numerics:
if str(df[col].dtype)[:3] == 'int':
if(( df[col].min() > np.iinfo(np.int8 ).min)and(df[col].max() < np.iinfo(np.int8 ).max)) :
df[col] = df[col].astype('int8')
 elif(( df[col].min() > np.iinfo(np.int16 ).min)and(df[col].max() < np.iinfo(np.int16 ).max)) :
df[col] = df[col].astype('int16')
 elif(( df[col].min() > np.iinfo(np.int32 ).min)and(df[col].max() < np.iinfo(np.int32 ).max)) :
df[col] = df[col].astype('int32')
 else:
df[col] = df[col].astype('int64')
 else:
if(( df[col].min() > np.finfo(np.float16 ).min)and df[col].max() < np.finfo(np.float16 ).max):
df[col] = df[col].astype('float16')
 elif(( df[col].min() > np.finfo(np.float32 ).min)and df[col].max() < np.finfo(np.float32 ).max):
df[col] = df[col].astype('float32')
 elif(( df[col].min() > np.finfo(np.float64 ).min)and df[col].max() < np.finfo(np.float64 ).max):
df[col] = df[col].astype('float64')
 else:
df[col] = df[col].astype('float128')
 return df
basic_features = ['distance', 'longitude_abs', 'latitude_abs','passenger_count']
original_features = ['pickup_longitude', 'pickup_latitude',
'dropoff_longitude', 'dropoff_latitude']
time_features = ['year', 'month', 'day', 'dayofweek', 'hour', 'minute']<drop_column>
X_train=df_train.iloc[:,2:]
Y_train=df_train.iloc[:,1]
ID_train=df_train.iloc[:,0]
X_test=df_test.iloc[:,1:]
ID_test=df_test.iloc[:,0]
train.drop('pickup_datetime', axis= 1, inplace = True)
 test.drop('pickup_datetime', axis=1, inplace = True )<split>
df_null_train=X_train.isna().sum().reset_index()
df_null_train.columns=['Column_name','Null_Count']
df_null_train=df_null_train[df_null_train['Null_Count']>0]
test_size = 100_000 / len(train)
 X_train, X_val, y_train, y_val = train_test_split(train, train.fare_amount, \
test_size = test_size, random_state = seed, stratify = train.fare_amount_bin)
 X_train.shape, X_val.shape, y_train.shape, y_val.shape, test.shape<drop_column>
df_null_test=X_test.isna().sum().reset_index()
df_null_test.columns=['Column_name','Null_Count']
df_null_test=df_null_test[df_null_test['Null_Count']>0]
del train<save_to_csv>
df_Unique_train=X_train.nunique().reset_index()
df_Unique_train.columns=['Column_name','Unique_Count']
df_Unique_train=df_Unique_train[df_Unique_train['Unique_Count']==1]
def calculate_rmse(y_true, y_pred):
root_mean_squared_error = sqrt(mean_squared_error(y_true, y_pred))
return root_mean_squared_error
def submit_fare_amount(y_pred, filename):

 if isinstance(y_pred, pd.Series):
submission.update(y_pred)
 elif isinstance(y_pred, float):
submission.fare_amount = y_pred
else:
raise ValueError('y_pred is neither Pandas Series nor float')
 submission.to_csv(filename, index=False)
 print('Successfully written in ' + filename)
 display(submission.fare_amount.head(5 ).ravel())
 tuples = [('RMSE', 'Train'),('RMSE', 'CV'),('RMSE', 'Val'),('RMSE', 'Submission')]
columns = pd.MultiIndex.from_tuples(tuples)
 Metrics = pd.DataFrame(columns = columns)
 Metrics.index.name = 'Model'
Metrics<choose_model_class>
df_Unique_test=X_test.nunique().reset_index()
df_Unique_test.columns=['Column_name','Unique_Count']
df_Unique_test=df_Unique_test[df_Unique_test['Unique_Count']==1]
kfold = KFold(n_splits = 5, shuffle = True, random_state = seed )<drop_column>
df_corr=X_train.corr().abs().unstack().sort_values().reset_index()
df_corr=df_corr[df_corr['level_0']!= df_corr['level_1']]
df_corr.head()
X_train = X_train.drop(['fare_amount','fare_amount_bin'], axis=1)
 X_val = X_val.drop(['fare_amount','fare_amount_bin'], axis=1)
 X_train.shape, X_val.shape, test.shape<find_best_model_class>
X_train['sum'] = X_train.sum(axis=1)
 X_train['min'] = X_train.min(axis=1)
 X_train['max'] = X_train.max(axis=1)
 X_train['mean'] = X_train.mean(axis=1)
 X_train['median'] = X_train.median(axis=1)
 X_train['std'] = X_train.std(axis=1)
 X_train['skew'] = X_train.skew(axis=1)
 X_train.head()
def run_model(model, model_name, features, target, usecv = False):
print('fitting the training data...')
 model.fit(features[0], target[0])
 print('model is trained')
 print('predicting train and test data')
 pred =[]
for f in features:
pred.append(model.predict(f))
if usecv == True:
print('using cross validation')
 cvres = cross_val_score(model, features[0],target[0], \
scoring = 'neg_mean_squared_error', cv = kfold, n_jobs = -1)
 cv_mean = np.mean(np.sqrt(np.abs(cvres)))
 cv_std = np.std(np.sqrt(np.abs(cvres)))
 else:
print('No cross validation')
 cv_mean =0
cv_std =0
rmse =[]
for t, p in zip(target, pred):
rmse.append(calculate_rmse(t, p))
print('submitting.. ')
 submission_pred = pd.Series(pred[2], index = submission.index)
 submission_pred.name = submission.fare_amount.name
print(submission_pred.head())
 submit_fare_amount(submission_pred, model_name + '.csv')
 print('updating metrics.. ')
 Metrics.loc[model_name] = [rmse[0], str(np.round(cv_mean,2)) +' +/- '+ str(np.round(cv_std,2)) ,rmse[1], '']
display(Metrics)
 return pred<concatenate>
X_test['sum'] = X_test.sum(axis=1)
 X_test['min'] = X_test.min(axis=1)
 X_test['max'] = X_test.max(axis=1)
 X_test['mean'] = X_test.mean(axis=1)
 X_test['median'] = X_test.median(axis=1)
 X_test['std'] = X_test.std(axis=1)
 X_test['skew'] = X_test.skew(axis=1)
 X_test.head()
X = pd.concat([X_train, y_train], axis = 1)
 X_sample = X.sample(frac = 0.1 , random_state = seed)
 y_train_sample = X_sample.fare_amount
X_train_sample = X_sample.drop('fare_amount', axis=1)
 del X
X_train_sample.shape, y_train_sample.shape<train_model>
skewed_columns=['var_0', 'var_1', 'var_2', 'var_5', 'var_13', 'var_18','var_19','var_20','var_21', 'var_22',
'var_24','var_26', 'var_35', 'var_36','var_40','var_44','var_45','var_47','var_48','var_49',
'var_51','var_52','var_54','var_56','var_61', 'var_66','var_67','var_70','var_74','var_75',
'var_76','var_80','var_81','var_82','var_83','var_86','var_87','var_90', 'var_92','var_94',
'var_97', 'var_99',
'var_100', 'var_101', 'var_102', 'var_107', 'var_109','var_110', 'var_115', 'var_117',
'var_118', 'var_122','var_123', 'var_127', 'var_128', 'var_136', 'var_137', 'var_139',
'var_147', 'var_149',
'var_150','var_151','var_154','var_155','var_157','var_160','var_163','var_164','var_165',
'var_167','var_170','var_172','var_173','var_174','var_177','var_178','var_179','var_180',
'var_184','var_187','var_188','var_190','var_191','var_196','var_198','var_199']
%%time
execute_model = False
if execute_model:
linReg = LinearRegression(n_jobs = -1)
 model_name = 'Linear_Regression'
features = [X_train_sample[basic_features],X_val[basic_features],test[basic_features]]
target = [y_train_sample, y_val]
pred = run_model(linReg, model_name, features, target )<choose_model_class>
def bin_feature(col_name,X):
interval=pd.qcut(X[col_name],5 ).value_counts().index.sort_values()
X.loc[(X[col_name] <= interval[0].right),col_name] = 0
X.loc[(( X[col_name] > interval[1].left)&(X[col_name] <= interval[1].right)) ,col_name] = 1
X.loc[(( X[col_name] > interval[2].left)&(X[col_name] <= interval[2].right)) ,col_name] = 2
X.loc[(( X[col_name] > interval[3].left)&(X[col_name] <= interval[3].right)) ,col_name] = 3
X.loc[(X[col_name] > interval[4].left),col_name] = 4
return X
%%time
execute_model = False
if execute_model:
linReg = LinearRegression(n_jobs = -1)
 model_name = 'Linear_Regression_with_time_features'
features = [X_train_sample[basic_features+time_features],X_val[basic_features+time_features],test[basic_features+time_features]]
target = [y_train_sample, y_val]
pred = run_model(linReg, model_name, features, target )<choose_model_class>
for col in skewed_columns:
binned_col = col + '_bin'
X_train[binned_col] = X_train[col]
X_train[binned_col] = bin_feature(binned_col, X_train )
%%time
execute_model = False
if execute_model:
linReg = LinearRegression(n_jobs = -1)
 model_name = 'Linear_Regression_with_time_and_original_features'
features = [X_train_sample[basic_features+time_features+original_features],\
X_val[basic_features+time_features+original_features],test[basic_features+time_features+original_features]]
target = [y_train_sample, y_val]
pred = run_model(linReg, model_name, features, target )<choose_model_class>
for col in skewed_columns:
binned_col = col + '_bin'
X_test[binned_col] = X_test[col]
X_test[binned_col] = bin_feature(binned_col, X_test )
%%time
execute_model = False
if execute_model:
randReg = RandomForestRegressor(n_jobs=-1, random_state=seed, max_depth = 10, n_estimators = 20, verbose = 0)
 model_name = 'Simple_RandomForest_Regressor_all_features'
features = [X_train_sample,X_val,test]
target = [y_train_sample, y_val]
pred = run_model(randReg, model_name, features, target )<choose_model_class>
X_dev, X_val, Y_dev, Y_val=train_test_split(X_train, Y_train, test_size=0.4, random_state=0)
 X_dev.shape, X_val.shape, Y_dev.shape, Y_val.shape
%%time
execute_model = False
if execute_model:
xgbReg = XGBRegressor(n_jobs=-1, random_state=seed, max_depth = 20, \
n_estimators = 50, silent = True)
 model_name = 'XGBoost_Regressor'
features = [X_train_sample,X_val,test]
target = [y_train_sample, y_val]
pred = run_model(xgbReg, model_name, features, target )<train_on_grid>
from sklearn.model_selection import KFold
from sklearn.metrics import roc_auc_score
import xgboost as xgb
from sklearn.ensemble import GradientBoostingClassifier
def xbg_GridSearchCV(model, params, param_grid, dtrain, num_boost_round = 200, \
nfold = 3, metrics = 'rmse', early_stopping_rounds = 1000, seed = seed):
cvres = {}
for param_name, grid in param_grid.items() :
for value in grid:
params[param_name] = value
res = model.cv(params, dtrain, num_boost_round = num_boost_round, \
nfold = nfold, metrics = metrics, early_stopping_rounds = early_stopping_rounds, seed = seed)
 cvres[param_name + '_' + str(value)] = res.iloc[-1,:]
display(pd.DataFrame(cvres ).T)
 print('
')
 cvres = pd.DataFrame(cvres ).T
idx = cvres['test-rmse-mean'].idxmin()
cvres.loc[idx, 'best'] = 'yes'
cvres.fillna('', inplace=True)
 return cvres<prepare_x_and_y>
def xgb_train(X_train, Y_train, X_test):
xgb_param = {
'objective' : "binary:logistic",
'eval_metric' : "auc",
'max_depth' : 6,
'eta' : 0.1,
'gamma' : 5,
'subsample' : 0.7,
'colsample_bytree' : 0.7,
'min_child_weight' : 50,
'colsample_bylevel' : 0.7,
'lambda' : 1,
'alpha' : 0,
'booster' : "gbtree",
'silent' : 1,
"random_state": 4950
}
n_folds = 10
oof_train = np.zeros(X_train.shape[0])
 oof_test = np.zeros(X_test.shape[0])
 oof_test_kf = np.empty(( n_folds, X_test.shape[0]))
oof_train.shape, oof_test.shape, oof_test_kf.shape
kf = KFold(n_splits=n_folds, shuffle=True, random_state=2019)
 for fold_,(dev_idx, val_idx)in enumerate(kf.split(X_train)) :
print('Fold: ' + str(fold_))
X_dev, Y_dev = X_train.iloc[dev_idx, :], Y_train[dev_idx]
X_val, Y_val = X_train.iloc[val_idx, :], Y_train[val_idx]
xgb_dev = xgb.DMatrix(data=X_dev, label=Y_dev)
 xgb_val = xgb.DMatrix(data=X_val, label=Y_val)
 xgb_test = xgb.DMatrix(X_test)
 watchlist = [(xgb_dev, 'train'),(xgb_val, 'valid')]
xgb_model = xgb.train(xgb_param, xgb_dev, num_boost_round=1000, evals=watchlist,maximize=True,verbose_eval=50,early_stopping_rounds = 50)
 Y_pred = xgb_model.predict(xgb_val, ntree_limit=xgb_model.best_ntree_limit+50)
 Y_test = xgb_model.predict(xgb_test, ntree_limit=xgb_model.best_ntree_limit+50)
 oof_train[val_idx] = Y_pred
oof_test_kf[fold_, :] = Y_test
val_acc = np.round(roc_auc_score(Y_val, Y_pred), 4)
 val_acc = np.round(roc_auc_score(Y_train, oof_train), 4)
 print('Overall Training Accuracy: {0}'.format(val_acc))
oof_test = oof_test_kf.mean(axis=0)
 return oof_train, oof_test
data = X_train_sample
label = y_train_sample
dtrain = xgb.DMatrix(data = data, label = label, feature_names = data.columns)
 dtest = xgb.DMatrix(data = X_val, label = y_val, feature_names = X_val.columns )<init_hyperparams>
oof_xgb_train, oof_xgb_test = xgb_train(X_train, Y_train, X_test )
params = {}
params['objective'] = 'reg:linear'
params['eval_metric'] = 'rmse'
params['seed'] = seed
params['silent'] = 1
params['nthread'] = -1
params['eta'] = 0.2
params['gamma'] = 0
params['max_depth'] = 5
params['min_child_weight'] = 1
params['max_delta_step'] = 0
params['subsample'] = 1
params['colsample_bytree'] = 1
params['colsample_bylevel'] = 1
params['lambda'] = 1
params['alpha'] = 0
watchlist = [(dtrain, 'train'),(dtest, 'eval')]<train_model>
df_sample_submission= pd.read_csv('.. /input/sample_submission.csv')
 df_sample_submission.head()
num_boost_round = 2
bst = xgb.train(params, dtrain, num_boost_round, watchlist, early_stopping_rounds = 50 )<train_model>
output=pd.DataFrame({
"ID_code" : df_sample_submission['ID_code'],
"target" : oof_xgb_test
})
 output.to_csv('SCP_xgb_test.csv', index=False)
 output.head()
%%time
tuned_model = True
if tuned_model == False:
num_boost_round = 115
bst = xgb.train(params, dtrain, num_boost_round,watchlist, early_stopping_rounds = 50)
 print(bst.eval_set(watchlist))<predict_on_test>
id_train=df_train['ID_code']
df_out = pd.DataFrame({'id_train':id_train.values, 'target':oof_xgb_train})
 df_out.to_csv('SCP_xgb_train.csv', index=False )
ptrain = bst.predict(dtrain, output_margin = True)
 ptest = bst.predict(dtest, output_margin = True)
 dtrain.set_base_margin(ptrain)
 dtest.set_base_margin(ptest )<train_on_grid>
roc_auc_score(Y_train, oof_xgb_train )