Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<define_variables>
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
from catboost import CatBoostClassifier,Pool
from IPython.display import display
import matplotlib.patches as patch
import matplotlib.pyplot as plt
from sklearn.svm import NuSVR
from scipy.stats import norm
from sklearn import svm
import lightgbm as lgb
import xgboost as xgb
import seaborn as sns
import pandas as pd
import numpy as np
import warnings
import time
import glob
import sys
import os
import gc
INPUT_DIR = ".. /input/"<load_from_csv>
fold_n=5
folds = StratifiedKFold(n_splits=fold_n, shuffle=True, random_state=10)
 %matplotlib inline
%precision 4
warnings.filterwarnings('ignore')
 plt.style.use('ggplot')
 np.set_printoptions(suppress=True)
 pd.set_option("display.precision", 15 )
def feature_engineering(is_train=True):
if is_train:
print("processing train_V2.csv")
 df = pd.read_csv(INPUT_DIR + 'train_V2.csv')
 df = df[df['maxPlace'] > 1]
else:
print("processing test_V2.csv")
 df = pd.read_csv(INPUT_DIR + 'test_V2.csv')
 state('totalDistance')
 s = timer()
df['totalDistance'] = df['rideDistance'] + df["walkDistance"] + df["swimDistance"]
e = timer()
state('totalDistance', False, e - s)
 state('rankPoints')
 s = timer()
df['rankPoints'] = np.where(df['rankPoints'] <= 0 ,0 , df['rankPoints'])
 e = timer()
state('rankPoints', False, e-s)
 target = 'winPlacePerc'
features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchDuration")
 features.remove("matchType")
 y = None
if is_train:
y = np.array(df.groupby(['matchId','groupId'])[target].agg('mean'), dtype=np.float64)
 features.remove(target)
 print("get group mean feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train: df_out = agg.reset_index() [['matchId','groupId']]
else: df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("get group max feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("get group min feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 print("get group size feature")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("get match mean feature")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print("get match size feature")
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 X = np.array(df_out, dtype=np.float64)
 del df, df_out, agg, agg_rank
gc.collect()
return X, y 
<normalization>
print('pandas: {}'.format(pd.__version__))
print('numpy: {}'.format(np.__version__))
print('Python: {}'.format(sys.version))
%%time
x_train, y = feature_engineering(True)
 scaler = preprocessing.MinMaxScaler(feature_range=(-1, 1), copy=False ).fit(x_train )<train_model>
from sklearn.metrics import roc_auc_score, roc_curve
print("x_train", x_train.shape, x_train.max() , x_train.min())
 scaler.transform(x_train)
 print("x_train", x_train.shape, x_train.max() , x_train.min() )<choose_model_class>
train= pd.read_csv(".. /input/train.csv")
 test = pd.read_csv('.. /input/test.csv' )
%%time
NN_model = Sequential()
NN_model.add(Dense(x_train.shape[1], input_dim = x_train.shape[1], activation='relu'))
NN_model.add(Dense(x_train.shape[1], activation='relu'))
NN_model.add(Dense(x_train.shape[1], activation='relu'))
NN_model.add(Dense(1, activation='linear'))
NN_model.compile(loss='mean_absolute_error', optimizer='adam', metrics=['mean_absolute_error'])
 NN_model.summary() 
<choose_model_class>
sample_submission = pd.read_csv('.. /input/sample_submission.csv')
 sample_submission.head()
checkpoint_name = 'Weights-{epoch:03d}--{val_loss:.5f}.hdf5'
checkpoint = ModelCheckpoint(checkpoint_name, monitor='val_loss', verbose = 1, save_best_only = True, mode ='auto')
 callbacks_list = [checkpoint] 
<train_model>
def reduce_mem_usage(df):
start_mem_usg = df.memory_usage().sum() / 1024**2
print("Memory usage of properties dataframe is :",start_mem_usg," MB")
 NAlist = []
for col in df.columns:
if df[col].dtype != object:
print("******************************")
 print("Column: ",col)
 print("dtype before: ",df[col].dtype)
 IsInt = False
mx = df[col].max()
mn = df[col].min()
if not np.isfinite(df[col] ).all() :
NAlist.append(col)
 df[col].fillna(mn-1,inplace=True)
 asint = df[col].fillna(0 ).astype(np.int64)
 result =(df[col] - asint)
 result = result.sum()
if result > -0.01 and result < 0.01:
IsInt = True
if IsInt:
if mn >= 0:
if mx < 255:
df[col] = df[col].astype(np.uint8)
 elif mx < 65535:
df[col] = df[col].astype(np.uint16)
 elif mx < 4294967295:
df[col] = df[col].astype(np.uint32)
 else:
df[col] = df[col].astype(np.uint64)
 else:
if mn > np.iinfo(np.int8 ).min and mx < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif mn > np.iinfo(np.int16 ).min and mx < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif mn > np.iinfo(np.int32 ).min and mx < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif mn > np.iinfo(np.int64 ).min and mx < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
df[col] = df[col].astype(np.float32)
 print("dtype after: ",df[col].dtype)
 print("******************************")
 print("___MEMORY USAGE AFTER COMPLETION:___")
 mem_usg = df.memory_usage().sum() / 1024**2
print("Memory usage is: ",mem_usg," MB")
 print("This is ",100*mem_usg/start_mem_usg,"% of the initial size")
 return df, NAlist
%%time
NN_model_h = NN_model.fit(x=x_train,
y=y,
batch_size=3000,
epochs=30,
verbose=1,
callbacks=callbacks_list,
validation_split=0.20,
validation_data=None,
shuffle=True,
class_weight=None,
sample_weight=None,
initial_epoch=0,
steps_per_epoch=None,
validation_steps=None)
 print(NN_model_h.history)
 del x_train, y
gc.collect()<normalization>
def check_missing_data(df):
flag=df.isna().sum().any()
if flag==True:
total = df.isnull().sum()
percent =(df.isnull().sum())/(df.isnull().count() *100)
 output = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 data_type = []
for col in df.columns:
dtype = str(df[col].dtype)
 data_type.append(dtype)
 output['Types'] = data_type
return(np.transpose(output))
else:
return(False )
x_test, _ = feature_engineering(False)
 scaler.transform(x_test)
 print("x_test", x_test.shape, x_test.max() , x_test.min())
 np.clip(x_test, out=x_test, a_min=-1, a_max=1)
 print("x_test", x_test.shape, x_test.max() , x_test.min() )<predict_on_test>
check_missing_data(train )
%%time
pred = NN_model.predict(x_test)
 del x_test
gc.collect()<prepare_output>
check_missing_data(test )
pred = pred.reshape(-1)
 pred =(pred + 1)/ 2<load_from_csv>
train['target'].value_counts()
df_test = pd.read_csv(INPUT_DIR + 'test_V2.csv' )<feature_engineering>
def check_balance(df,target):
check=[]
print('size of data is:',df.shape[0])
 for i in [0,1]:
print('for target {} ='.format(i))
print(df[target].value_counts() [i]/df.shape[0]*100,'%')
%%time
print("fix winPlacePerc")
 for i in range(len(df_test)) :
winPlacePerc = pred[i]
maxPlace = int(df_test.iloc[i]['maxPlace'])
 if maxPlace == 0:
winPlacePerc = 0.0
elif maxPlace == 1:
winPlacePerc = 1.0
else:
gap = 1.0 /(maxPlace - 1)
 winPlacePerc = round(winPlacePerc / gap)* gap
if winPlacePerc < 0: winPlacePerc = 0.0
if winPlacePerc > 1: winPlacePerc = 1.0
pred[i] = winPlacePerc<feature_engineering>
check_balance(train,'target' )
df_test['winPlacePerc'] = pred<save_to_csv>
cols=["target","ID_code"]
X = train.drop(cols,axis=1)
 y = train["target"]
submission = df_test[['Id', 'winPlacePerc']]
submission.to_csv('submission.csv', index=False )<set_options>
X_test = test.drop("ID_code",axis=1 )
warnings.filterwarnings('ignore')
 %matplotlib inline
py.init_notebook_mode(connected=True)
<load_from_csv>
train_X, val_X, train_y, val_y = train_test_split(X, y, random_state=1)
 rfc_model = RandomForestClassifier(random_state=0 ).fit(train_X, train_y )
debug = True
if debug == True:
df_train = pd.read_csv('.. /input/train_V2.csv', nrows=1000)
 df_test = pd.read_csv('.. /input/test_V2.csv', nrows=1000)
 else:
df_train = pd.read_csv('.. /input/train_V2.csv')
 df_test = pd.read_csv('.. /input/test_V2.csv' )<filter>
perm = PermutationImportance(rfc_model, random_state=1 ).fit(val_X, val_y )
df_train[df_train['groupId']=='4d4b580de459be']<filter>
eli5.show_weights(perm, feature_names = val_X.columns.tolist() , top=150 )
len(df_train[df_train['matchId']=='a10357fd1a4a91'] )<filter>
train_X, val_X, train_y, val_y = train_test_split(X, y, random_state=1)
 tree_model = DecisionTreeClassifier(random_state=0, max_depth=5, min_samples_split=5 ).fit(train_X, train_y )
df_train = df_train[df_train['Id']!='f70c74418bb064']<feature_engineering>
features = [c for c in train.columns if c not in ['ID_code', 'target']]
headshot = df_train[['kills','winPlacePerc','headshotKills']]
headshot['headshotrate'] = headshot['kills'] / headshot['headshotKills']<drop_column>
tree_graph = tree.export_graphviz(tree_model, out_file=None, feature_names=features )
del headshot<feature_engineering>
row_to_show = 5
data_for_prediction = val_X.iloc[row_to_show]
data_for_prediction_array = data_for_prediction.values.reshape(1, -1)
 rfc_model.predict_proba(data_for_prediction_array);
df_train['headshotrate'] = df_train['kills']/df_train['headshotKills']
df_test['headshotrate'] = df_test['kills']/df_test['headshotKills']<feature_engineering>
params = {'objective' : "binary",
'boost':"gbdt",
'metric':"auc",
'boost_from_average':"false",
'num_threads':8,
'learning_rate' : 0.01,
'num_leaves' : 13,
'max_depth':-1,
'tree_learner' : "serial",
'feature_fraction' : 0.05,
'bagging_freq' : 5,
'bagging_fraction' : 0.4,
'min_data_in_leaf' : 80,
'min_sum_hessian_in_leaf' : 10.0,
'verbosity' : 1}
killStreak = df_train[['kills','winPlacePerc','killStreaks']]
killStreak['killStreakrate'] = killStreak['killStreaks']/killStreak['kills']
killStreak.corr()<drop_column>
%%time
y_pred_lgb = np.zeros(len(X_test))
num_round = 1000000
for fold_n,(train_index, valid_index)in enumerate(folds.split(X,y)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 lgb_model = lgb.train(params,train_data,num_round,
valid_sets = [train_data, valid_data],verbose_eval=1000,early_stopping_rounds = 3500)
 y_pred_lgb += lgb_model.predict(X_test, num_iteration=lgb_model.best_iteration)/5
del healthitems<feature_engineering>
y_pred_rfc = rfc_model.predict(X_test )
kills = df_train[['assists','winPlacePerc','kills']]
kills['kills_assists'] =(kills['kills'] + kills['assists'])
 kills.corr()<set_options>
y_pred_tree = tree_model.predict(X_test )
del df_train,df_test;
gc.collect()<load_from_csv>
train_pool = Pool(train_X,train_y)
 cat_model = CatBoostClassifier(
 iterations=3000,
learning_rate=0.03,
objective="Logloss",
eval_metric='AUC', 
)
 cat_model.fit(train_X,train_y,silent=True)
 y_pred_cat = cat_model.predict(X_test )
%%time
def feature_engineering(is_train=True,debug=True):
test_idx = None
if is_train:
print("processing train.csv")
 if debug == True:
df = pd.read_csv('.. /input/train_V2.csv', nrows=10000)
 else:
df = pd.read_csv('.. /input/train_V2.csv')
 df = df[df['maxPlace'] > 1]
else:
print("processing test.csv")
 df = pd.read_csv('.. /input/test_V2.csv')
 test_idx = df.Id
print("remove some columns")
 target = 'winPlacePerc'
features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchType")
 y = None
if is_train:
print("get target")
 y = np.array(df.groupby(['matchId','groupId'])[target].agg('mean'), dtype=np.float64)
 features.remove(target)
 print("get group mean feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train: df_out = agg.reset_index() [['matchId','groupId']]
else: df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("get group max feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("get group min feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 print("get group size feature")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("get match mean feature")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print("get match size feature")
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 X = df_out
feature_names = list(df_out.columns)
 del df, df_out, agg, agg_rank
gc.collect()
return X, y, feature_names, test_idx
x_train, y_train, train_columns, _ = feature_engineering(True,False)
 x_test, _, _ , test_idx = feature_engineering(False,True)
<feature_engineering>
submission_rfc = pd.DataFrame({
"ID_code": test["ID_code"],
"target": y_pred_rfc
})
 submission_rfc.to_csv('submission_rfc.csv', index=False )
%%time
x_train['headshotrate'] = x_train['kills']/x_train['headshotKills']
x_test['headshotrate'] = x_test['kills']/x_test['headshotKills']
x_train['killStreakrate'] = x_train['killStreaks']/x_train['kills']
x_test['killStreakrate'] = x_test['killStreaks']/x_test['kills']
x_train['healthitems'] = x_train['heals'] + x_train['boosts']
x_test['healthitems'] = x_test['heals'] + x_test['boosts']
del x_train['heals'];del x_test['heals']
train_columns.append('headshotrate')
 train_columns.append('killStreakrate')
 train_columns.append('healthitems')
 train_columns.remove('heals' )<drop_column>
submission_tree = pd.DataFrame({
"ID_code": test["ID_code"],
"target": y_pred_tree
})
 submission_tree.to_csv('submission_tree.csv', index=False )
%%time
x_train = reduce_mem_usage(x_train)
 x_test = reduce_mem_usage(x_test )<set_options>
submission_cat = pd.DataFrame({
"ID_code": test["ID_code"],
"target": y_pred_cat
})
 submission_cat.to_csv('submission_cat.csv', index=False )
warnings.filterwarnings("ignore")
<define_variables>
submission_lgb = pd.DataFrame({
"ID_code": test["ID_code"],
"target": y_pred_lgb
})
 submission_lgb.to_csv('submission_lgb.csv', index=False )
%%time
folds = KFold(n_splits=3,random_state=6)
 oof_preds = np.zeros(x_train.shape[0])
 sub_preds = np.zeros(x_test.shape[0])
 start = time.time()
valid_score = 0
feature_importance_df = pd.DataFrame()
for n_fold,(trn_idx, val_idx)in enumerate(folds.split(x_train, y_train)) :
trn_x, trn_y = x_train.iloc[trn_idx], y_train[trn_idx]
val_x, val_y = x_train.iloc[val_idx], y_train[val_idx]
train_data = lgb.Dataset(data=trn_x, label=trn_y)
 valid_data = lgb.Dataset(data=val_x, label=val_y)
 params = {"objective" : "regression",
"metric" : "mae",
'n_estimators':1500,
'early_stopping_rounds':100,
"num_leaves" : 31,
"learning_rate" : 0.05,
"bagging_fraction" : 0.9,
"bagging_seed" : 0,
"num_threads" : 4,
"colsample_bytree" : 0.7
}
lgb_model = lgb.train(params, train_data, valid_sets=[train_data, valid_data], verbose_eval=1000)
 oof_preds[val_idx] = lgb_model.predict(val_x, num_iteration=lgb_model.best_iteration)
 oof_preds[oof_preds>1] = 1
oof_preds[oof_preds<0] = 0
sub_pred = lgb_model.predict(x_test, num_iteration=lgb_model.best_iteration)
 sub_pred[sub_pred>1] = 1
sub_pred[sub_pred<0] = 0
sub_preds += sub_pred/ folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = train_columns
fold_importance_df["importance"] = lgb_model.feature_importance()
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 gc.collect()
end = time.time()
print("Take Time :",(end-start))<load_from_csv>
submission_rfc_cat = pd.DataFrame({
"ID_code": test["ID_code"],
"target":(y_pred_rfc +y_pred_cat)/2
})
 submission_rfc_cat.to_csv('submission_rfc_cat.csv', index=False )
%%time
df_test = pd.read_csv('.. /input/' + 'test_V2.csv')
 pred = sub_preds
print("fix winPlacePerc")
 for i in range(len(df_test)) :
winPlacePerc = pred[i]
maxPlace = int(df_test.iloc[i]['maxPlace'])
 if maxPlace == 0:
winPlacePerc = 0.0
elif maxPlace == 1:
winPlacePerc = 1.0
else:
gap = 1.0 /(maxPlace - 1)
 winPlacePerc = round(winPlacePerc / gap)* gap
if winPlacePerc < 0: winPlacePerc = 0.0
if winPlacePerc > 1: winPlacePerc = 1.0
pred[i] = winPlacePerc
if(i + 1)% 100000 == 0:
print(i, flush=True, end=" " )<save_to_csv>
submission_lgb_cat = pd.DataFrame({
"ID_code": test["ID_code"],
"target":(y_pred_lgb +y_pred_cat)/2
})
 submission_lgb_cat.to_csv('submission_lgb_cat.csv', index=False )
<set_options><EOS> 
submission_rfc_lgb = pd.DataFrame({
"ID_code": test["ID_code"],
"target":(y_pred_rfc +y_pred_lgb)/2
})
 submission_rfc_lgb.to_csv('submission_rfc_lgb.csv', index=False )