Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<load_from_csv>
%matplotlib inline
sns.set(color_codes=True)
 np.random.seed(25)
def feature_engineering(is_train=True,debug=True):
test_idx = None
if is_train:
print("processing train.csv")
 if debug == True:
df = pd.read_csv('.. /input/train_V2.csv', nrows=10000)
 else:
df = pd.read_csv('.. /input/train_V2.csv')
 df = df[df['maxPlace'] > 1]
else:
print("processing test.csv")
 df = pd.read_csv('.. /input/test_V2.csv')
 test_idx = df.Id
print("remove some columns")
 target = 'winPlacePerc'
print("Adding Features")
 df['headshotrate'] = df['kills']/df['headshotKills']
df['killStreakrate'] = df['killStreaks']/df['kills']
df['healthitems'] = df['heals'] + df['boosts']
df['totalDistance'] = df['rideDistance'] + df["walkDistance"] + df["swimDistance"]
df['killPlace_over_maxPlace'] = df['killPlace'] / df['maxPlace']
df['headshotKills_over_kills'] = df['headshotKills'] / df['kills']
df['distance_over_weapons'] = df['totalDistance'] / df['weaponsAcquired']
df['walkDistance_over_heals'] = df['walkDistance'] / df['heals']
df['walkDistance_over_kills'] = df['walkDistance'] / df['kills']
df['killsPerWalkDistance'] = df['kills'] / df['walkDistance']
df["skill"] = df["headshotKills"] + df["roadKills"]
df[df == np.Inf] = np.NaN
df[df == np.NINF] = np.NaN
print("Removing Na's From DF")
 df.fillna(0, inplace=True)
 features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchType")
 y = None
if is_train:
print("get target")
 y = np.array(df.groupby(['matchId','groupId'])[target].agg('mean'), dtype=np.float64)
 features.remove(target)
 print("get group mean feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train: df_out = agg.reset_index() [['matchId','groupId']]
else: df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("get group max feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("get group min feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 print("get group size feature")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("get match mean feature")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print("get match size feature")
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 X = df_out
feature_names = list(df_out.columns)
 del df, df_out, agg, agg_rank
gc.collect()
return X, y, feature_names, test_idx<prepare_x_and_y>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
x_train, y_train, train_columns, _ = feature_engineering(True,False )<feature_engineering>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
x_test, _, _ , test_idx = feature_engineering(False,True )<set_options>
target = train['target']
warnings.filterwarnings("ignore")
 color = sns.color_palette() 
<train_model>
train.isnull().sum()
train_index = round(int(x_train.shape[0]*0.9))
dev_X = x_train[:train_index]
val_X = x_train[train_index:]
dev_y = y_train[:train_index]
val_y = y_train[train_index:]
gc.collect() ;
def run_lgb(train_X, train_y, val_X, val_y, x_test):
params = {"objective" : "regression", "metric" : "mae", 'n_estimators':20000, 'early_stopping_rounds':100,
"num_leaves" : 31, "learning_rate" : 0.05, "bagging_fraction" : 0.9,
"bagging_seed" : 0, "num_threads" : 4,"colsample_bytree" : 0.7
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 model = lgb.train(params, lgtrain, valid_sets=[lgtrain, lgval], early_stopping_rounds=200, verbose_eval=1000)
 pred_test_y = model.predict(x_test, num_iteration=model.best_iteration)
 return pred_test_y, model
pred_test, model = run_lgb(dev_X, dev_y, val_X, val_y, x_test )<load_from_csv>
train.isnull().sum()
df_sub = pd.read_csv(".. /input/sample_submission_V2.csv")
 df_test = pd.read_csv(".. /input/test_V2.csv")
 df_sub['winPlacePerc'] = pred_test
df_sub = df_sub.merge(df_test[["Id", "matchId", "groupId", "maxPlace", "numGroups"]], on="Id", how="left")
 df_sub_group = df_sub.groupby(["matchId", "groupId"] ).first().reset_index()
df_sub_group["rank"] = df_sub_group.groupby(["matchId"])["winPlacePerc"].rank()
df_sub_group = df_sub_group.merge(
 df_sub_group.groupby("matchId")["rank"].max().to_frame("max_rank" ).reset_index() ,
on="matchId", how="left")
 df_sub_group["adjusted_perc"] =(df_sub_group["rank"] - 1)/(df_sub_group["numGroups"] - 1)
 df_sub = df_sub.merge(df_sub_group[["adjusted_perc", "matchId", "groupId"]], on=["matchId", "groupId"], how="left")
 df_sub["winPlacePerc"] = df_sub["adjusted_perc"]
df_sub.loc[df_sub.maxPlace == 0, "winPlacePerc"] = 0
df_sub.loc[df_sub.maxPlace == 1, "winPlacePerc"] = 1
subset = df_sub.loc[df_sub.maxPlace > 1]
gap = 1.0 /(subset.maxPlace.values - 1)
 new_perc = np.around(subset.winPlacePerc.values / gap)* gap
df_sub.loc[df_sub.maxPlace > 1, "winPlacePerc"] = new_perc
df_sub.loc[(df_sub.maxPlace > 1)&(df_sub.numGroups == 1), "winPlacePerc"] = 0
assert df_sub["winPlacePerc"].isnull().sum() == 0
df_sub[["Id", "winPlacePerc"]].to_csv("submission_adjusted.csv", index=False )<count_unique_values>
print(any(train.duplicated()))
matchcount = train.matchId.nunique()
print("Number of unique matches: {}".format(train.matchId.nunique()))
print("Train Shape Before: {} Rows, {} Cols".format(*train.shape))
train = train.loc[train.matchId.isin(sorted(train.matchId.unique())[int(matchcount* 0.50):]),:]
print("Train Shape After: {} Rows, {} Cols".format(*train.shape))<categorify>
correlations = train.corr().abs().unstack().sort_values(kind="quicksort" ).reset_index()
correlations = correlations[correlations['level_0'] != correlations['level_1']]
correlations.tail(10 )
lbl = preprocessing.LabelEncoder()
for col in ['matchType']:
lbl.fit(train[col])
 train[col] = lbl.transform(train[col])
 test[col] = lbl.transform(test[col])
 id_cols = ["Id","groupId","matchId"]
exclude = ["Id","groupId","matchId"]
trainlen = train.shape[0]
matchcount = train.matchId.nunique()
training = train.loc[train.matchId.isin(sorted(train.matchId.unique())[:int(matchcount* 0.85)]),
[x for x in train.columns if x not in exclude]]
print("Training Shape: {} Rows, {} Cols".format(*training.shape))
validating = train.loc[train.matchId.isin(sorted(train.matchId.unique())[int(matchcount* 0.85):]),
[x for x in train.columns if x not in exclude]]
print("Validating Shape: {} Rows, {} Cols".format(*validating.shape))
train_y = training.winPlacePerc
training.drop("winPlacePerc", axis =1, inplace=True)
 valid_y = validating.winPlacePerc
validating.drop("winPlacePerc", axis =1, inplace=True)
 lgb_train = lgb.Dataset(training, train_y,feature_name = "auto")
 lgb_valid = lgb.Dataset(validating, valid_y, feature_name = "auto")
 del training, validating<choose_model_class>
feature_names = [x for x in train.columns if x not in ['ID_code','target']]
print("Light Gradient Boosting Regressor: ")
 lgbm_params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': 'rmse',
'num_boost_round' : 5000
}<save_to_csv>
model = CatBoostClassifier(iterations=5000,learning_rate=0.01,eval_metric='AUC',random_seed=42,verbose=False,task_type='GPU')
 model.fit(train[feature_names],target)
 pred1 = model.predict_proba(test[feature_names] )
pred = gbm.predict(test.loc[:,[x for x in test.columns if x not in id_cols]])
 test['winPlacePercPred'] = np.clip(pred, a_min=0, a_max=1)
 aux = test.groupby(['matchId','groupId'])['winPlacePercPred'].agg('mean' ).groupby('matchId' ).rank(pct=True ).reset_index()
aux.columns = ['matchId','groupId','winPlacePerc']
test_sub = test.merge(aux, how='left', on=['matchId','groupId'])
 submission = test_sub[['Id', 'winPlacePerc']]
submission.to_csv('submissions_PubGG_LGBM.csv', index=False)
 display(submission.head() )<prepare_x_and_y>
model = lgb.LGBMClassifier(n_estimators=5000,n_jobs = -1,learning_rate=0.01)
 model.fit(train[feature_names],target)
 pred2 = model.predict_proba(test[feature_names] )
notcat = ["assists","boosts","damageDealt","DBNOs","heals","headshotKills","heals","killPlace","killPoints","kills",
"killStreaks","longestKill","maxPlace","numGroups","revives","rideDistance","roadKills","swimDistance",
"teamKills","vehicleDestroys","walkDistance","weaponsAcquired","winPoints"]
y = train.winPlacePerc.copy()
train.drop("winPlacePerc",axis =1, inplace=True)
 def agg_dataset(dataset):
for id_col in id_cols:
agg_features = dataset.groupby(id_col ).agg({k:["sum","mean","std"] for k in
["killPlace","walkDistance","numGroups","maxPlace","kills","longestKill","weaponsAcquired"]})
 agg_features.columns = pd.Index(["{}_agg_".format(id_col)+ e[0] +"_"+ e[1] for e in agg_features.columns.tolist() ])
 dataset = pd.merge(dataset,agg_features, on = id_col, how= "left")
 return dataset
train = agg_dataset(train)
 test = agg_dataset(test)
 missing = round(train.isnull().sum() / train.shape[0]*100 ).reset_index().rename({"index":"columns",0:"missing"}, axis =1)
 high_missing_columns = missing.loc[missing.missing > 65, "columns"]
print("Columns to remove(65% missing Values and Over)
 ", list(high_missing_columns))
train.drop(high_missing_columns,axis =1, inplace= True)
 test.drop(high_missing_columns,axis =1, inplace= True)
 train = pd.concat([train.reset_index(drop=True), y.reset_index(drop=True)], axis=1)
 training = train.loc[train.matchId.isin(sorted(train.matchId.unique())[:int(matchcount* 0.85)]),
[x for x in train.columns if x not in exclude]]
print("Training Shape: {} Rows, {} Cols".format(*training.shape))
validating = train.loc[train.matchId.isin(sorted(train.matchId.unique())[int(matchcount* 0.85):]),
[x for x in train.columns if x not in exclude]]
print("Validating Shape: {} Rows, {} Cols".format(*validating.shape))
train_y = training.winPlacePerc
training.drop("winPlacePerc", axis =1, inplace=True)
 valid_y = validating.winPlacePerc
validating.drop("winPlacePerc", axis =1, inplace=True)
 lgb_train = lgb.Dataset(training, train_y,feature_name = "auto")
 lgb_valid = lgb.Dataset(validating, valid_y, feature_name = "auto" )<train_model>
model = xgb.XGBClassifier(n_estimators=5000,n_jobs = -1,learning_rate=0.01)
 model.fit(train[feature_names],target)
 pred3 = model.predict_proba(test[feature_names] )
print("Light Gradient Boosting Regressor: ")
 lgbm_params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': 'rmse',
'num_boost_round' : 5000
}
stage = 'model training'
gbm = lgb.train(lgbm_params,
lgb_train,
num_boost_round=10000,
valid_sets=[lgb_train, lgb_valid],
feature_name='auto',
early_stopping_rounds=50,
verbose_eval=250 
)
 f, ax = plt.subplots(figsize=[7,10])
 lgb.plot_importance(gbm, max_num_features=25, ax=ax)
 plt.title("Light GBM Feature Importance
")
 plt.show()<save_to_csv>
model = RandomForestClassifier(n_estimators=1000,n_jobs = -1)
 model.fit(train[feature_names],target)
 pred4 = model.predict_proba(test[feature_names] )
pred = gbm.predict(test.loc[:,[x for x in test.columns if x not in id_cols]])
 test['winPlacePercPred'] = np.clip(pred, a_min=0, a_max=1)
 aux = test.groupby(['matchId','groupId'])['winPlacePercPred'].agg('mean' ).groupby('matchId' ).rank(pct=True ).reset_index()
aux.columns = ['matchId','groupId','winPlacePerc']
test_sub = test.merge(aux, how='left', on=['matchId','groupId'])
 submission = test_sub[['Id', 'winPlacePerc']]
submission.to_csv('submissions_AGG_PubGG_LGBM.csv', index=False)
 print("Notebook Runtime: %0.2f Minutes"%(( time.time() - notebookstart)/60))
submission.head()<import_modules>
pred = []
for i in range(len(pred1)) :
pred.append(( pred1[i][1] + pred2[i][1] + pred3[i][1] + pred4[i][1])/4 )
<install_modules><EOS> 
sub = pd.DataFrame()
sub['ID_code'] = test['ID_code']
sub['target'] = pred
sub.to_csv('result.csv', index=False )