Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<groupby>
from fastai.tabular import *
def feature(df):
features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchType")
 condition='False'
if 'winPlacePerc' in df.columns:
y = np.array(df.groupby(['matchId','groupId'])['winPlacePerc'].agg('mean'), dtype=np.float64)
 features.remove("winPlacePerc")
 condition='True'
print("get group mean feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = agg.reset_index() [['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("get group max feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("get group min feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 print("get match mean feature")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 df_id=df_out[["matchId", "groupId"]].copy()
df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 del df, agg, agg_rank
gc.collect()
if condition=='True':
return df_out,pd.DataFrame(y),df_id
else:
return df_out,df_id<prepare_x_and_y>
PATH = Path('.. /input' )
x,y,id_train=feature(reduce_mem_usage(train))<feature_engineering>
df = pd.read_csv(PATH/'train.csv')
 test_df = pd.read_csv(PATH/'test.csv' )
x_test,id_test=feature(reduce_mem_usage(test))<set_options>
features = [feature for feature in df.columns if 'var_' in feature]
len(features )
del train,test
gc.collect()<split>
def augment_df(df):
for feature in features:
df[f'sq_{feature}'] = df[feature]**2
df[f'repo_{feature}'] = df[feature].apply(lambda x: 0 if x==0 else 1/x)
 df['min'] = df[features].min(axis=1)
 df['mean'] = df[features].mean(axis=1)
 df['max'] = df[features].max(axis=1)
 df['median'] = df[features].median(axis=1)
 df['std'] = df[features].std(axis=1)
 df['var'] = df[features].var(axis=1)
 df['abs_mean'] = df[features].abs().mean(axis=1)
 df['abs_median'] = df[features].abs().median(axis=1)
 df['abs_std'] = df[features].abs().std(axis=1)
 df['skew'] = df[features].skew(axis=1)
 df['kurt'] = df[features].kurt(axis=1)
 df['sq_kurt'] = df[[f'sq_{feature}' for feature in features]].kurt(axis=1 )
x['matchId']=id_train['matchId']
x['groupId']=id_train['groupId']
x_train,x_val,y_train,y_val=train_test_split(reduce_mem_usage(x),y,test_size=.1)
 x_test=reduce_mem_usage(x_test)
 id_val=x_val[['matchId','groupId']]
x_val.drop(['matchId','groupId'],axis=1,inplace=True)
 x_train.drop(['matchId','groupId'],axis=1,inplace=True)
 x.drop(['matchId','groupId'],axis=1,inplace=True)
 del y
gc.collect()<init_hyperparams>
features = features + [f'sq_{feature}' for feature in features] + [f'repo_{feature}' for feature in features]
num_features = len(features )
params = {
"objective" : "regression",
"metric" : "mae",
"num_leaves" : 149,
"learning_rate" : 0.03,
"bagging_fraction" : 0.9,
"bagging_seed" : 0,
"num_threads" : 4,
"colsample_bytree" : 0.5,
'min_data_in_leaf':1900,
'min_split_gain':0.00011,
'lambda_l2':9
}<create_dataframe>
class roc(Callback):
def on_epoch_begin(self, **kwargs):
self.total = 0
self.batch_count = 0
def on_batch_end(self, last_output, last_target, **kwargs):
preds = F.softmax(last_output, dim=1)
 try:
roc_score = roc_auc_score(to_np(last_target), to_np(preds[:, -1]))
self.total+=roc_score
self.batch_count+=1
except:
pass
def on_epoch_end(self, num_batch, **kwargs):
self.metric = self.total/self.batch_count
lgb_train = lgb.Dataset(x_train, y_train,
free_raw_data=False)
 lgb_eval = lgb.Dataset(x_val, y_val, reference=lgb_train,
free_raw_data=False )<train_model>
def train_and_eval_tabular_learner(train_df, train_features, valid_idx, add_noise=False, lr=0.02, epochs=1, layers=[200, 100], ps=[0.5, 0.2], name='learner'):
data = TabularDataBunch.from_df(path='.', df=train_df, dep_var='target', valid_idx=valid_idx,
cat_names=[], cont_names=train_features, bs=bs, procs=[FillMissing, Normalize], test_df=test_df)
 learn = tabular_learner(data, layers=layers, ps=ps, metrics=[roc() ])
 if add_noise:
learn.data = None
data = None
noise = np.random.normal(summary[features].loc['mean'].values, summary[features].loc['std'].values,(len(train_df), num_features)) /100
train_df[features]+=noise
data = TabularDataBunch.from_df(path='.', df=train_df, dep_var='target', valid_idx=valid_idx,
cat_names=[], cont_names=train_features, bs=bs, procs=[FillMissing, Normalize], test_df=test_df)
 learn.data = data
learn.fit_one_cycle(epochs, lr)
 train_df[features]-=noise
noise = None
learn.fit_one_cycle(epochs, lr)
 learn.save(name, with_opt=False)
 valid_preds, _ = learn.get_preds(ds_type=DatasetType.Valid)
 valid_probs = np.array(valid_preds[:, -1])
 valid_targets = train_df.loc[valid_idx].target.values
valid_score = roc_auc_score(valid_targets, valid_probs)
 test_preds, _ = learn.get_preds(ds_type=DatasetType.Test)
 test_probs = to_np(test_preds[:, -1])
 del(data)
 del(learn)
 gc.collect()
return valid_score, valid_probs, test_probs
model = lgb.train(params,
lgb_train,
num_boost_round=22000,
valid_sets=lgb_eval,
early_stopping_rounds=10,
verbose_eval=1000 )<load_from_csv>
sub_features = []
valid_scores = []
valid_preds = []
preds = []
num_epochs = 10
cv_counts = len(df)//num_epochs
saved_model_prefix = 'learner'
y_pred_val = model.predict(x, num_iteration=model.best_iteration)
 id_train['win_pred']=y_pred_val
id_train.set_index(['matchId','groupId'])
 train = reduce_mem_usage(pd.read_csv(".. /input/train_V2.csv"))
df=pd.merge(train,id_train,on=['matchId','groupId'],how='right')
 df<compute_test_metric>
augmented_features = ['min', 'mean', 'max', 'median', 'std', 'abs_mean', 'abs_median', 'abs_std', 'skew', 'kurt', 'sq_kurt']
print('The mae score is {}'.format(mean_absolute_error(df['winPlacePerc'],df['win_pred'])))
 df = df[["Id", "matchId", "groupId", "maxPlace", "numGroups",'winPlacePerc', 'win_pred']]<groupby>
bs = 2048
df_grouped = df.groupby(["matchId", "groupId"] ).first().reset_index()
df_grouped["team_place"] = df_grouped.groupby(["matchId"])["win_pred"].rank()
df_grouped<merge>
import gc
df_grouped["win_perc"] =(df_grouped["team_place"] - 1)/(df_grouped["numGroups"]-1)
 df = df.merge(df_grouped[["win_perc","matchId", "groupId"]], on=["matchId", "groupId"], how="left" )<feature_engineering>
gc.collect()
df.loc[df['maxPlace'] == 0, "win_perc"] = 0
df.loc[df['maxPlace'] == 1, "win_perc"] = 1
df.loc[(df['maxPlace'] > 1)&(df['numGroups'] == 1), "win_perc"] = 0
df.loc[df['win_perc'] < 0,"win_perc"] = 0
df.loc[df['win_perc'] > 1,"win_perc"] = 1
df['win_perc'].fillna(df['win_pred'],inplace=True )<filter>
from sklearn.metrics import roc_auc_score
df_grouped[df_grouped['maxPlace']>1][['winPlacePerc','win_perc','maxPlace','numGroups','team_place']]<feature_engineering>
for i in range(num_epochs):
print('Training model: ', i)
 sub_features.append(random.sample(list(features), int(num_features*0.75)) + augmented_features)
 name = f'{saved_model_prefix}_{i}'
score, valid_probs, test_probs = train_and_eval_tabular_learner(df, sub_features[-1], valid_idx,
add_noise=True, epochs=3, lr=0.02, name=name)
 valid_scores.append(score)
 valid_preds.append(valid_probs)
 preds.append(test_probs )
subset = df.loc[df['maxPlace'] > 1]
gap = 1 /(subset['maxPlace'].values-1)
 new_perc = np.around(subset['win_perc'].values / gap)* gap
df.loc[df.maxPlace > 1, "win_perc"] = new_perc<compute_test_metric>
import lightgbm as lgb
from sklearn.model_selection import KFold, StratifiedKFold
print('The new mae score is {}'.format(mean_absolute_error(df['winPlacePerc'],df['win_perc'])) )<set_options>
target = df['target']
del x,train,df
gc.collect()<load_from_csv>
import time
y_pred = model.predict(x_test, num_iteration=model.best_iteration)
 id_test['win_pred']=y_pred
id_test.set_index(['matchId','groupId'])
 del x_train,x_val,y_train,y_val,x_test
gc.collect()
test = reduce_mem_usage(pd.read_csv(".. /input/test_V2.csv"))
df=pd.merge(test,id_test,on=['matchId','groupId'],how='right')
 del id_test,test
gc.collect()
df<drop_column>
param = {
'bagging_freq': 5,
'bagging_fraction': 0.4,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.05,
'learning_rate': 0.01,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1
}
df = df[["Id", "matchId", "groupId", "maxPlace", "numGroups",'win_pred']]<groupby>
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=2019)
 oof = np.zeros(len(df))
predictions = np.zeros(len(test_df))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(skf.split(df.values, target.values)) :
print("fold n°{}".format(fold_))
trn_data = lgb.Dataset(df.iloc[trn_idx][features], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(df.iloc[val_idx][features], label=target.iloc[val_idx])
 num_round = 10000
clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 100)
 oof[val_idx] = clf.predict(df.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test_df[features], num_iteration=clf.best_iteration)/ 5
df_grouped = df.groupby(["matchId", "groupId"] ).first().reset_index()
df_grouped["team_place"] = df_grouped.groupby(["matchId"])["win_pred"].rank()
df_grouped<merge>
print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)) )
df_grouped["win_perc"] =(df_grouped["team_place"] - 1)/(df_grouped["numGroups"]-1)
 df = df.merge(df_grouped[["win_perc", "matchId", "groupId"]], on=["matchId", "groupId"], how="left" )<feature_engineering>
preds1 = np.array(sum(preds)/num_epochs)
 preds1
df.loc[df.maxPlace == 0, "win_perc"] = 0
df.loc[df.maxPlace == 1, "win_perc"] = 1
df.loc[(df.maxPlace > 1)&(df.numGroups == 1), "win_perc"] = 0
df.loc[df['win_perc'] < 0,"win_perc"] = 0
df.loc[df['win_perc'] > 1,"win_perc"] = 1
df['win_perc'].fillna(df['win_pred'],inplace=True )<feature_engineering>
all_ensemble_values = [0., 0.125, 0.25, 0.375, 0.5, 0.675, 0.75, 0.875, 1.0]
subset = df.loc[df['maxPlace'] > 1]
gap = 1 /(subset['maxPlace'].values-1)
 new_perc = np.around(subset['win_perc'].values / gap)* gap
df.loc[df.maxPlace > 1, "win_perc"] = new_perc
df['winPlacePerc']=df['win_perc']<save_to_csv>
sample_ensemble_array = np.array(all_ensemble_values[0]*preds1 +(1-all_ensemble_values[0])*predictions)
 sample_ensemble_array
df=df[['Id','winPlacePerc']]
df.to_csv("submission_final.csv", index=False )<string_transform>
for i in range(len(all_ensemble_values)) :
predict_array = np.array(all_ensemble_values[i]*preds1 +(1-all_ensemble_values[i])*predictions)
 sub_df = pd.DataFrame({'ID_code': test_df['ID_code'].values})
 sub_df['target'] = predict_array
sub_df.to_csv(f'submission_{i}.csv', index=False )