Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<groupby>
!pip install bayesian-optimization
train_monthly = train.sort_values('date' ).groupby(['date_block_num', 'shop_id','item_category_id', 'item_id'], as_index=False )<drop_column>
pd.set_option('display.max_columns', 200 )
train_monthly = train_monthly.agg({'item_price':['median', 'mean'], 'item_cnt_day':['sum', 'count']} )<rename_columns>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )
train_monthly.columns = ['date_block_num', 'shop_id', 'item_category_id','item_id', 'item_price_median', 'item_price_mean', 'item_cnt', 'transactions']<groupby>
target = 'target'
predictors = train_df.columns.values.tolist() [2:]
train_monthly_by_category = train_monthly.groupby(['date_block_num','shop_id', 'item_category_id'], as_index=False )<groupby>
train_df.target.value_counts()
train_monthly_by_category = train_monthly_by_category.agg({'item_price_median':['mean'], 'item_price_mean':['mean'],'item_cnt':['sum', 'mean'], 'transactions': ['mean']} )<groupby>
bayesian_tr_index, bayesian_val_index = list(StratifiedKFold(n_splits=2, shuffle=True, random_state=1 ).split(train_df, train_df.target.values)) [0]
train_cat_no_shop = train_monthly.groupby(['date_block_num', 'item_category_id'], as_index=False )<groupby>
def LGB_bayesian(
 num_leaves,
min_data_in_leaf,
learning_rate,
min_sum_hessian_in_leaf,
feature_fraction,
lambda_l1,
lambda_l2,
min_gain_to_split,
max_depth):
num_leaves = int(num_leaves)
 min_data_in_leaf = int(min_data_in_leaf)
 max_depth = int(max_depth)
 assert type(num_leaves)== int
assert type(min_data_in_leaf)== int
assert type(max_depth)== int
param = {
'num_leaves': num_leaves,
'max_bin': 63,
'min_data_in_leaf': min_data_in_leaf,
'learning_rate': learning_rate,
'min_sum_hessian_in_leaf': min_sum_hessian_in_leaf,
'bagging_fraction': 1.0,
'bagging_freq': 5,
'feature_fraction': feature_fraction,
'lambda_l1': lambda_l1,
'lambda_l2': lambda_l2,
'min_gain_to_split': min_gain_to_split,
'max_depth': max_depth,
'save_binary': True,
'seed': 1337,
'feature_fraction_seed': 1337,
'bagging_seed': 1337,
'drop_seed': 1337,
'data_random_seed': 1337,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'auc',
'is_unbalance': True,
'boost_from_average': False,
}
xg_train = lgb.Dataset(train_df.iloc[bayesian_tr_index][predictors].values,
label=train_df.iloc[bayesian_tr_index][target].values,
feature_name=predictors,
free_raw_data = False 
)
 xg_valid = lgb.Dataset(train_df.iloc[bayesian_val_index][predictors].values,
label=train_df.iloc[bayesian_val_index][target].values,
feature_name=predictors,
free_raw_data = False 
)
 num_round = 5000
clf = lgb.train(param, xg_train, num_round, valid_sets = [xg_valid], verbose_eval=250, early_stopping_rounds = 50)
 predictions = clf.predict(train_df.iloc[bayesian_val_index][predictors].values, num_iteration=clf.best_iteration)
 score = metrics.roc_auc_score(train_df.iloc[bayesian_val_index][target].values, predictions)
 return score
train_cat_no_shop = train_cat_no_shop.agg({'item_price_median':['mean'], 'item_price_mean':['mean'],'item_cnt':['sum', 'mean'], 'transactions': ['mean']} )<feature_engineering>
bounds_LGB = {
'num_leaves':(5, 20),
'min_data_in_leaf':(5, 20),
'learning_rate':(0.01, 0.3),
'min_sum_hessian_in_leaf':(0.00001, 0.01),
'feature_fraction':(0.05, 0.5),
'lambda_l1':(0, 5.0),
'lambda_l2':(0, 5.0),
'min_gain_to_split':(0, 1.0),
'max_depth':(3,15),
}
train_monthly['year'] = train_monthly['date_block_num'].apply(lambda x:(( x//12+2013)))
 train_monthly['month'] = train_monthly['date_block_num'].apply(lambda x:(x%12+1))<feature_engineering>
from bayes_opt import BayesianOptimization
train_monthly_by_category['year'] = train_monthly_by_category['date_block_num'].apply(lambda x:(( x//12+2013)))
 train_monthly_by_category['month'] = train_monthly_by_category['date_block_num'].apply(lambda x:(x%12+1))<feature_engineering>
LGB_BO = BayesianOptimization(LGB_bayesian, bounds_LGB, random_state=13 )
train_cat_no_shop['year'] = train_cat_no_shop['date_block_num'].apply(lambda x:(( x//12+2013)))
 train_cat_no_shop['month'] = train_cat_no_shop['date_block_num'].apply(lambda x:(x%12+1))<rename_columns>
init_points = 5
n_iter = 5
train_monthly_by_category.columns = ['date_block_num', 'shop_id','item_category_id','item_price_median', 'item_price_mean', 'item_cnt_sum', 'item_cnt', 'transactions', 'year', 'month']
train_cat_no_shop.columns = ['date_block_num', 'item_category_id','item_price_median', 'item_price_mean', 'item_cnt_sum', 'item_cnt', 'transactions', 'year', 'month']<filter>
print('-' * 130)
 with warnings.catch_warnings() :
warnings.filterwarnings('ignore')
 LGB_BO.maximize(init_points=init_points, n_iter=n_iter, acq='ucb', xi=0.0, alpha=1e-6 )
train_monthly[train_monthly['item_cnt']>1000]<groupby>
LGB_BO.probe(
 params={'feature_fraction': 0.1403,
'lambda_l1': 4.218,
'lambda_l2': 1.734,
'learning_rate': 0.07,
'max_depth': 14,
'min_data_in_leaf': 17,
'min_gain_to_split': 0.1501,
'min_sum_hessian_in_leaf': 0.000446,
'num_leaves': 6},
lazy=True,
)
uniq_pairs = train_monthly.groupby(['shop_id','item_id'] ).size().reset_index()
uniq_pairs.shape<create_dataframe>
LGB_BO.maximize(init_points=0, n_iter=0 )
empty_df = pd.DataFrame(index = uniq_pairs.index, columns = ['date_block_num','shop_id','item_id'])
 empty_df[['shop_id', 'item_id']] = uniq_pairs[['shop_id','item_id']]
empty_df_2 = pd.DataFrame(columns = ['date_block_num','shop_id','item_id'])
 for i in range(34):
empty_df_1 = empty_df.copy()
empty_df_1['date_block_num'] = i
empty_df_2 = pd.concat([empty_df_2, empty_df_1] )<merge>
for i, res in enumerate(LGB_BO.res):
print("Iteration {}:
\t{}".format(i, res))
full_train_monthly = empty_df_2.merge(train_monthly,on=['date_block_num','shop_id', 'item_id'], how='left' ).fillna(0 ).drop(['index'], axis=1 )<feature_engineering>
param_lgb = {
'num_leaves': int(LGB_BO.max['params']['num_leaves']),
'max_bin': 63,
'min_data_in_leaf': int(LGB_BO.max['params']['min_data_in_leaf']),
'learning_rate': LGB_BO.max['params']['learning_rate'],
'min_sum_hessian_in_leaf': LGB_BO.max['params']['min_sum_hessian_in_leaf'],
'bagging_fraction': 1.0,
'bagging_freq': 5,
'feature_fraction': LGB_BO.max['params']['feature_fraction'],
'lambda_l1': LGB_BO.max['params']['lambda_l1'],
'lambda_l2': LGB_BO.max['params']['lambda_l2'],
'min_gain_to_split': LGB_BO.max['params']['min_gain_to_split'],
'max_depth': int(LGB_BO.max['params']['max_depth']),
'save_binary': True,
'seed': 1337,
'feature_fraction_seed': 1337,
'bagging_seed': 1337,
'drop_seed': 1337,
'data_random_seed': 1337,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'auc',
'is_unbalance': True,
'boost_from_average': False,
}
full_train_monthly['year'] = full_train_monthly['date_block_num'].apply(lambda x:(( x//12+2013)))
 full_train_monthly['month'] = full_train_monthly['date_block_num'].apply(lambda x:(x%12+1))<merge>
nfold = 5
full_train_monthly = full_train_monthly.join(items, on='item_id', rsuffix='_' ).drop(['item_id_', 'item_name', 'item_category_id'], axis=1)
 full_train_monthly = full_train_monthly.rename(columns={'item_category_id_':'item_category_id'} )<groupby>
gc.collect()
uniq_pairs_cat = train_monthly_by_category.groupby(['shop_id','item_category_id'] ).size().reset_index()<create_dataframe>
skf = StratifiedKFold(n_splits=nfold, shuffle=True, random_state=2019 )
empty_df = pd.DataFrame(index = uniq_pairs_cat.index, columns = ['date_block_num','shop_id','item_category_id'])
 empty_df[['shop_id', 'item_category_id']] = uniq_pairs_cat[['shop_id','item_category_id']]
empty_df_2 = pd.DataFrame(columns = ['date_block_num','shop_id','item_category_id'])
 for i in range(34):
empty_df_1 = empty_df.copy()
empty_df_1['date_block_num'] = i
empty_df_2 = pd.concat([empty_df_2, empty_df_1] )<merge>
oof = np.zeros(len(train_df))
predictions = np.zeros(( len(test_df),nfold))
i = 1
for train_index, valid_index in skf.split(train_df, train_df.target.values):
print("
fold {}".format(i))
xg_train = lgb.Dataset(train_df.iloc[train_index][predictors].values,
label=train_df.iloc[train_index][target].values,
feature_name=predictors,
free_raw_data = False 
)
 xg_valid = lgb.Dataset(train_df.iloc[valid_index][predictors].values,
label=train_df.iloc[valid_index][target].values,
feature_name=predictors,
free_raw_data = False 
)
 clf = lgb.train(param_lgb, xg_train, 5000, valid_sets = [xg_valid], verbose_eval=250, early_stopping_rounds = 50)
 oof[valid_index] = clf.predict(train_df.iloc[valid_index][predictors].values, num_iteration=clf.best_iteration)
 predictions[:,i-1] += clf.predict(test_df[predictors], num_iteration=clf.best_iteration)
 i = i + 1
print("
CV AUC: {:<0.2f}".format(metrics.roc_auc_score(train_df.target.values, oof)) )
full_train_monthly_by_category = empty_df_2.merge(train_monthly_by_category,
on=['date_block_num','shop_id', 'item_category_id'],how='left' ).fillna(0 ).drop(['index'], axis=1 )<create_dataframe>
print("Rank averaging on", nfold, "fold predictions")
 rank_predictions = np.zeros(( predictions.shape[0],1))
for i in range(nfold):
rank_predictions[:, 0] = np.add(rank_predictions[:, 0], rankdata(predictions[:, i].reshape(-1,1)) /rank_predictions.shape[0])
 rank_predictions /= nfold
empty_df = pd.DataFrame(index = uniq_cat, columns = ['date_block_num','item_category_id'])
 empty_df['item_category_id'] = uniq_cat
empty_df_2 = pd.DataFrame(columns = ['date_block_num','item_category_id'])
 for i in range(34):
empty_df_1 = empty_df.copy()
empty_df_1['date_block_num'] = i
empty_df_2 = pd.concat([empty_df_2, empty_df_1] )<merge>
sub_df = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub_df["target"] = rank_predictions
sub_df[:10]
<feature_engineering><EOS> 
sub_df.to_csv("Customer_Transaction_rank_predictions.csv", index=False )