Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<count_unique_values>
print(os.listdir(".. /input"))
warnings.simplefilter(action='ignore', category=FutureWarning)
 warnings.filterwarnings('ignore')
 plt.style.use('seaborn')
 sns.set(font_scale=1 )
out_cats = np.sort(df_out.item_category_id.unique())
 num_out_cats = len(out_cats)
 print("
Number of categories appearing in Output: {}".format(num_out_cats))
print(out_cats)
 trans_cats = np.sort(df_trans.item_category_id.unique())
 print("
Number of categories in Transactions: {}".format(len(trans_cats)))
 print(trans_cats)
 print("")
 missing_out = False
missing_categories = []
for iout in out_cats:
if iout not in trans_cats:
print("Missing Output category: {}".format(iout))
missing_categories.append(iout)
 missing_out = True
if missing_out:
print("
Some Output categories are missing from the Transactions data.")
 else:
print("
All of the Output categories are in the Transaction data.
 " )<filter>
random_state = 42
np.random.seed(random_state)
 df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )
df_categs.loc[missing_categories]<define_variables>
train_X = df_train.drop(['ID_code', 'target'], axis = 1)
 test_X = df_test.drop(['ID_code'], axis = 1 )
<count_unique_values>
def augment(x,y,t=2):
xs,xn = [],[]
for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xs.append(x1)
 for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xn.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x,xs,xn])
 y = np.concatenate([y,ys,yn])
 return x,y
out_items = np.sort(df_out.item_id.unique())
 num_out_items = len(out_items)
 print("
Number of items in Output: {}".format(num_out_items))
print("
Output has {} items x {} shops = {} pairs;
which agrees with the length of df_out = {}".format(
 num_out_items, num_out_shops, num_out_shops * num_out_items, len(df_out)))
 trans_items = np.sort(df_trans.item_id.unique())
 print("
Number of items in Transactions: {}".format(len(trans_items)))
 print("")
 missing_items = []
for item in out_items:
if item not in trans_items:
missing_items.append(item)
 if len(missing_items)> 0:
print("Some Output Items, {} of them, are missing from the Transactions data:".format(
 len(missing_items)) ," 
(However, these all have known",
"categories,", "except for 5441 and 6439)
 ")
 if SHOW_EDA:
print(missing_items)
 else:
print("
All of the Test Items are in the Train data.
 ")
 plt.hist(missing_items, bins=50)
 plt.xlabel("item_id")
 plt.title("Output items that are missing from Transactions")
 plt.show()<filter>
mmscale = MinMaxScaler()
train_X_scaled = mmscale.fit_transform(train_X)
 test_X_scaled = mmscale.transform(test_X )
<define_variables>
pca = PCA()
factors_train = pca.fit_transform(train_X_scaled)
 factors_test = pca.transform(test_X_scaled )
trans_sum_cnt = df_trans['item_cnt_day'].sum()
trans_sum_clip = df_trans['item_cnt_clip'].sum()
print("
Total count = Sum of all item_cnt_day is: {}(clipped ~ {})".format(trans_sum_cnt,trans_sum_clip))
average_totcnt_month = trans_sum_cnt/num_months
average_totcnt_clip = trans_sum_clip/num_months
print("
Average Total-count /month is: {:.3f}(clipped ~ {:.3f})
 ".format(average_totcnt_month,average_totcnt_clip)) 
<categorify>
pca_columns_name = ["pca_" + str(col)for col in range(0, 200)]
factors_train = pd.DataFrame(factors_train, columns = pca_columns_name)
 factors_test = pd.DataFrame(factors_test, columns = pca_columns_name)
 train_pca = df_train.merge(factors_train, left_index = True, right_index = True)
 test_pca = df_test.merge(factors_test, left_index = True, right_index = True )
<filter>
important_factors = ['pca_2', 'pca_7', 'pca_14', 'pca_19']
vars_to_exclude = ['var_9', 'var_94', 'var_109', 'var_151']
df_train_columns = list(df_train.columns)
 df_test_columns = list(df_test.columns)
 train_columns_with_factors = important_factors + df_train_columns
test_columns_with_factors = important_factors + df_test_columns
train_main_pca = train_pca[train_columns_with_factors].drop(vars_to_exclude, axis = 1)
 test_main_pca = test_pca[test_columns_with_factors].drop(vars_to_exclude, axis = 1 )
its_missing = []
for iit in df_items.index:
its_missing.append(df_items.loc[iit,'item_id'] in missing_items)
 newhot20_items = df_items[(df_items['item_category_id'] == 20)& its_missing].item_id.values
newhot28_items = df_items[(df_items['item_category_id'] == 28)& its_missing].item_id.values
print("
Missing items that are in Cats 20 or 28:")
 print(newhot28_items,"
",newhot20_items)
<filter>
df_train = train_main_pca
df_test = test_main_pca
its_missing = []
for iit in df_items.index:
its_missing.append(df_items.loc[iit,'item_id'] in missing_items)
 newhot37_items = df_items[(df_items['item_category_id'] == 37)& its_missing].item_id.values
newhot40_items = df_items[(df_items['item_category_id'] == 40)& its_missing].item_id.values
newhot55_items = df_items[(df_items['item_category_id'] == 55)& its_missing].item_id.values
print("
Missing items that are in Cats 37, 40, 55:")
 print(newhot37_items,"
",newhot40_items,"
",newhot55_items )<define_variables>
lgb_params = {
"objective" : "binary",
"metric" : "auc",
"boosting": 'gbdt',
"max_depth" : -1,
"num_leaves" : 13,
"learning_rate" : 0.01,
"bagging_freq": 5,
"bagging_fraction" : 0.4,
"feature_fraction" : 0.05,
"min_data_in_leaf": 80,
"min_sum_heassian_in_leaf": 10,
"tree_learner": "serial",
"boost_from_average": "false",
"bagging_seed" : random_state,
"verbosity" : 1,
"seed": random_state
}
df_categs.loc[[9,20,28,47]]
its_missing = []
for iit in df_items.index:
its_missing.append(df_items.loc[iit,'item_id'] in missing_items)
 newhot47_items = df_items[(df_items['item_category_id'] == 47)& its_missing].item_id.values
print("
Missing items that are in Cat 47:")
 print(newhot47_items)
<groupby>
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=random_state)
 oof = df_train[['ID_code', 'target']]
oof['predict'] = 0
predictions = df_test[['ID_code']]
val_aucs = []
feature_importance_df = pd.DataFrame()
item_cnt_col = 'item_cnt_day'
if USE_CNT_CLIP:
item_cnt_col = 'item_cnt_clip'
totcnt_vs_month = df_trans[['date_block_num',item_cnt_col]].groupby('date_block_num' ).agg(sum)
 if USE_CNT_CLIP:
sel_totcnt_month = average_totcnt_clip
else:
sel_totcnt_month = average_totcnt_month
nov15_oct15_ratio = totcnt_vs_month.loc[22,item_cnt_col]/totcnt_vs_month.loc[21,item_cnt_col]
nov15_totcnt = totcnt_vs_month.loc[33,item_cnt_col] * nov15_oct15_ratio
nov15_ave_ratio = nov15_totcnt / sel_totcnt_month
dec15_totcnt =(totcnt_vs_month.loc[33,item_cnt_col]*
totcnt_vs_month.loc[23,item_cnt_col]/totcnt_vs_month.loc[21,item_cnt_col])
 ax = df_trans[['date_block_num',item_cnt_col]].groupby('date_block_num' ).agg(sum ).plot(
 kind='line',marker='o',figsize=(8,5))
plt.plot([34.0,34.0],[sel_totcnt_month,nov15_totcnt],color='gray')
 plt.plot([34],[nov15_totcnt],color='red',marker="*",markersize=15)
 plt.plot([35.0,35.0],[sel_totcnt_month,dec15_totcnt],color='green')
 plt.plot([35],[dec15_totcnt],color='green',marker="*",markersize=15)
 plt.plot([22],[totcnt_vs_month.loc[22,item_cnt_col]],color='red',marker="*",markersize=15)
 plt.plot([23],[totcnt_vs_month.loc[23,item_cnt_col]],color='green',marker="*",markersize=15)
 min_date_block = df_trans.date_block_num.min()
plt.plot([min_date_block,36],[sel_totcnt_month, sel_totcnt_month],color='gray')
 plt.ylim(0,)
 plt.xlim(min_date_block-1,36)
 plt.title("Total counts in each month.Stars: Nov(red), Dec(green)")
 plt.show()
print("Average line is at {:.1f}".format(sel_totcnt_month))
print("Nov'15 to average ratio is {:.4f}".format(nov15_ave_ratio))
print("Nov'15 to Oct'15 ratio is {:.4f}".format(nov15_oct15_ratio))
print("November 2015 total cnt ~ {:.1f}".format(nov15_totcnt)) 
<feature_engineering>
features = [col for col in df_train.columns if col not in ['target', 'ID_code']]
X_test = df_test[features].values
if False:
predict_avecnt = 0.265
print("
Using a manual-provided constant value of {:.3f}
".format(predict_avecnt))
df_out['item_cnt_month'] = predict_avecnt
if False:
if USE_CNT_CLIP:
predict_avecnt = nov15_totcnt /(len(out_shops)*len(out_items))
print("
Using a Nov'15 constant value of {:.3f}
".format(predict_avecnt))
df_out['item_cnt_month'] = predict_avecnt
else:
print("Best to set USE_CLIP_CNT = True for this estimate.")
<define_variables>
for fold,(trn_idx, val_idx)in enumerate(skf.split(df_train, df_train['target'])) :
X_train, y_train = df_train.iloc[trn_idx][features], df_train.iloc[trn_idx]['target']
X_valid, y_valid = df_train.iloc[val_idx][features], df_train.iloc[val_idx]['target']
N = 3
p_valid,yp = 0,0
for i in range(N):
X_t, y_t = augment(X_train.values, y_train.values)
 X_t = pd.DataFrame(X_t)
 X_t = X_t.add_prefix('var_')
 trn_data = lgb.Dataset(X_t, label=y_t)
 val_data = lgb.Dataset(X_valid, label=y_valid)
 evals_result = {}
lgb_clf = lgb.train(lgb_params,
trn_data,
100000,
valid_sets = [trn_data, val_data],
early_stopping_rounds=3000,
verbose_eval = 500,
evals_result=evals_result 
)
 p_valid += lgb_clf.predict(X_valid)
 yp += lgb_clf.predict(X_test)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = lgb_clf.feature_importance()
fold_importance_df["fold"] = fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 oof['predict'][val_idx] = p_valid/N
val_score = roc_auc_score(y_valid, p_valid)
 val_aucs.append(val_score)
 predictions['fold{}'.format(fold+1)] = yp/N
if False:
using_month = 10
using_year = 2015
if USE_CNT_CLIP:
print("
* Warning: better to have USE_CNT_CLIP = False for this.
 ")
 df_month = df_trans[(df_trans['year'] == using_year)&(df_trans['month'] == using_month)].copy()
out_vals = []
for iout in df_out.index:
this_shop = df_out.loc[iout,'shop_id']
this_item = df_out.loc[iout,'item_id']
out_vals.append(df_month.loc[(df_month['shop_id'] == this_shop)& 
(df_month['item_id'] == this_item),'item_cnt_day'].sum())
 print("
Unscaled cnt data from month {} of year {}.
 ".format(using_month, using_year))
df_out['item_cnt_month'] = np.array(out_vals )<define_variables>
mean_auc = np.mean(val_aucs)
 std_auc = np.std(val_aucs)
 all_auc = roc_auc_score(oof['target'], oof['predict'])
 print("Mean auc: %.9f, std: %.9f.All auc: %.9f." %(mean_auc, std_auc, all_auc))
<define_variables><EOS> 
predictions['target'] = np.mean(predictions[[col for col in predictions.columns if col not in ['ID_code', 'target']]].values, axis=1)
 predictions.to_csv('lgb_all_predictions.csv', index=None)
 sub_df = pd.DataFrame({"ID_code":df_test["ID_code"].values})
 sub_df["target"] = predictions['target']
sub_df.to_csv("lgb_submission.csv", index=False)
 oof.to_csv('lgb_oof.csv', index=False )