Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<feature_engineering>
warnings.filterwarnings('ignore' )
if ShopCatIt:
this_ID = 6004
this_shop = df_out.loc[this_ID,'shop_id']
this_cat = df_out.loc[this_ID,'item_category_id']
shop_frac = shop_cat_ctab.T.apply(np.sum)[this_shop]
cat_frac = shop_cat_ctab.apply(np.sum)[this_cat]
print("Actual ctab: {} Approx product: {}".format(shop_cat_ctab.loc[this_shop, this_cat],shop_frac*cat_frac))<categorify>
df = pd.read_csv('.. /input/train.csv')
 x_data = df.drop(['ID_code','target'],axis=1 ).values
y_data = df['target'].values
df_test = pd.read_csv('.. /input/test.csv')
 x_test = df_test.drop('ID_code',axis=1 ).values
if ShopCatIt:
df_out['shop_frac'] = df_out['shop_id'].map(shops_fracs)
 df_out['cat_frac'] = df_out['item_category_id'].map(cats_fracs)
 df_out.fillna(0.0, inplace=True)
<feature_engineering>
np.max(np.triu(np.corrcoef(x_data,rowvar=False),k=1))
if ShopCatIt:
df_out['item_frac'] = 0.0
sc_categories = list(shop_cat_ctab.columns)
 ic_categories = list(item_cat_ctab.columns)
 for this_cat in df_out['item_category_id'].unique() :
if(this_cat in ic_categories):
items_fracs =(item_cat_ctab[this_cat]/sum(item_cat_ctab[this_cat]))
select = df_out['item_category_id'] == this_cat
df_out.loc[select,'item_frac'] = df_out.loc[select,'item_id'].map(items_fracs)
 if(this_cat in sc_categories):
df_out.loc[select,'shopcat_frac'] = df_out.loc[select,'shop_id'].map(shop_cat_ctab[this_cat])
 df_out.fillna(0.0, inplace=True)
<groupby>
np.max(np.triu(np.corrcoef(x_data[y_data==1],rowvar=False),k=1))
current_missing_sum = sum(df_out.set_index('item_id' ).loc[missing_items,'item_cnt_month'])
 num_missing_itshops = len(missing_items)* num_out_shops
if True:
print("
The sum of the missing items is {:.2f}, giving an average of {:.3f} per item-shop
".format(
 current_missing_sum, current_missing_sum/num_missing_itshops))
missing_value = 0.475
if SHIFT_1MONTH:
missing_value = 0.479
for imiss in range(len(missing_items)) :
df_out.loc[df_out['item_id'] == missing_items[imiss] ,'item_cnt_month'] = missing_value
print("Set {} missing items to {}".format(len(missing_items), missing_value))
newhot20_value = 3.60
newhot28_value = 6.6
newhot47_value = 0.530
if SHIFT_1MONTH:
newhot20_value = 3.76
newhot28_value = 6.79
newhot47_value = 1.19
for imiss in range(len(newhot20_items)) :
df_out.loc[df_out['item_id'] == newhot20_items[imiss] ,'item_cnt_month'] = newhot20_value
print("Set {} newhot-20 items to {}".format(len(newhot20_items), newhot20_value))
for imiss in range(len(newhot28_items)) :
df_out.loc[df_out['item_id'] == newhot28_items[imiss] ,'item_cnt_month'] = newhot28_value
print("Set {} newhot-28 items to {}".format(len(newhot28_items), newhot28_value))
for imiss in range(len(newhot47_items)) :
df_out.loc[df_out['item_id'] == newhot47_items[imiss] ,'item_cnt_month'] = newhot47_value
print("Set {} newhot-47 items to {}".format(len(newhot47_items), newhot47_value))
newhot37_value = 1.8
newhot40_value = 0.30
newhot55_value = 0.85
if SHIFT_1MONTH:
newhot37_value = 1.81
newhot40_value = 0.95
newhot55_value = 0.62
for imiss in range(len(newhot37_items)) :
df_out.loc[df_out['item_id'] == newhot37_items[imiss] ,'item_cnt_month'] = newhot37_value
print("Set {} newhot-37 items to {}".format(len(newhot37_items), newhot37_value))
for imiss in range(len(newhot40_items)) :
df_out.loc[df_out['item_id'] == newhot40_items[imiss] ,'item_cnt_month'] = newhot40_value
print("Set {} newhot-40 items to {}".format(len(newhot40_items), newhot40_value))
for imiss in range(len(newhot55_items)) :
df_out.loc[df_out['item_id'] == newhot55_items[imiss] ,'item_cnt_month'] = newhot55_value
print("Set {} newhot-55 items to {}".format(len(newhot55_items), newhot55_value))
cnt_5441 = 0.0
cnt_6439 = 0.0
df_out.loc[df_out['item_id'] == 5441 ,'item_cnt_month'] = cnt_5441
df_out.loc[df_out['item_id'] == 6439 ,'item_cnt_month'] = cnt_6439
print("Set 1 item 5541 to {}".format(cnt_5441))
print("Set 1 item 6439 to {}".format(cnt_6439))
if ShopCatIt:
shop_modulation = 0.34
if SHIFT_1MONTH:
shop_modulation = 0.8
for imiss in range(len(missing_items)) :
select =(df_out['item_id'] == missing_items[imiss])
 df_out.loc[select, 'item_cnt_month'] =(df_out.loc[select, 'item_cnt_month'] * 
(1.0 - shop_modulation + shop_modulation*42.0*df_out.loc[select, 'shop_frac']))
print("Scaled all {} missing items by shop_frac with modulation = {:.3f}.".format(
 len(missing_items),shop_modulation))
current_missing_sum = sum(df_out.set_index('item_id' ).loc[missing_items,'item_cnt_month'])
 else:
print("The {} 'missing items' were left as is.".format(len(missing_items)))
 if SHIFT_1MONTH:
for imiss in range(len(future_items)) :
df_out.loc[df_out['item_id'] == future_items[imiss] ,'item_cnt_month'] = 0.0
print("
* Set {} Future items to {} *".format(len(future_items), 0.0))
print(" Leaving {} missing-minus-future items.".format(len(missing_items)- len(future_items)))
 current_missing_sum = sum(df_out.set_index('item_id' ).loc[missing_items,'item_cnt_month'])
 num_missing_itshops =(len(missing_items)- len(future_items)) * num_out_shops
print("The sum of the missing-minus-future items is {:.2f}, giving an average of {:.3f} per item-shop".format(
 current_missing_sum, current_missing_sum/num_missing_itshops))
else:
print("
The sum of the missing items is {:.2f}, giving an average of {:.3f} per item-shop".format(
 current_missing_sum, current_missing_sum/num_missing_itshops))<feature_engineering>
def GetAucRoc(x_train, x_test, y_train, y_test,
train_test_merger = False,
n_bins = 23,
range_limit = 3,
bins_finding_method = 'Equal steps',
n_vars = 200,
smoothing_type=None,
smoothing_epsilon=0):

 y_hat, y_hat_test = GetSubmission(x_train, x_test, y_train,
train_test_merger = train_test_merger,
n_bins = n_bins,
range_limit = range_limit,
bins_finding_method = bins_finding_method,
n_vars = n_vars,
smoothing_type = smoothing_type,
smoothing_epsilon = smoothing_epsilon)
 res = list()
res.append(roc_auc_score(y_train,y_hat))
res.append(roc_auc_score(y_test,y_hat_test))
return res
def GetSubmission(x_train, x_test, y_train,
train_test_merger = False,
n_bins = 23,
range_limit = 3,
bins_finding_method = 'Equal steps',
n_vars = 200,
smoothing_type = None,
smoothing_epsilon = 0):

 Scaler = StandardScaler()
if train_test_merger:
x_data = np.vstack(( x_train,x_test))
else:
x_data = x_train.copy()
x_data_scaled = Scaler.fit_transform(x_data)
 x_train_scaled = Scaler.transform(x_train)
 x_test_scaled = Scaler.transform(x_test)
 bins = list()
if bins_finding_method == "Equal normal probability":
for i in range(n_vars):
norm_limit = norm.cdf(range_limit,loc=0,scale=1)
 bins.append(norm.isf(np.linspace(norm_limit, 1-norm_limit, n_bins+1),loc=0,scale=1))
elif bins_finding_method == "Linearly changing probability":
for i in range(n_vars):
norm_limit = norm.cdf(range_limit,loc=0,scale=1)
 u = np.arange(1,n_bins//2+n_bins%2)
 u = np.hstack(( u,np.flip(u)[n_bins%2:]))
u_cum = np.cumsum(u)
 u_sum = u.sum()
v =(2*norm_limit-1)*(1-np.r_[0,u_cum/u_sum])+(1-norm_limit)
 bins.append(norm.isf(v,loc=0,scale=1))
elif bins_finding_method == "Equal a posteriori probability":
for i in range(n_vars):
r = np.zeros(n_bins+1)
 r[0] = - range_limit
r[-1] = range_limit
u =(np.floor(np.linspace(0,x_data_scaled[:,i].size,n_bins+1)) [1:-1] ).astype(int)
 v = x_data_scaled[:,i].copy()
v.sort()
r[1:-1] = np.minimum(np.maximum(( v[u] + v[u+1])/ 2,-range_limit),range_limit)
 bins.append(r)
 else:
for i in range(n_vars):
bins.append(np.linspace(-range_limit, range_limit, n_bins+1))
P_Vij = list()
for i in range(n_vars):
a0 =(x_data_scaled[:,i] < bins[i][0] ).sum()
if smoothing_type is None:
a10 =(x_data_scaled[:,i].reshape(x_data_scaled[:,i].shape[0],1)>= bins[i][:-1])
 a11 =(x_data_scaled[:,i].reshape(x_data_scaled[:,i].shape[0],1)< bins[i][1:])
 a1 =(a10 & a11 ).sum(axis=0)
 elif smoothing_type == 'plus_minus_epsilon':
a10 =(x_data_scaled[:,i].reshape(x_data_scaled[:,i].shape[0],1)>= bins[i][:-1]-smoothing_epsilon)
 a11 =(x_data_scaled[:,i].reshape(x_data_scaled[:,i].shape[0],1)< bins[i][1:]+smoothing_epsilon)
 a1 =(a10 & a11 ).sum(axis=0)*(bins[i][1:]-bins[i][:-1])/(bins[i][1:]-bins[i][:-1]+2*smoothing_epsilon)
 a2 =(x_data_scaled[:,i] >= bins[i][n_bins] ).sum()
P_Vij.append(np.r_[a0,a1,a2]/x_data_scaled[:,i].size)
 P_Vij_given_T = list()
for i in range(n_vars):
a0 =(( x_train_scaled[:,i] < bins[i][0])* y_train ).sum()
if smoothing_type is None:
a10 =(x_train_scaled[:,i].reshape(x_train_scaled[:,i].shape[0],1)>= bins[i][:-1])
 a11 =(x_train_scaled[:,i].reshape(x_train_scaled[:,i].shape[0],1)< bins[i][1:])
 a1 =(( a10 & a11)*y_train.reshape(y_train.shape[0],1)).sum(axis=0)
 elif smoothing_type == 'plus_minus_epsilon':
a10 =(x_train_scaled[:,i].reshape(x_train_scaled[:,i].shape[0],1)>= bins[i][:-1]-smoothing_epsilon)
 a11 =(x_train_scaled[:,i].reshape(x_train_scaled[:,i].shape[0],1)< bins[i][1:]+smoothing_epsilon)
 a1 =(( a10 & a11)*y_train.reshape(y_train.shape[0],1)).sum(axis=0)*(bins[i][1:]-bins[i][:-1])/(bins[i][1:]-bins[i][:-1]+2*smoothing_epsilon)
 a2 =(( x_train_scaled[:,i] >= bins[i][n_bins])* y_train ).sum()
P_Vij_given_T.append(np.r_[a0,a1,a2]/x_train_scaled[(y_train==1),i].size)
 P_T = y_train.sum() / y_train.size
P_T_given_Vij = list()
for i in range(n_vars):
P_T_given_Vij.append(np.nan_to_num(P_Vij_given_T[i] * P_T / P_Vij[i]))
y_hat = np.ones(( x_train_scaled.shape[0],),dtype=float)
 y_hat = y_hat * P_T
for i in range(n_vars):
in_bin = n_bins + 1 -(x_train_scaled[:,i].reshape(( x_train_scaled[:,i].shape[0],1)) < bins[i] ).sum(axis=1)
 y_hat = y_hat * P_T_given_Vij[i][in_bin] / P_T
y_hat_test = np.ones(( x_test_scaled.shape[0],),dtype=float)
 y_hat_test = y_hat_test * P_T
for i in range(n_vars):
in_bin = n_bins + 1 -(x_test_scaled[:,i].reshape(( x_test_scaled[:,i].shape[0],1)) < bins[i] ).sum(axis=1)
 y_hat_test = y_hat_test * P_T_given_Vij[i][in_bin] / P_T
res = list()
res.append(y_hat)
 res.append(y_hat_test)
 return res
clip_level = 20
num_over20 = sum(df_out['item_cnt_month'] > clip_level)
 print("
Check for over-20 values, found/clipped {} of them.".format(num_over20))
if num_over20 > 0:
df_out.loc[df_out['item_cnt_month'] > clip_level, 'item_cnt_month'] = clip_level
print("
Sum of output item_cnt_month: {:.0f}, average count-per-month per shop-item: {:.3f}
".format(
 df_out['item_cnt_month'].sum() , df_out['item_cnt_month'].sum() /(len(df_out))))<save_to_csv>
subm = GetSubmission(x_data, x_test, y_data,
train_test_merger=False,
n_bins=1000,
bins_finding_method='Equal steps',
smoothing_type = 'plus_minus_epsilon',
smoothing_epsilon = 0.3 )
<load_from_csv><EOS> 
subm = pd.DataFrame({'ID_code':df_test['ID_code'], 'target':pd.Series(subm[1])})
 subm.to_csv('submissionS.csv', index=False )