Kaggle competition: Santander Customer Transaction Prediction
del a,b,b_over,b_under,df_count,df_percentage
gc.collect()<load_pretrained>
train_df = pd.read_csv('.. /input/train.csv')
 print('Rows: ',train_df.shape[0],'Columns: ',train_df.shape[1])
 train_df.info()
tmp=pd.read_pickle(os.path.join(DATA_FOLDER,'processed/train_X0.pkl'))
tmp=tmp[['month_id','item_seniority']].groupby('month_id')['item_seniority'].value_counts(normalize=True ).unstack()
print(tmp)
 del tmp<feature_engineering>
train_df['target'].value_counts()
tmp=train_X.loc[:,['month_id','item_seniority','item_sold_in_shop']]
tmp['all']=1
tmp['seniority_0']=(tmp['item_seniority']==0)
 tmp['seniority_1']=(tmp['item_seniority']==1)
 tmp['seniority_2']=(tmp['item_seniority']==2)
 tmp['sold_0']=tmp['item_sold_in_shop']&tmp['seniority_0']
tmp['sold_1']=tmp['item_sold_in_shop']&tmp['seniority_1']
tmp['sold_2']=tmp['item_sold_in_shop']&tmp['seniority_2']
tmp_count=tmp[['month_id','seniority_0','seniority_1','seniority_2','sold_0','sold_1','sold_2','item_sold_in_shop','all']].groupby('month_id' ).sum()
fig,axes=plt.subplots(2,2,figsize=(15,10))
axes[0,0].plot(tmp_count['seniority_0']/tmp_count['all'],'-o')
 axes[0,0].plot(tmp_count['seniority_1']/tmp_count['all'],'-o')
 axes[0,0].plot(tmp_count['seniority_2']/tmp_count['all'],'-o')
 axes[0,0].grid(True)
 axes[0,0].set_ylim(0,1)
 axes[0,0].set_title('proportion of items of each seniority in the dataset')
 axes[0,1].plot(tmp_count['item_sold_in_shop']/tmp_count['all'],'-o')
 axes[0,1].grid(True)
 axes[0,1].set_ylim(0,1)
 axes[0,1].set_title('fraction of items sold overall')
 axes[1,0].plot(tmp_count['sold_0']/tmp_count['item_sold_in_shop'],'-o')
 axes[1,0].plot(tmp_count['sold_1']/tmp_count['item_sold_in_shop'],'-o')
 axes[1,0].plot(tmp_count['sold_2']/tmp_count['item_sold_in_shop'],'-o')
 axes[1,0].grid(True)
 axes[1,0].set_ylim(0,1)
 axes[1,0].set_title('contribution of seniority to items sold in shop')
 axes[1,1].plot(tmp_count['sold_0']/tmp_count['seniority_0'],'-o')
 axes[1,1].plot(tmp_count['sold_1']/tmp_count['seniority_1'],'-o')
 axes[1,1].plot(tmp_count['sold_2']/tmp_count['seniority_2'],'-o')
 axes[1,1].grid(True)
 axes[1,1].set_ylim(0,1)
 axes[1,1].set_title('fraction of items sold per seniority')
 axes[1,1].legend(['seniority 0','seniority 1','seniority 2'])
 del fig,axes,tmp,tmp_count 
<drop_column>
test_df = pd.read_csv('.. /input/test.csv' )
train_X.drop(train_X.loc[train_X['month_id']<12].index,axis=0,inplace=True )<groupby>
X_test = test_df.drop('ID_code',axis=1 )
tmp_avg=train_X[['month_id','item_quantity']].groupby('month_id' ).mean()
tmp_avg.rename({'item_quantity':'overall'},axis=1,inplace=True)
 idxbool=(train_X['item_quantity']>0)
 tmp_avg['over_sold']=train_X.loc[idxbool,['month_id','item_quantity']].groupby('month_id' ).mean()
idxbool=(train_X['item_new'])
 tmp_avg['over_new']=train_X.loc[idxbool,['month_id','item_quantity']].groupby('month_id' ).mean()
for i in range(1,13):
idxbool=(train_X['item_months_since_release']==i)
 tmp_avg['over_released_since_'+str(i)+'_months']=train_X.loc[idxbool,['month_id','item_quantity']].groupby('month_id' ).mean()
idxbool=(train_X['item_never_sold_in_shop_before'])
 tmp_avg['over_never_sold_in_shop_before']=train_X.loc[idxbool,['month_id','item_quantity']].groupby('month_id' ).mean()
for i in range(1,13):
idxbool=(train_X['item_months_since_first_sale_in_shop']==i)
 tmp_avg['over_first_sold_in_shop_since_'+str(i)+'_months']=train_X.loc[idxbool,['month_id','item_quantity']].groupby('month_id' ).mean()
plt.figure(figsize=(15,10))
sns.heatmap(tmp_avg.T)
 plt.title('average sales')
 del tmp_avg<set_options>
X = train_df.drop(['ID_code','target'],axis=1)
 y = train_df['target']
del train_X
gc.collect()<define_variables>
n_fold = 5
folds = StratifiedKFold(n_splits=n_fold, shuffle=True, random_state=42 )
reset_variable_space<load_pretrained>
params = {'num_leaves': 8,
'min_data_in_leaf': 42,
'objective': 'binary',
'max_depth': 16,
'learning_rate': 0.0123,
'boosting': 'gbdt',
'bagging_freq': 5,
'bagging_fraction': 0.8,
'feature_fraction': 0.8201,
'bagging_seed': 11,
'reg_alpha': 1.728910519108444,
'reg_lambda': 4.9847051755586085,
'random_state': 42,
'metric': 'auc',
'verbosity': -1,
'subsample': 0.81,
'min_gain_to_split': 0.01077313523861969,
'min_child_weight': 19.428902804238373,
'num_threads': 4}
train_X=pd.read_pickle(os.path.join(DATA_FOLDER,'processed/train_X0.pkl'))
items=pd.read_pickle(os.path.join(DATA_FOLDER,'cleaned/items.pkl'))
item_categories=pd.read_pickle(os.path.join(DATA_FOLDER,'cleaned/item_categories.pkl'))<categorify>
prediction = np.zeros(len(X_test))
for fold_n,(train_index, valid_index)in enumerate(folds.split(X,y)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 model = lgb.train(params,train_data,num_boost_round=20000,
valid_sets = [train_data, valid_data],verbose_eval=300,early_stopping_rounds = 200)
 prediction += model.predict(X_test, num_iteration=model.best_iteration)/5
def encoder(agg_func,target_label,groupby_labels,idxbool=None,target_dtype=None):
if target_dtype is None:
target_dtype=train_X[target_label].dtype
if agg_func=='std':
if idxbool is None:
return train_X.loc[:,groupby_labels+[target_label]].groupby(groupby_labels,as_index=True ).agg(agg_func,ddof=0)[target_label].astype(target_dtype)
 return train_X.loc[idxbool,groupby_labels+[target_label]].groupby(groupby_labels,as_index=True ).agg(agg_func,ddof=0)[target_label].astype(target_dtype)
 else:
if idxbool is None:
return train_X.loc[:,groupby_labels+[target_label]].groupby(groupby_labels,as_index=True ).agg(agg_func)[target_label].astype(target_dtype)
 return train_X.loc[idxbool,groupby_labels+[target_label]].groupby(groupby_labels,as_index=True ).agg(agg_func)[target_label].astype(target_dtype )<merge>
train_pool = Pool(X,y)
 m = CatBoostClassifier(iterations=300,eval_metric="AUC", boosting_type = 'Ordered')
 m.fit(X,y,silent=True)
 y_pred1 = m.predict(X_test)
 m.best_score_
def encode_comparison(df,df_super,join_labels,target_labels,comparison_labels,new_labels):
df=df.join(df_super[join_labels+comparison_labels].set_index(join_labels),on=join_labels)
 for target_label,cp_label in zip(target_labels,comparison_labels):
df[cp_label]=df[target_label]/df[cp_label]
return df.rename({cp_label:new_label for(cp_label,new_label)in zip(comparison_labels,new_labels)},axis=1 )<categorify>
sub = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub["target"] = prediction
sub.to_csv("submission.csv", index=False )
all_items=(train_X['item_quantity']>=0)
 def encoding_0(idxbool,groupby_labels,prefix,suffix):
idxbool_sold=idxbool&train_X['item_sold_in_shop']
df=pd.DataFrame(encoder('mean','item_quantity',groupby_labels,idxbool)).rename({'item_quantity':prefix+'_avg_sales'+suffix},axis=1)
 df[prefix+'_fraction_of_items_sold'+suffix]=encoder('mean','item_sold_in_shop',groupby_labels,idxbool,target_dtype=np.float32)
 df[prefix+'_avg_sales_over_sold'+suffix]=encoder('mean','item_quantity',groupby_labels,idxbool_sold)
 df.reset_index(inplace=True)
 return df
def encoding_1(idxbool,groupby_labels,prefix,suffix):
idxbool_sold=idxbool&train_X['item_sold_in_shop']
df=pd.DataFrame(encoder('mean','item_quantity',groupby_labels,idxbool)).rename({'item_quantity':prefix+'_avg_sales'+suffix},axis=1)
 df[prefix+'_fraction_of_items_sold'+suffix]=encoder('mean','item_sold_in_shop',groupby_labels,idxbool,target_dtype=np.float32)
 df[prefix+'_avg_sales_over_sold'+suffix]=encoder('mean','item_quantity',groupby_labels,idxbool_sold)
 df[prefix+'_min_quantity_over_sold'+suffix]=encoder('min','item_quantity',groupby_labels,idxbool_sold)
 df[prefix+'_max_quantity'+suffix]=encoder('max','item_quantity',groupby_labels,idxbool)
 df.reset_index(inplace=True)
 return df
def add_comparison_super(df,idxbool,groupby_labels,prefix,suffix,df_super,label_super,prefix_super,df_mapping):
compared_features=['_avg_sales','_fraction_of_items_sold','_avg_sales_over_sold']
df[label_super]=df[groupby_labels[1]].map(df_mapping[label_super])
 df=encode_comparison(df,df_super,['month_id',label_super],[prefix+feature+suffix for feature in compared_features],[prefix_super+feature+suffix for feature in compared_features],[prefix+feature+'_compared_to_'+prefix_super+suffix for feature in compared_features])
 df.drop(label_super,axis=1,inplace=True)
 return df
def add_comparison_lower(df,idxbool,groupby_labels,prefix_1,prefix_2,suffix,df_compare_1,df_compare_2):
prefix=prefix_1+'_'+prefix_2
compared_features=['_avg_sales','_fraction_of_items_sold','_avg_sales_over_sold']
df=encode_comparison(df,df_compare_1,['month_id',groupby_labels[1]],[prefix+feature+suffix for feature in compared_features],[prefix_1+feature+suffix for feature in compared_features],[prefix+feature+'_compared_to_'+prefix_1+suffix for feature in compared_features])
 df=encode_comparison(df,df_compare_2,['month_id',groupby_labels[2]],[prefix+feature+suffix for feature in compared_features],[prefix_2+feature+suffix for feature in compared_features],[prefix+feature+'_compared_to_'+prefix_2+suffix for feature in compared_features])
 return df
def encoding_2(idxbool,groupby_labels,prefix_1,prefix_2,suffix,df_compare_1,df_compare_2):
prefix=prefix_1+'_'+prefix_2
df=encoding_1(idxbool,groupby_labels,prefix,suffix)
 return add_comparison_lower(df,idxbool,groupby_labels,prefix_1,prefix_2,suffix,df_compare_1,df_compare_2)
 def encoding_3(idxbool,groupby_labels,prefix_1,prefix_2,prefix_3,suffix):
prefix=prefix_1+'_'+prefix_2+'_'+prefix_3
df=encoding_1(idxbool,groupby_labels,prefix,suffix)
 return df<categorify>
sub1 = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub1["target"] = y_pred1
sub1.to_csv("submission1.csv", index=False )
month_encoding=encoding_0(all_items,['month_id'],'month','')
 month_encoding_seniority_0=encoding_0(( train_X['item_seniority']==0),['month_id'],'month','_seniority_0')
 month_encoding_seniority_1=encoding_0(( train_X['item_seniority']==1),['month_id'],'month','_seniority_1')
 month_encoding_seniority_2=encoding_0(( train_X['item_seniority']==2),['month_id'],'month','_seniority_2')
 print_infos(month_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(month_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(month_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(month_encoding_seniority_2)
 print()
fig,axes=plt.subplots(1,3,figsize=(15,4))
axes[0].plot(month_encoding['month_avg_sales'].values[:-1],'-o')
 axes[0].plot(month_encoding_seniority_0['month_avg_sales_seniority_0'].values[:-1],'-o')
 axes[0].plot(month_encoding_seniority_1['month_avg_sales_seniority_1'].values[:-1],'-o')
 axes[0].plot(month_encoding_seniority_2['month_avg_sales_seniority_2'].values[:-1],'-o')
 axes[0].grid(True)
 axes[0].set_ylim(0,1.5)
 axes[0].legend(['overall','seniority 0','seniority 1','seniority 2'])
 axes[0].set_title('average sales')
 axes[0].set_xlabel('month_id')
 axes[1].plot(month_encoding['month_fraction_of_items_sold'].values[:-1],'-o')
 axes[1].plot(month_encoding_seniority_0['month_fraction_of_items_sold_seniority_0'].values[:-1],'-o')
 axes[1].plot(month_encoding_seniority_1['month_fraction_of_items_sold_seniority_1'].values[:-1],'-o')
 axes[1].plot(month_encoding_seniority_2['month_fraction_of_items_sold_seniority_2'].values[:-1],'-o')
 axes[1].grid(True)
 axes[1].set_ylim(0,0.3)
 axes[1].set_title('fraction of items sold')
 axes[1].set_xlabel('month_id')
 axes[2].plot(month_encoding['month_avg_sales_over_sold'].values[:-1],'-o')
 axes[2].plot(month_encoding_seniority_0['month_avg_sales_over_sold_seniority_0'].values[:-1],'-o')
 axes[2].plot(month_encoding_seniority_1['month_avg_sales_over_sold_seniority_1'].values[:-1],'-o')
 axes[2].plot(month_encoding_seniority_2['month_avg_sales_over_sold_seniority_2'].values[:-1],'-o')
 axes[2].grid(True)
 axes[2].set_ylim(0,5)
 axes[2].set_title('average sales over sold')
 axes[2].set_xlabel('month_id' )<categorify>
sub2 = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub2["target"] =(prediction + y_pred1)/2
sub2.to_csv("submission2.csv", index=False )