Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<categorify>
tqdm.pandas()
supercategory_month_of_release_encoding=encoding_2(all_items,['month_id','item_supercategory_id','item_month_of_release'],'supercategory','month_of_release','',supercategory_encoding,month_of_release_encoding)
 supercategory_month_of_release_encoding_seniority_0=encoding_2(( train_X['item_seniority']==0),['month_id','item_supercategory_id','item_month_of_release'],'supercategory','month_of_release','_seniority_0',supercategory_encoding_seniority_0,month_of_release_encoding_seniority_0)
 supercategory_month_of_release_encoding_seniority_1=encoding_2(( train_X['item_seniority']==1),['month_id','item_supercategory_id','item_month_of_release'],'supercategory','month_of_release','_seniority_1',supercategory_encoding_seniority_1,month_of_release_encoding_seniority_1)
 supercategory_month_of_release_encoding_seniority_2=encoding_2(( train_X['item_seniority']==2),['month_id','item_supercategory_id','item_month_of_release'],'supercategory','month_of_release','_seniority_2',supercategory_encoding_seniority_2,month_of_release_encoding_seniority_2)
 print_infos(supercategory_month_of_release_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(supercategory_month_of_release_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(supercategory_month_of_release_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(supercategory_month_of_release_encoding_seniority_2)
 print()
display_6(supercategory_month_of_release_encoding,'supercategory_month_of_release_avg_sales' )<categorify>
%matplotlib inline
%precision 4
warnings.filterwarnings('ignore')
 plt.style.use('ggplot')
 np.set_printoptions(suppress=True)
 pd.set_option("display.precision", 15 )
category_months_since_first_sale_in_shop_encoding=encoding_2(all_items,['month_id','item_category_id','item_months_since_first_sale_in_shop'],'category','months_since_first_sale_in_shop','',category_encoding,months_since_first_sale_in_shop_encoding)
 category_months_since_first_sale_in_shop_encoding_seniority_0=encoding_2(( train_X['item_seniority']==0),['month_id','item_category_id','item_months_since_first_sale_in_shop'],'category','months_since_first_sale_in_shop','_seniority_0',category_encoding_seniority_0,months_since_first_sale_in_shop_encoding_seniority_0)
 category_months_since_first_sale_in_shop_encoding_seniority_1=encoding_2(( train_X['item_seniority']==1),['month_id','item_category_id','item_months_since_first_sale_in_shop'],'category','months_since_first_sale_in_shop','_seniority_1',category_encoding_seniority_1,months_since_first_sale_in_shop_encoding_seniority_1)
 category_months_since_first_sale_in_shop_encoding_seniority_2=encoding_2(( train_X['item_seniority']==2),['month_id','item_category_id','item_months_since_first_sale_in_shop'],'category','months_since_first_sale_in_shop','_seniority_2',category_encoding_seniority_2,months_since_first_sale_in_shop_encoding_seniority_2)
 category_months_since_first_sale_in_shop_encoding.loc[category_months_since_first_sale_in_shop_encoding['item_months_since_first_sale_in_shop']==-1,category_months_since_first_sale_in_shop_encoding.columns[3:]]=-1
category_months_since_first_sale_in_shop_encoding_seniority_0.loc[category_months_since_first_sale_in_shop_encoding_seniority_0['item_months_since_first_sale_in_shop']==-1,category_months_since_first_sale_in_shop_encoding_seniority_0.columns[3:]]=-1
category_months_since_first_sale_in_shop_encoding_seniority_1.loc[category_months_since_first_sale_in_shop_encoding_seniority_1['item_months_since_first_sale_in_shop']==-1,category_months_since_first_sale_in_shop_encoding_seniority_1.columns[3:]]=-1
category_months_since_first_sale_in_shop_encoding_seniority_2.loc[category_months_since_first_sale_in_shop_encoding_seniority_2['item_months_since_first_sale_in_shop']==-1,category_months_since_first_sale_in_shop_encoding_seniority_2.columns[3:]]=-1
print_infos(category_months_since_first_sale_in_shop_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(category_months_since_first_sale_in_shop_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(category_months_since_first_sale_in_shop_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(category_months_since_first_sale_in_shop_encoding_seniority_2)
 print()
display_6(category_months_since_first_sale_in_shop_encoding_seniority_2,'category_months_since_first_sale_in_shop_avg_sales_seniority_2' )<categorify>
train= pd.read_csv(".. /input/train.csv")
 test = pd.read_csv('.. /input/test.csv')
 sample_submission = pd.read_csv('.. /input/sample_submission.csv' )
supercategory_months_since_first_sale_in_shop_encoding=encoding_2(all_items,['month_id','item_supercategory_id','item_months_since_first_sale_in_shop'],'supercategory','months_since_first_sale_in_shop','',supercategory_encoding,months_since_first_sale_in_shop_encoding)
 supercategory_months_since_first_sale_in_shop_encoding_seniority_0=encoding_2(( train_X['item_seniority']==0),['month_id','item_supercategory_id','item_months_since_first_sale_in_shop'],'supercategory','months_since_first_sale_in_shop','_seniority_0',supercategory_encoding_seniority_0,months_since_first_sale_in_shop_encoding_seniority_0)
 supercategory_months_since_first_sale_in_shop_encoding_seniority_1=encoding_2(( train_X['item_seniority']==1),['month_id','item_supercategory_id','item_months_since_first_sale_in_shop'],'supercategory','months_since_first_sale_in_shop','_seniority_1',supercategory_encoding_seniority_1,months_since_first_sale_in_shop_encoding_seniority_1)
 supercategory_months_since_first_sale_in_shop_encoding_seniority_2=encoding_2(( train_X['item_seniority']==2),['month_id','item_supercategory_id','item_months_since_first_sale_in_shop'],'supercategory','months_since_first_sale_in_shop','_seniority_2',supercategory_encoding_seniority_2,months_since_first_sale_in_shop_encoding_seniority_2)
 supercategory_months_since_first_sale_in_shop_encoding.loc[supercategory_months_since_first_sale_in_shop_encoding['item_months_since_first_sale_in_shop']==-1,supercategory_months_since_first_sale_in_shop_encoding.columns[3:]]=-1
supercategory_months_since_first_sale_in_shop_encoding_seniority_0.loc[supercategory_months_since_first_sale_in_shop_encoding_seniority_0['item_months_since_first_sale_in_shop']==-1,supercategory_months_since_first_sale_in_shop_encoding_seniority_0.columns[3:]]=-1
supercategory_months_since_first_sale_in_shop_encoding_seniority_1.loc[supercategory_months_since_first_sale_in_shop_encoding_seniority_1['item_months_since_first_sale_in_shop']==-1,supercategory_months_since_first_sale_in_shop_encoding_seniority_1.columns[3:]]=-1
supercategory_months_since_first_sale_in_shop_encoding_seniority_2.loc[supercategory_months_since_first_sale_in_shop_encoding_seniority_2['item_months_since_first_sale_in_shop']==-1,supercategory_months_since_first_sale_in_shop_encoding_seniority_2.columns[3:]]=-1
print_infos(supercategory_months_since_first_sale_in_shop_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(supercategory_months_since_first_sale_in_shop_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(supercategory_months_since_first_sale_in_shop_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(supercategory_months_since_first_sale_in_shop_encoding_seniority_2)
 print()
display_6(supercategory_months_since_first_sale_in_shop_encoding_seniority_2,'supercategory_months_since_first_sale_in_shop_avg_sales_seniority_2' )<categorify>
shop_category_months_since_release_encoding=encoding_3(all_items,['month_id','shop_id','item_category_id','item_months_since_release'],'shop','category','months_since_release','')
 shop_category_months_since_release_encoding_seniority_0=encoding_3(( train_X['item_seniority']==0),['month_id','shop_id','item_category_id','item_months_since_release'],'shop','category','months_since_release','_seniority_0')
 shop_category_months_since_release_encoding_seniority_1=encoding_3(( train_X['item_seniority']==1),['month_id','shop_id','item_category_id','item_months_since_release'],'shop','category','months_since_release','_seniority_1')
 shop_category_months_since_release_encoding_seniority_2=encoding_3(( train_X['item_seniority']==2),['month_id','shop_id','item_category_id','item_months_since_release'],'shop','category','months_since_release','_seniority_2')
 print_infos(shop_category_months_since_release_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_category_months_since_release_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_category_months_since_release_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_category_months_since_release_encoding_seniority_2)
 print()<categorify>
def check_missing_data(df):
flag=df.isna().sum().any()
if flag==True:
total = df.isnull().sum()
percent =(df.isnull().sum())/(df.isnull().count() *100)
 output = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 data_type = []
for col in df.columns:
dtype = str(df[col].dtype)
 data_type.append(dtype)
 output['Types'] = data_type
return(np.transpose(output))
else:
return(False )
shop_category_months_since_first_sale_in_shop_encoding=encoding_3(all_items,['month_id','shop_id','item_category_id','item_months_since_first_sale_in_shop'],'shop','category','months_since_first_sale_in_shop','')
 shop_category_months_since_first_sale_in_shop_encoding_seniority_0=encoding_3(( train_X['item_seniority']==0),['month_id','shop_id','item_category_id','item_months_since_first_sale_in_shop'],'shop','category','months_since_first_sale_in_shop','_seniority_0')
 shop_category_months_since_first_sale_in_shop_encoding_seniority_1=encoding_3(( train_X['item_seniority']==1),['month_id','shop_id','item_category_id','item_months_since_first_sale_in_shop'],'shop','category','months_since_first_sale_in_shop','_seniority_1')
 shop_category_months_since_first_sale_in_shop_encoding_seniority_2=encoding_3(( train_X['item_seniority']==2),['month_id','shop_id','item_category_id','item_months_since_first_sale_in_shop'],'shop','category','months_since_first_sale_in_shop','_seniority_2')
 print_infos(shop_category_months_since_first_sale_in_shop_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_category_months_since_first_sale_in_shop_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_category_months_since_first_sale_in_shop_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_category_months_since_first_sale_in_shop_encoding_seniority_2)
 print()<categorify>
print('missing in train: ',check_missing_data(train))
print('missing in test: ',check_missing_data(test))
shop_supercategory_months_since_release_encoding=encoding_3(all_items,['month_id','shop_id','item_supercategory_id','item_months_since_release'],'shop','supercategory','months_since_release','')
 shop_supercategory_months_since_release_encoding_seniority_0=encoding_3(( train_X['item_seniority']==0),['month_id','shop_id','item_supercategory_id','item_months_since_release'],'shop','supercategory','months_since_release','_seniority_0')
 shop_supercategory_months_since_release_encoding_seniority_1=encoding_3(( train_X['item_seniority']==1),['month_id','shop_id','item_supercategory_id','item_months_since_release'],'shop','supercategory','months_since_release','_seniority_1')
 shop_supercategory_months_since_release_encoding_seniority_2=encoding_3(( train_X['item_seniority']==2),['month_id','shop_id','item_supercategory_id','item_months_since_release'],'shop','supercategory','months_since_release','_seniority_2')
 print_infos(shop_supercategory_months_since_release_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_supercategory_months_since_release_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_supercategory_months_since_release_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_supercategory_months_since_release_encoding_seniority_2)
 print()<categorify>
shop_supercategory_months_since_first_sale_in_shop_encoding=encoding_3(all_items,['month_id','shop_id','item_supercategory_id','item_months_since_first_sale_in_shop'],'shop','supercategory','months_since_first_sale_in_shop','')
 shop_supercategory_months_since_first_sale_in_shop_encoding_seniority_0=encoding_3(( train_X['item_seniority']==0),['month_id','shop_id','item_supercategory_id','item_months_since_first_sale_in_shop'],'shop','supercategory','months_since_first_sale_in_shop','_seniority_0')
 shop_supercategory_months_since_first_sale_in_shop_encoding_seniority_1=encoding_3(( train_X['item_seniority']==1),['month_id','shop_id','item_supercategory_id','item_months_since_first_sale_in_shop'],'shop','supercategory','months_since_first_sale_in_shop','_seniority_1')
 shop_supercategory_months_since_first_sale_in_shop_encoding_seniority_2=encoding_3(( train_X['item_seniority']==2),['month_id','shop_id','item_supercategory_id','item_months_since_first_sale_in_shop'],'shop','supercategory','months_since_first_sale_in_shop','_seniority_2')
 print_infos(shop_supercategory_months_since_first_sale_in_shop_encoding)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_supercategory_months_since_first_sale_in_shop_encoding_seniority_0)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_supercategory_months_since_first_sale_in_shop_encoding_seniority_1)
 print()
print('------------------------------------------------')
 print()
print_infos(shop_supercategory_months_since_first_sale_in_shop_encoding_seniority_2)
 print()<define_variables>
idx = features = train.columns.values[2:202]
for df in [train, test]:
df['sum'] = df[idx].sum(axis=1)
 df['min'] = df[idx].min(axis=1)
 df['max'] = df[idx].max(axis=1)
 df['mean'] = df[idx].mean(axis=1)
 df['std'] = df[idx].std(axis=1)
 df['skew'] = df[idx].skew(axis=1)
 df['kurt'] = df[idx].kurtosis(axis=1)
 df['med'] = df[idx].median(axis=1)
dfs=['month_encoding',
'months_since_release_encoding',
'month_of_release_encoding',
'months_since_first_sale_in_shop_encoding',
'month_of_first_sale_in_shop_encoding',
'shop_encoding',
'item_encoding',
'category_encoding',
'supercategory_encoding',
'shop_category_encoding',
'shop_supercategory_encoding',
'category_months_since_release_encoding',
'supercategory_months_since_release_encoding',
'category_month_of_release_encoding',
'supercategory_month_of_release_encoding',
'category_months_since_first_sale_in_shop_encoding',
'supercategory_months_since_first_sale_in_shop_encoding',
'shop_category_months_since_release_encoding',
'shop_supercategory_months_since_release_encoding',
'shop_category_months_since_first_sale_in_shop_encoding',
'shop_supercategory_months_since_first_sale_in_shop_encoding'
]<categorify>
cols=["target","ID_code"]
X = train.drop(cols,axis=1)
 y = train["target"]
test_ID = test["ID_code"]
create_directory(os.path.join(DATA_FOLDER,'processed/target_encodings'))
for df in dfs+['shop_item_encoding']:
exec(df+".to_pickle(os.path.join(DATA_FOLDER,'processed/target_encodings/"+df+".pkl')) ")
 exec("del "+df)
 for df in dfs:
for seniority in range(0,3):
exec(df+"_seniority_"+str(seniority)+".to_pickle(os.path.join(DATA_FOLDER,'processed/target_encodings/"+df+"_seniority_"+str(seniority)+".pkl')) ")
 exec("del "+df+"_seniority_"+str(seniority))<drop_column>
X_test = test.drop("ID_code",axis=1)
del train_X
del items, item_categories
gc.collect()<define_variables>
train_X, val_X, train_y, val_y = train_test_split(X, y, random_state=1 )
reset_variable_space<categorify>
fold_n=5
folds = StratifiedKFold(n_splits=fold_n, shuffle=True, random_state=10 )
def lag_features(df,df_features,col_agg,col_lag,lagging_values,fill_value=None):
col_agg=list(col_agg)
 col_lag=list(col_lag)
 def lag_dataframe(df_original,lag):
df_shift=df_original.copy()
df_shift['month_id']+=lag
for col in col_lag:
df_shift.rename({col:col+'_lag_'+str(lag)},axis=1,inplace=True)
 return df_shift
df_features_lag=df.loc[:,col_agg]
tmp=df_features.loc[:,col_agg+col_lag]
for lag in lagging_values:
df_features_lag=df_features_lag.join(lag_dataframe(tmp,lag ).set_index(col_agg),on=col_agg)
 if fill_value is not None:
for col in col_lag:
for lag in lagging_values:
df_features_lag[col+'_lag_'+str(lag)].fillna(fill_value,inplace=True)
 return df_features_lag.drop(col_agg,axis=1 )<create_dataframe>
params = {
'bagging_freq': 5,
'bagging_fraction': 0.4,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.05,
'learning_rate': 0.01,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': -1,
'reg_alpha': 0.1,
'reg_lambda': 8
}
def moving_statistics(df,df_features,col_agg,col_avg,windowsize,func,suffix='',fill_value=None):
col_agg=list(col_agg)
 col_lag=list(col_avg)
 def lag_dataframe(df_original,lag):
df_shift=df_original.copy()
df_shift['month_id']+=lag
for col in col_lag:
df_shift.rename({col:col+'_lag_'+str(lag)},axis=1,inplace=True)
 return df_shift
df_features_lag=df.loc[:,col_agg]
tmp=df_features.loc[:,col_agg+col_lag]
for lag in range(1,windowsize+1):
df_features_lag=df_features_lag.join(lag_dataframe(tmp,lag ).set_index(col_agg),on=col_agg)
 df_features_avg=df.loc[:,col_agg]
for col in col_avg:
if func=='mean':
if suffix=='':
suffix='_movavg_'+str(windowsize)
 df_features_avg[col+suffix]=df_features_lag.loc[:,[col+'_lag_'+str(lag)for lag in range(1,windowsize+1)]].mean(axis=1)
 elif func=='max':
if suffix=='':
suffix='_movmax_'+str(windowsize)
 df_features_avg[col+suffix]=df_features_lag.loc[:,[col+'_lag_'+str(lag)for lag in range(1,windowsize+1)]].max(axis=1)
 elif func=='min':
if suffix=='':
suffix='_movmin_'+str(windowsize)
 df_features_avg[col+suffix]=df_features_lag.loc[:,[col+'_lag_'+str(lag)for lag in range(1,windowsize+1)]].min(axis=1)
 elif func=='std':
if suffix=='':
suffix='_movstd_'+str(windowsize)
 df_features_avg[col+suffix]=df_features_lag.loc[:,[col+'_lag_'+str(lag)for lag in range(1,windowsize+1)]].std(ddof=0,axis=1)
 elif func=='rsd':
if suffix=='':
suffix='_movrsd_'+str(windowsize)
 denom_avg=df_features_lag.loc[:,[col+'_lag_'+str(lag)for lag in range(1,windowsize+1)]].mean(axis=1)
 num_std=df_features_lag.loc[:,[col+'_lag_'+str(lag)for lag in range(1,windowsize+1)]].std(ddof=0,axis=1)
 df_features_avg[col+suffix]=num_std/denom_avg
df_features_avg.loc[num_std==0,col+suffix]=0
del denom_avg
del num_std
if fill_value is not None:
for col in col_lag:
df_features_avg[col+suffix].fillna(fill_value,inplace=True)
 return df_features_avg.drop(col_agg,axis=1 )<create_dataframe>
%%time
y_pred_lgb = np.zeros(len(X_test))
for fold_n,(train_index, valid_index)in enumerate(folds.split(X,y)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 lgb_model = lgb.train(params,train_data,num_boost_round=100000,
valid_sets = [train_data, valid_data],verbose_eval=1000,early_stopping_rounds = 3000)
 y_pred_lgb += lgb_model.predict(X_test, num_iteration=lgb_model.best_iteration)/5
def temporal_compare(df,df_features,col_agg,col_avg,lags_num,lags_denom,suffix,fill_value=None):
col_agg=list(col_agg)
 col_lag=list(col_avg)
 def lag_dataframe(df_original,lag):
df_shift=df_original.copy()
df_shift['month_id']+=lag
for col in col_lag:
df_shift.rename({col:col+'_lag_'+str(lag)},axis=1,inplace=True)
 return df_shift
tmp=df_features.loc[:,col_agg+col_lag]
df_features_lag_num=df.loc[:,col_agg]
for lag in lags_num:
df_features_lag_num=df_features_lag_num.join(lag_dataframe(tmp,lag ).set_index(col_agg),on=col_agg)
 df_features_lag_denom=df.loc[:,col_agg]
for lag in lags_denom:
df_features_lag_denom=df_features_lag_denom.join(lag_dataframe(tmp,lag ).set_index(col_agg),on=col_agg)
 df_features_comp=df.loc[:,col_agg]
for col in col_avg:
df_features_lag_num[col+'_avg']=df_features_lag_num.loc[:,[col+'_lag_'+str(lag)for lag in lags_num]].mean(axis=1)
 df_features_lag_denom[col+'_avg']=df_features_lag_denom.loc[:,[col+'_lag_'+str(lag)for lag in lags_denom]].mean(axis=1)
 df_features_comp[col+suffix]=df_features_lag_num[col+'_avg']/df_features_lag_denom[col+'_avg']
if fill_value is not None:
for col in col_lag:
df_features_comp[col+suffix].fillna(fill_value,inplace=True)
 return df_features_comp.drop(col_agg,axis=1 )<concatenate>
train_preds_lgb = np.zeros(len(X))
train_preds_lgb = lgb_model.predict(X, num_iteration=lgb_model.best_iteration)/5
auc_lgb = round(roc_auc_score(train['target'], train_preds_lgb), 4 )
def rational_fraction(df,df_features,col_agg,col_avg,weights_num,weights_denom,suffix,fill_value=None):
col_agg=list(col_agg)
 col_lag=list(col_avg)
 windowsize=len(weights_num)
 def lag_dataframe(df_original,lag):
df_shift=df_original.copy()
df_shift['month_id']+=lag
for col in col_lag:
df_shift.rename({col:col+'_lag_'+str(lag)},axis=1,inplace=True)
 return df_shift
tmp=df_features.loc[:,col_agg+col_lag]
df_features_lag_num=df.loc[:,col_agg]
for lag in range(1,windowsize+1):
df_features_lag_num=df_features_lag_num.join(lag_dataframe(tmp,lag ).set_index(col_agg)*weights_num[lag-1],on=col_agg)
 df_features_lag_denom=df.loc[:,col_agg]
for lag in range(1,windowsize+1):
df_features_lag_denom=df_features_lag_denom.join(lag_dataframe(tmp,lag ).set_index(col_agg)*weights_denom[lag-1],on=col_agg)
 df_features_comp=df.loc[:,col_agg]
for col in col_avg:
df_features_lag_num[col+'_avg']=df_features_lag_num.loc[:,[col+'_lag_'+str(lag)for lag in range(1,windowsize+1)]].sum(axis=1)
 df_features_lag_denom[col+'_avg']=df_features_lag_denom.loc[:,[col+'_lag_'+str(lag)for lag in range(1,windowsize+1)]].sum(axis=1)
 df_features_comp[col+suffix]=df_features_lag_num[col+'_avg']/df_features_lag_denom[col+'_avg']
if fill_value is not None:
for col in col_lag:
df_features_comp[col+suffix].fillna(fill_value,inplace=True)
 return df_features_comp.drop(col_agg,axis=1 )<categorify>
train_features = train.drop(['target','ID_code'], axis = 1)
 test_features = test.drop(['ID_code'],axis = 1)
 train_target = train['target']
sc = StandardScaler()
train_features = sc.fit_transform(train_features)
 test_features = sc.transform(test_features)
 n_splits = 5
splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True ).split(train_features, train_target))
def time_mapping(df,df_features,join_labels,col_labels,map_to_label,suffix):
df_mapped=df.loc[:,[map_to_label]+join_labels]
shifted_df=df_features.loc[:,['month_id']+join_labels+col_labels].rename({'month_id':map_to_label},axis=1 ).set_index([map_to_label]+join_labels)
 shifted_df.rename({col:col+suffix for col in col_labels},axis=1,inplace=True)
 return df_mapped.join(shifted_df,on=[map_to_label]+join_labels ).drop([map_to_label]+join_labels,axis=1 )<merge>
class CyclicLR(object):
def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,
step_size=2000, mode='triangular', gamma=1.,
scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):
if not isinstance(optimizer, Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if isinstance(base_lr, list)or isinstance(base_lr, tuple):
if len(base_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} base_lr, got {}".format(
 len(optimizer.param_groups), len(base_lr)))
 self.base_lrs = list(base_lr)
 else:
self.base_lrs = [base_lr] * len(optimizer.param_groups)
 if isinstance(max_lr, list)or isinstance(max_lr, tuple):
if len(max_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} max_lr, got {}".format(
 len(optimizer.param_groups), len(max_lr)))
 self.max_lrs = list(max_lr)
 else:
self.max_lrs = [max_lr] * len(optimizer.param_groups)
 self.step_size = step_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.batch_step(last_batch_iteration + 1)
 self.last_batch_iteration = last_batch_iteration
def batch_step(self, batch_iteration=None):
if batch_iteration is None:
batch_iteration = self.last_batch_iteration + 1
self.last_batch_iteration = batch_iteration
for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
param_group['lr'] = lr
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):
step_size = float(self.step_size)
 cycle = np.floor(1 + self.last_batch_iteration /(2 * step_size))
x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)
 lrs = []
param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)
 for param_group, base_lr, max_lr in param_lrs:
base_height =(max_lr - base_lr)* np.maximum(0,(1 - x))
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)
 lrs.append(lr)
 return lrs
def feature_correlation_analysis(train_df,feature_df,feature_df_seniority,feature_name,groupby_labels,seniority):
tmp=feature_df_seniority.loc[(feature_df_seniority['month_id']>12)&(feature_df_seniority['month_id']<34),groupby_labels+[feature_name+'_avg_sales_seniority_'+str(seniority),feature_name+'_avg_sales_over_sold_seniority_'+str(seniority)]]
tmp2=feature_df.loc[(feature_df['month_id']>12)&(feature_df['month_id']<34),groupby_labels+[feature_name+'_avg_sales',feature_name+'_avg_sales_over_sold']]
tmp=tmp.join(tmp2.set_index(groupby_labels),on=groupby_labels)
 tmp_full=train_df[groupby_labels+['item_quantity']].join(tmp.set_index(groupby_labels),on=groupby_labels)
 fig=plt.figure(figsize=(5,5))
sns.heatmap(tmp_full.drop(groupby_labels,axis=1 ).corr() ,annot=True)
 def compare_features(df,df_seniority,name,labels,seniority,avg_sales_max=20,avg_sales_seniority_max=20,avg_sales_over_sold_max=20):
n=len(labels)
 tmp=df.join(df_seniority.set_index(labels),on=labels)
 labels.remove('month_id')
 labels.append('month')
 tmp['month']=tmp['month_id']%12+1
if name=='item':
fig,axes=plt.subplots(1,3,figsize=(15,5))
sns.scatterplot(data=tmp,x=name+'_avg_sales',y=name+'_avg_sales_seniority_'+str(seniority),hue='month',ax=axes[0])
 axes[0].plot([0,20],[0,20],'k')
 axes[0].grid(True)
 axes[0].set_xlim(0,avg_sales_max)
 axes[0].set_ylim(0,avg_sales_seniority_max)
 sns.scatterplot(data=tmp,x=name+'_avg_sales',y=name+'_avg_sales_over_sold',hue='month',ax=axes[1])
 axes[1].plot([0,20],[0,20],'k')
 axes[1].grid(True)
 axes[1].set_xlim(0,avg_sales_max)
 axes[1].set_ylim(0,avg_sales_over_sold_max)
 sns.scatterplot(data=tmp,x=name+'_avg_sales_over_sold',y=name+'_avg_sales_seniority_'+str(seniority),hue='month',ax=axes[2])
 axes[2].plot([0,20],[0,20],'k')
 axes[2].grid(True)
 axes[2].set_xlim(0,avg_sales_over_sold_max)
 axes[2].set_ylim(0,avg_sales_seniority_max)
 else:
fig,axes=plt.subplots(n,3,figsize=(15,n*5))
for i,label in enumerate(labels):
sns.scatterplot(data=tmp,x=name+'_avg_sales',y=name+'_avg_sales_seniority_'+str(seniority),hue=label,ax=axes[i,0])
 axes[i,0].plot([0,20],[0,20],'k')
 axes[i,0].grid(True)
 axes[i,0].set_xlim(0,avg_sales_max)
 axes[i,0].set_ylim(0,avg_sales_seniority_max)
 sns.scatterplot(data=tmp,x=name+'_avg_sales',y=name+'_avg_sales_over_sold',hue=label,ax=axes[i,1])
 axes[i,1].plot([0,20],[0,20],'k')
 axes[i,1].grid(True)
 axes[i,1].set_xlim(0,avg_sales_max)
 axes[i,1].set_ylim(0,avg_sales_over_sold_max)
 sns.scatterplot(data=tmp,x=name+'_avg_sales_over_sold',y=name+'_avg_sales_seniority_'+str(seniority),hue=label,ax=axes[i,2])
 axes[i,2].plot([0,20],[0,20],'k')
 axes[i,2].grid(True)
 axes[i,2].set_xlim(0,avg_sales_over_sold_max)
 axes[i,2].set_ylim(0,avg_sales_seniority_max )<set_options>
class Simple_NN(nn.Module):
def __init__(self ,input_dim ,hidden_dim, dropout = 0.1):
super(Simple_NN, self ).__init__()
self.inpt_dim = input_dim
self.hidden_dim = hidden_dim
self.relu = nn.ReLU()
self.dropout = nn.Dropout(dropout)
 self.fc1 = nn.Linear(input_dim, hidden_dim)
 self.fc2 = nn.Linear(hidden_dim, int(hidden_dim/2))
self.fc3 = nn.Linear(int(hidden_dim/2), int(hidden_dim/4))
self.fc4 = nn.Linear(int(hidden_dim/4), int(hidden_dim/8))
self.fc5 = nn.Linear(int(hidden_dim/8), 1)
 self.bn1 = nn.BatchNorm1d(hidden_dim)
 self.bn2 = nn.BatchNorm1d(int(hidden_dim/2))
self.bn3 = nn.BatchNorm1d(int(hidden_dim/4))
self.bn4 = nn.BatchNorm1d(int(hidden_dim/8))
def forward(self, x):
y = self.fc1(x)
 y = self.relu(y)
 y = self.dropout(y)
 y = self.fc2(y)
 y = self.relu(y)
 y = self.dropout(y)
 y = self.fc3(y)
 y = self.relu(y)
 y = self.dropout(y)
 y = self.fc4(y)
 y = self.relu(y)
 y = self.dropout(y)
 out= self.fc5(y)
 return out
loaded=%who_ls
loaded.append('loaded' )<load_pretrained>
def sigmoid(x):
return 1 /(1 + np.exp(-x))
train_X=pd.read_pickle(os.path.join(DATA_FOLDER,'processed/train_X0.pkl'))<create_dataframe>
model = Simple_NN(208,512)
 model.cuda()
optimizer = torch.optim.Adam(model.parameters() , lr = 0.0002 )
def moving_statistics_0(df,df_features,col_agg,col_avg,windowsize,func,suffix='',fill_value=None):
col_agg=list(col_agg)
 col_lag=list(col_avg)
 def lag_dataframe(df_original,lag):
df_shift=df_original.copy()
df_shift['month_id']+=lag
for col in col_lag:
df_shift.rename({col:col+'_lag_'+str(lag)},axis=1,inplace=True)
 return df_shift
df_features_lag=df.loc[:,col_agg]
tmp=df_features.loc[:,col_agg+col_lag]
for lag in range(0,windowsize):
df_features_lag=df_features_lag.join(lag_dataframe(tmp,lag ).set_index(col_agg),on=col_agg)
 df_features_avg=df.loc[:,col_agg]
for col in col_avg:
if func=='mean':
if suffix=='':
suffix='_movavg_'+str(windowsize)
 df_features_avg[col+suffix]=df_features_lag.loc[:,[col+'_lag_'+str(lag)for lag in range(0,windowsize)]].mean(axis=1)
 elif func=='max':
if suffix=='':
suffix='_movmax_'+str(windowsize)
 df_features_avg[col+suffix]=df_features_lag.loc[:,[col+'_lag_'+str(lag)for lag in range(0,windowsize)]].max(axis=1)
 elif func=='min':
if suffix=='':
suffix='_movmin_'+str(windowsize)
 df_features_avg[col+suffix]=df_features_lag.loc[:,[col+'_lag_'+str(lag)for lag in range(0,windowsize)]].min(axis=1)
 elif func=='std':
if suffix=='':
suffix='_movstd_'+str(windowsize)
 df_features_avg[col+suffix]=df_features_lag.loc[:,[col+'_lag_'+str(lag)for lag in range(0,windowsize)]].std(ddof=0,axis=1)
 elif func=='rsd':
if suffix=='':
suffix='_movrsd_'+str(windowsize)
 denom_avg=df_features_lag.loc[:,[col+'_lag_'+str(lag)for lag in range(0,windowsize)]].mean(axis=1)
 num_std=df_features_lag.loc[:,[col+'_lag_'+str(lag)for lag in range(0,windowsize)]].std(ddof=0,axis=1)
 df_features_avg[col+suffix]=num_std/denom_avg
df_features_avg.loc[num_std==0,col+suffix]=0
del denom_avg
del num_std
if fill_value is not None:
for col in col_lag:
df_features_avg[col+suffix].fillna(fill_value,inplace=True)
 return df_features_avg.drop(col_agg,axis=1)
 def categorical_variability_encoding(agg_label,name,windowsize_inner,suffix,fill_value=None,windowsize_outer=34):
df=train_X[['month_id','shop_id','item_id',agg_label,'item_quantity']]
tmp=moving_statistics_0(df,df,['month_id','shop_id','item_id'],['item_quantity'],windowsize_inner,'rsd',suffix,fill_value=None)
 tmp=pd.concat([df,tmp],axis=1,sort=False)
 tmp=tmp.groupby(['month_id',agg_label] ).agg({'item_quantity'+suffix:'mean'} ).rename({'item_quantity'+suffix:name+suffix},axis=1)
 tmp=moving_statistics(df,tmp.reset_index() ,['month_id',agg_label],[name+suffix],windowsize_outer,'mean','_',fill_value)
 tmp.rename({name+suffix+'_':name+suffix},axis=1,inplace=True)
 return tmp<categorify>
n_epochs = 40
batch_size = 512
train_preds = np.zeros(( len(train_features)))
 test_preds = np.zeros(( len(test_features)))
 x_test = np.array(test_features)
 x_test_cuda = torch.tensor(x_test, dtype=torch.float ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 avg_losses_f = []
avg_val_losses_f = []
for i,(train_idx, valid_idx)in enumerate(splits):
x_train = np.array(train_features)
 y_train = np.array(train_target)
 x_train_fold = torch.tensor(x_train[train_idx.astype(int)], dtype=torch.float ).cuda()
y_train_fold = torch.tensor(y_train[train_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(x_train[valid_idx.astype(int)], dtype=torch.float ).cuda()
y_val_fold = torch.tensor(y_train[valid_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
loss_fn = torch.nn.BCEWithLogitsLoss()
step_size = 2000
base_lr, max_lr = 0.0001, 0.005
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
lr=max_lr)
 scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,
step_size=step_size, mode='exp_range',
gamma=0.99994)
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(n_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for i,(x_batch, y_batch)in enumerate(train_loader):
y_pred = model(x_batch)
 if scheduler:
scheduler.batch_step()
loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() /len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(( len(test_features)))
 avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))
avg_losses_f.append(avg_loss)
 avg_val_losses_f.append(avg_val_loss)
 for i,(x_batch,)in enumerate(test_loader):
y_pred = model(x_batch ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits)
 auc_nn = round(roc_auc_score(train_target,train_preds),4)
 print('All \t loss={:.4f} \t val_loss={:.4f} \t auc={:.4f}'.format(np.average(avg_losses_f),np.average(avg_val_losses_f),auc_nn))
category_variability_rs=categorical_variability_encoding('item_category_id','category_semiannual_avg',3,'_recent_rsd',fill_value=None,windowsize_outer=6)
 category_variability_as=categorical_variability_encoding('item_category_id','category_semiannual_avg',12,'_annual_rsd',fill_value=None,windowsize_outer=6)
 train_X=pd.concat([train_X,category_variability_rs,category_variability_as],axis=1,sort=False)
 del category_variability_rs, category_variability_as<drop_column>
ensemble = 0.5*train_preds_lgb + 0.5* train_preds
ensemble_test = 0.5* y_pred_lgb + 0.5* test_preds
train_X.drop(train_X.loc[train_X['month_id']<18].index,axis=0,inplace=True )<create_dataframe>
print('LightBGM auc = {:<8.5f}'.format(auc_lgb))
print('NN auc = {:<8.5f}'.format(auc_nn))
print('NN+LightBGM auc = {:<8.5f}'.format(roc_auc_score(train_target, ensemble)) )
train_0=train_X.loc[train_X['item_seniority']==0,:]
train_1=train_X.loc[train_X['item_seniority']==1,:]
train_2=train_X.loc[train_X['item_seniority']==2,:]
pd.DataFrame([train_X.count().rename('global'),train_0.count().rename('seniority_0'),train_1.count().rename('seniority_1'),train_2.count().rename('seniority_2')] ).T<load_pretrained>
submission_lgb = pd.DataFrame({
"ID_code": test_ID,
"target": y_pred_lgb
})
 submission_lgb.to_csv('submission_lgb.csv', index=False )
os.remove(os.path.join(DATA_FOLDER,'processed/train_X0.pkl'))
train_0.to_pickle(os.path.join(DATA_FOLDER,'processed/train_0.pkl'))
train_1.to_pickle(os.path.join(DATA_FOLDER,'processed/train_1.pkl'))
train_2.to_pickle(os.path.join(DATA_FOLDER,'processed/train_2.pkl'))<drop_column>
submission_nn = pd.DataFrame({
"ID_code": test_ID,
"target": test_preds
})
 submission_nn.to_csv('submission_nn.csv', index=False )
del train_X
del train_0
del train_1
del train_2
gc.collect()<define_variables>
submission_ens = pd.DataFrame({
"ID_code": test_ID,
"target": ensemble_test
})
 submission_ens.to_csv('submission_ens.csv', index=False )
<load_pretrained><EOS>