Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<feature_engineering>
pd.set_option('display.max_columns', 200)
 InteractiveShell.ast_node_interactivity = "all"
warnings.filterwarnings('ignore')
train_X['item_sold_in_shop']=(train_X['item_quantity']>0 )<merge>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )
ts=time.time()
train_X=train_X.join(shops.set_index('shop_id'),on='shop_id')
 train_X=train_X.join(items.set_index('item_id'),on='item_id')
 train_X=train_X.join(item_categories.set_index('item_category_id'),on='item_category_id')
 print('time : ' +str(time.time() - ts))
gc.collect()<categorify>
train_df.isnull().sum().sum()
test_df.isnull().sum().sum()
ts=time.time()
train_X=train_X.join(train_X.loc[:,['month_id','item_category_id']].groupby('month_id')['item_category_id'].value_counts(normalize=True ).rename('item_category_freq',axis=1 ).astype(np.float32),on=['month_id','item_category_id'])
 train_X=train_X.join(train_X.loc[:,['month_id','item_supercategory_id']].groupby('month_id')['item_supercategory_id'].value_counts(normalize=True ).rename('item_supercategory_freq',axis=1 ).astype(np.float32),on=['month_id','item_supercategory_id'])
 train_X=train_X.join(train_X.loc[:,['month_id','item_category_console_id']].groupby('month_id')['item_category_console_id'].value_counts(normalize=True ).rename('item_category_console_freq',axis=1 ).astype(np.float32),on=['month_id','item_category_console_id'])
 train_X=train_X.join(train_X.loc[:,['month_id','item_category_is_digital']].groupby('month_id')['item_category_is_digital'].value_counts(normalize=True ).rename('item_category_digital_freq',axis=1 ).astype(np.float32),on=['month_id','item_category_is_digital'])
 print('time : ' +str(time.time() - ts))
gc.collect()<data_type_conversions>
def add_new_feature_row(df,features):
for feature in features:
df[feature+"_pct"] = df[feature].pct_change()
df[feature+"_diff"] = df[feature].diff()
df.drop(feature,axis=1)
 return df
ts=time.time()
train_X=train_X.join(train_X.loc[:,['month_id','item_seniority','item_id']].groupby(['month_id','item_seniority'])['item_id'].value_counts(normalize=True ).rename('item_freq_in_seniority',axis=1 ).astype(np.float32),on=['month_id','item_seniority','item_id'])
 train_X=train_X.join(train_X.loc[:,['month_id','item_seniority','shop_id']].groupby(['month_id','item_seniority'])['shop_id'].value_counts(normalize=True ).rename('shop_freq_in_seniority',axis=1 ).astype(np.float32),on=['month_id','item_seniority','shop_id'])
 train_X=train_X.join(train_X.loc[:,['month_id','item_seniority','item_category_id']].groupby(['month_id','item_seniority'])['item_category_id'].value_counts(normalize=True ).rename('item_category_freq_in_seniority',axis=1 ).astype(np.float32),on=['month_id','item_seniority','item_category_id'])
 train_X=train_X.join(train_X.loc[:,['month_id','item_seniority','item_supercategory_id']].groupby(['month_id','item_seniority'])['item_supercategory_id'].value_counts(normalize=True ).rename('item_supercategory_freq_in_seniority',axis=1 ).astype(np.float32),on=['month_id','item_seniority','item_supercategory_id'])
 train_X=train_X.join(train_X.loc[:,['month_id','item_seniority','item_category_console_id']].groupby(['month_id','item_seniority'])['item_category_console_id'].value_counts(normalize=True ).rename('item_category_console_freq_in_seniority',axis=1 ).astype(np.float32),on=['month_id','item_seniority','item_category_console_id'])
 train_X=train_X.join(train_X.loc[:,['month_id','item_seniority','item_category_is_digital']].groupby(['month_id','item_seniority'])['item_category_is_digital'].value_counts(normalize=True ).rename('item_category_digital_freq_in_seniority',axis=1 ).astype(np.float32),on=['month_id','item_seniority','item_category_is_digital'])
 print('time : ' +str(time.time() - ts))
gc.collect()<categorify>
def add_feature_df(df,features):
df['count+'] = np.array(df>0 ).sum(axis=1)
 df['count-'] = np.array(df<0 ).sum(axis=1)
 for feature in features:
df['tanh_'+feature] = np.tanh(df[feature])
 return df
train_df_expand=train_df.set_index(['month_id','shop_id','item_id'])['item_price']
train_df_expand=pd.DataFrame(train_df_expand.repeat(train_df['item_quantity']))
train_df_expand.reset_index(inplace=True)
 train_df_expand=pd.concat([train_df_expand,test_df],sort=False)
 train_df_expand['item_category_id']=train_df_expand['item_id'].map(items['item_category_id'])
 train_df_expand['item_supercategory_id']=train_df_expand['item_category_id'].map(item_categories['item_supercategory_id'])
 train_df_expand['item_category_console_id']=train_df_expand['item_category_id'].map(item_categories['item_category_console_id'] )<categorify>
gc.collect()
def encode_prices(groupby_labels,name):
df_prices=train_df_expand.groupby(groupby_labels ).agg({'item_price':['min','max','median','mean']})
 df_prices.columns=[name+'_price_'+col for col in df_prices.columns.get_level_values(1)]
df_prices.reset_index(inplace=True)
 df_prices=downcast_dtypes(df_prices)
 return df_prices
def compare_to_super(df,df_super,join_labels,label,label_super,name,name_super,df_mapping):
col_list=['price_min','price_max','price_median','price_mean']
df[label_super]=df[label].map(df_mapping[label_super])
 df=df.join(df_super[join_labels+[label_super]+[name_super+'_'+col for col in col_list]].set_index(join_labels+[label_super]),on=join_labels+[label_super])
 df.drop(label_super,axis=1,inplace=True)
 for col in col_list:
df[name_super+'_'+col]=df[name+'_'+col]/df[name_super+'_'+col]
df.rename({name_super+'_'+col:name+'_'+col+'_compared_to_'+name_super+'_'+col},axis=1,inplace=True)
 df=downcast_dtypes(df)
 return df
def compare_to_lower(df,df_lower,label_lower,name,name_lower):
col_list=['price_min','price_max','price_median','price_mean']
df=df.join(df_lower[['month_id',label_lower]+[name_lower+'_'+col for col in col_list]].set_index(['month_id',label_lower]),on=['month_id',label_lower])
 for col in col_list:
df[name_lower+'_'+col]=df[name+'_'+col]/df[name_lower+'_'+col]
df.rename({name_lower+'_'+col:name+'_'+col+'_compared_to_'+name_lower+'_'+col},axis=1,inplace=True)
 df=downcast_dtypes(df)
 return df<categorify>
test_df['target']= np.nan
combine_df = train_df.append(test_df,ignore_index=True )
supercategory_prices=encode_prices(['month_id','item_supercategory_id'],'supercategory')
 supercategory_prices['month_id'] = supercategory_prices['month_id'].astype(np.int8)
 print(supercategory_prices.loc[supercategory_prices['month_id']<34,:].info(null_counts=True))
supercategory_prices<data_type_conversions>
train_df = combine_df[combine_df['target'].notnull() ].reset_index(drop=True)
 test_df = combine_df[combine_df['target'].isnull() ].reset_index(drop=True)
category_prices=encode_prices(['month_id','item_category_id'],'category')
 category_prices=compare_to_super(category_prices,supercategory_prices,['month_id'],'item_category_id','item_supercategory_id','category','supercategory',item_categories)
 category_prices['month_id'] = category_prices['month_id'].astype(np.int8)
 print(category_prices.loc[category_prices['month_id']<34,:].info(null_counts=True))
category_prices<data_type_conversions>
fs = FeatureSelector(data = train_df[features], labels = train_df["target"] )
item_prices=encode_prices(['month_id','item_id'],'item')
 item_prices=compare_to_super(item_prices,category_prices,['month_id'],'item_id','item_category_id','item','category',items)
 item_prices['month_id'] = item_prices['month_id'].astype(np.int8)
 print(item_prices.loc[item_prices['month_id']<34,:].info(null_counts=True))
item_prices<data_type_conversions>
fs.identify_all(selection_params = {'missing_threshold': 0.6,
'correlation_threshold': 0.95,
'task': 'classification',
'eval_metric': 'auc',
'cumulative_importance': 0.95} )
shop_supercategory_prices=encode_prices(['month_id','shop_id','item_supercategory_id'],'shop_supercategory')
 shop_supercategory_prices=compare_to_lower(shop_supercategory_prices,supercategory_prices,'item_supercategory_id','shop_supercategory','supercategory')
 shop_supercategory_prices['month_id'] = shop_supercategory_prices['month_id'].astype(np.int8)
 shop_supercategory_prices['shop_id'] = shop_supercategory_prices['shop_id'].astype(np.int8)
 print(shop_supercategory_prices.loc[shop_supercategory_prices['month_id']<34,:].info(null_counts=True))
shop_supercategory_prices<data_type_conversions>
zero_importance_features = fs.ops['zero_importance']
zero_importance_features
shop_category_prices=encode_prices(['month_id','shop_id','item_category_id'],'shop_category')
 shop_category_prices=compare_to_lower(shop_category_prices,category_prices,'item_category_id','shop_category','category')
 shop_category_prices=compare_to_super(shop_category_prices,shop_supercategory_prices,['month_id','shop_id'],'item_category_id','item_supercategory_id','shop_category','shop_supercategory',item_categories)
 shop_category_prices['month_id'] = shop_category_prices['month_id'].astype(np.int8)
 shop_category_prices['shop_id'] = shop_category_prices['shop_id'].astype(np.int8)
 print(shop_category_prices.loc[shop_category_prices['month_id']<34,:].info(null_counts=True))
shop_category_prices<categorify>
train_removed_all = fs.remove(methods = 'all',
keep_one_hot=False )
shop_item_prices=encode_prices(['month_id','shop_id','item_id'],'shop_item')
 shop_item_prices=compare_to_lower(shop_item_prices,item_prices,'item_id','shop_item','item')
 shop_item_prices=compare_to_super(shop_item_prices,shop_category_prices,['month_id','shop_id'],'item_id','item_category_id','shop_item','shop_category',items)
 shop_item_prices['month_id'] = shop_item_prices['month_id'].astype(np.int8)
 shop_item_prices['shop_id'] = shop_item_prices['shop_id'].astype(np.int8)
 print(shop_item_prices.loc[shop_item_prices['month_id']<34,:].info(null_counts=True))
shop_item_prices<load_pretrained>
with open("feat_sel.csv","w")as fp:
for line in feat_sel:
fp.write(str(line)+"
")
create_directory(os.path.join(DATA_FOLDER, 'processed'))
create_directory(os.path.join(DATA_FOLDER, 'processed/price_features'))
train_X.to_pickle(os.path.join(DATA_FOLDER,'processed/train_X0.pkl'))
shop_item_prices.to_pickle(os.path.join(DATA_FOLDER,'processed/price_features/shop_item_prices.pkl'))
item_prices.to_pickle(os.path.join(DATA_FOLDER,'processed/price_features/item_prices.pkl'))
shop_category_prices.to_pickle(os.path.join(DATA_FOLDER,'processed/price_features/shop_category_prices.pkl'))
category_prices.to_pickle(os.path.join(DATA_FOLDER,'processed/price_features/category_prices.pkl'))
shop_supercategory_prices.to_pickle(os.path.join(DATA_FOLDER,'processed/price_features/shop_supercategory_prices.pkl'))
supercategory_prices.to_pickle(os.path.join(DATA_FOLDER,'processed/price_features/supercategory_prices.pkl'))<drop_column>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 test_df['target']= np.nan
combine_df = train_df.append(test_df,ignore_index=True)
 features = train_df.columns[~train_df.columns.isin(['target','ID_code'])]
combine_df = add_feature_df(combine_df,features)
 train_df = combine_df[combine_df['target'].notnull() ].reset_index(drop=True)
 test_df = combine_df[combine_df['target'].isnull() ].reset_index(drop=True )
del items
del item_categories
del shops
del train_df
del train_df_expand
del train_agg
del train_X
del shop_item_prices
del item_prices
del shop_category_prices
del category_prices
del shop_supercategory_prices
del supercategory_prices
gc.collect()<define_variables>
imp_df.sort_values(by="importance_gain",ascending=False ).to_csv("imp_lgb.csv", index=False)
reset_variable_space<load_pretrained>
imp_df.sort_values(by="importance_gain",ascending=False)[:150]
<feature_engineering><EOS> 
sub_df = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub_df["target"] = predictions
sub_df.to_csv("sant_lgb.csv", index=False)
 sub_df[:10]