Kaggle competition: Santander Customer Transaction Prediction
alles = lag_feature(alles, [1, 2, 3, 6, 12], 'item_cnt_month' )<feature_engineering>
seed = 2357
random.seed(seed )
alles['proj_item_cnt'] = 2 *(alles['item_cnt_month_lag_1'] -
alles['item_cnt_month_lag_2'])+ alles['item_cnt_month_lag_2']
alles['proj_item_cnt_2'] = 3 *(alles['item_cnt_month_lag_1'] -
alles['item_cnt_month_lag_3'])/ 2 + alles['item_cnt_month_lag_3']
alles['proj_item_cnt_12'] = 12 *(alles['item_cnt_month_lag_1'] -
alles['item_cnt_month_lag_12'])/ 11 + alles['item_cnt_month_lag_12']<feature_engineering>
train_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/train.csv')
 test_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/test.csv' )
alles['mean_proj_item_cnt'] = np.mean([alles['proj_item_cnt'],
alles['proj_item_cnt_2'], alles['proj_item_cnt_12']], axis=0 )<concatenate>
index_f_data = np.load('.. /input/split-test-dataset/index_of_fake_data.npy')
 index_r_data = np.load('.. /input/split-test-dataset/index_of_real_data.npy' )
alles = lag_feature(alles, [1, 2, 3, 6, 12], 'revenue')
 alles = lag_feature(alles, [1, 2, 3, 6, 12], 'revenue_usd' )<feature_engineering>
fold_information = pd.read_csv(".. /input/fold-information/10_fold_information.csv" )
alles['proj_monthly_revenue'] = 2 *(alles['revenue_lag_1'] -
alles['revenue_lag_2'])+ alles['revenue_lag_2']
alles['proj_monthly_revenue_2'] = 3 *(alles['revenue_lag_1'] -
alles['revenue_lag_3'])/ 2 + alles['revenue_lag_3']
alles['proj_monthly_revenue_12'] = 12 *(alles['revenue_lag_1'] -
alles['revenue_lag_12'])/ 11 + alles['revenue_lag_12']<feature_engineering>
splits=[]
for i in fold_information.columns:
a = np.array(list(set(train_df.index)-set(list(fold_information[i].dropna()))))
a.sort()
splits.append(( np.array(fold_information[i].dropna().astype(int)) ,a))
alles['mean_proj_revenue'] = np.mean([alles['proj_monthly_revenue'],
alles['proj_monthly_revenue_2'],
alles['proj_monthly_revenue_12']], axis=0 )<drop_column>
del fold_information
del a
gc.collect()
alles = alles.drop(['revenue', 'revenue_usd'], axis=1 )<groupby>
train_features = train_df.drop(['target','ID_code'], axis = 1)
 test_features = test_df.drop(['ID_code'],axis = 1)
 train_target = train_df['target']
gr_shop = train_df.groupby(['shop_id', 'date_block_num'] )<merge>
fake_data = test_features.iloc[index_f_data]
real_data = test_features.iloc[index_r_data]
pdf = pd.merge(all_vals, gr_shop.count() ['date'], on=['shop_id', 'date_block_num'], how='left' )<remove_duplicates>
train_all_real = pd.concat([train_features,real_data], axis = 0 )
pdf = pdf.rename(columns={'date':'purchases_per_month'})[['date_block_num',
'shop_id',
'purchases_per_month']].drop_duplicates()<merge>
for f in train_all_real.columns[0:200]:
train_all_real[f+'duplicate'] = train_all_real.duplicated(f,False ).astype(int )
tmp = pdf[['date_block_num','shop_id', 'purchases_per_month']]
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id', 'purchases_per_month'+'_lag_'+str(1)]
shifted['date_block_num'] += 1
pdf = pd.merge(pdf, shifted, on=['date_block_num','shop_id'], how='left' )<drop_column>
for f in train_all_real.columns[0:200]:
train_all_real[f+'duplicate_count'] = train_all_real.groupby([f])[f].transform('count')/300000
train_all_real[f+'duplicate_count'] = train_all_real[f+'duplicate_count']*train_all_real[f+'duplicate']
pdf = reduce_mem_usage(pdf )<drop_column>
for f in train_all_real.columns[0:200]:
train_all_real[f+'duplicate_value'] = train_all_real[f]*train_all_real[f+'duplicate']
pdf = pdf.drop(['purchases_per_month'], axis=1 )<merge>
train_features_real = train_all_real.iloc[:len(train_target)]
real_data = train_all_real.iloc[len(train_target):len(train_all_real)]
alles = pd.merge(alles, pdf, on=['shop_id', 'date_block_num'], how='left' )<create_dataframe>
del train_all_real
gc.collect()
cc_df = pd.DataFrame(gr_shop.mean().groupby('shop_id' ).cumcount() , columns=['active_months'])
 cc_df = cc_df.reset_index()<merge>
train_all_fake = pd.concat([train_features,fake_data], axis = 0 )
cc_df = pd.merge(all_vals[['date_block_num', 'shop_id']], cc_df,
on=['date_block_num', 'shop_id'], how='left' ).drop_duplicates()<merge>
for f in train_all_fake.columns[0:200]:
train_all_fake[f+'duplicate'] = train_all_fake.duplicated(f,False ).astype(int )
tmp = cc_df[['date_block_num','shop_id', 'active_months']]
for i in [1]:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id', 'active_months'+'_lag_'+str(i)]
shifted['date_block_num'] += i
cc_df = pd.merge(cc_df, shifted, on=['date_block_num','shop_id'], how='left' )<drop_column>
for f in train_all_fake.columns[0:200]:
train_all_fake[f+'duplicate_value'] = train_all_fake[f]*train_all_fake[f+'duplicate']
cc_df = cc_df.drop(['active_months'], axis=1 )<data_type_conversions>
train_features_fake = train_all_fake.iloc[:len(train_target)]
fake_data = train_all_fake.iloc[len(train_target):len(train_all_fake)]
for i in [1]:
cc_df['active_months_lag_'+str(i)] += i
cc_df['active_months_lag_'+str(i)] = cc_df['active_months_lag_'+str(i)].fillna(0 ).astype('int' )<merge>
del train_all_fake
gc.collect()
alles = pd.merge(alles, cc_df, on=['shop_id', 'date_block_num'], how='left' )<groupby>
for f in train_features_real.columns[0:200]:
train_features_real[f+'distance_of_mean'] = train_features_real[f]-train_features_real[f].mean()
real_data[f+'distance_of_mean'] = real_data[f]-real_data[f].mean()
train_features_real[f+'distance_of_mean'] = train_features_real[f+'distance_of_mean']*train_features_real[f+'duplicate']
real_data[f+'distance_of_mean'] = real_data[f+'distance_of_mean']*real_data[f+'duplicate']
gr_item = train_df.groupby(['item_id', 'date_block_num'] )<create_dataframe>
del train_features
del test_features
gc.collect()
cc_df = pd.DataFrame(gr_item.mean().groupby('item_id' ).cumcount() , columns=['item_active_months'])
 cc_df = cc_df.reset_index()<merge>
n_splits = 10
cc_df = pd.merge(all_vals[['date_block_num', 'item_id']], cc_df,
on=['date_block_num', 'item_id'], how='left' ).drop_duplicates()<merge>
param = {
'bagging_freq': 5,
'bagging_fraction': 0.33,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.05,
'learning_rate': 0.0085,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 12,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1
}
tmp = cc_df[['date_block_num','item_id', 'item_active_months']]
for i in [1]:
shifted = tmp.copy()
shifted.columns = ['date_block_num','item_id', 'item_active_months'+'_lag_'+str(i)]
shifted['date_block_num'] += i
cc_df = pd.merge(cc_df, shifted, on=['date_block_num','item_id'], how='left' )<drop_column>
oof = np.zeros(len(train_features_real))
predictions = np.zeros(len(test_df))
for i,(train_idx, valid_idx)in enumerate(splits):
print(f'Fold {i + 1}')
 x_train = np.array(train_features_fake)
 y_train = np.array(train_target)
 trn_data = lgb.Dataset(x_train[train_idx.astype(int)], label=y_train[train_idx.astype(int)])
 val_data = lgb.Dataset(x_train[valid_idx.astype(int)], label=y_train[valid_idx.astype(int)])
 num_round = 100000
clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 1000)
 oof[valid_idx] = clf.predict(x_train[valid_idx], num_iteration=clf.best_iteration)
 predictions[fake_data.index] += clf.predict(fake_data, num_iteration=clf.best_iteration)/ n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(train_target, oof)) )
cc_df = cc_df.drop(['item_active_months'], axis=1 )<data_type_conversions>
oof = np.zeros(len(train_features_real))
feature_importance_df = pd.DataFrame()
features = [c for c in train_features_real.columns if c not in ['ID_code', 'target']]
for i,(train_idx, valid_idx)in enumerate(splits):
print(f'Fold {i + 1}')
 x_train = np.array(train_features_real)
 y_train = np.array(train_target)
 trn_data = lgb.Dataset(x_train[train_idx.astype(int)], label=y_train[train_idx.astype(int)])
 val_data = lgb.Dataset(x_train[valid_idx.astype(int)], label=y_train[valid_idx.astype(int)])
 num_round = 100000
clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 1000)
 oof[valid_idx] = clf.predict(x_train[valid_idx], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = i + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions[real_data.index] += clf.predict(real_data, num_iteration=clf.best_iteration)/ n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(train_target, oof)) )
for i in [1]:
cc_df['item_active_months_lag_'+str(i)] += i
cc_df['item_active_months_lag_'+str(i)] = cc_df['item_active_months_lag_'+str(i)].fillna(0 ).astype('int' )<merge>
print('LightBGM auc = {:<8.5f}'.format(roc_auc_score(train_target, oof)))
alles = pd.merge(alles, cc_df, on=['item_id', 'date_block_num'], how='left' )<create_dataframe>
id_code_test = test_df['ID_code']
id_code_train = train_df['ID_code']
cc_df = pd.DataFrame(gr.mean().groupby(['shop_id','item_id'] ).cumcount() , columns=['shop_item_active_months'])
 cc_df = cc_df.reset_index()<merge>
my_submission_lbgm = pd.DataFrame({"ID_code" : id_code_test, "target" : predictions})
 my_submission_train = pd.DataFrame({"ID_code" : id_code_train, "target" : oof})
cc_df = pd.merge(all_vals, cc_df,
on=['date_block_num', 'shop_id', 'item_id'], how='left' ).drop_duplicates()<concatenate>
my_submission_lbgm.to_csv('submission_lbgm.csv', index = False, header = True)
 my_submission_train.to_csv('submission_lbgm_train.csv', index = False, header = True)