Kaggle competition: Santander Customer Transaction Prediction
top20_games = pd.merge(games_df, item_df ).sort_values(by='revenue', ascending=False ).head(20 )<groupby>
seed = 2357
random.seed(seed )
consoles_df = train_df.loc[train_df['item_category']=='Игровые консоли',
:].groupby('item_id', as_index=False ).sum() [['item_id', 'revenue', 'item_cnt_day']]<merge>
train_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/train.csv')
 test_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/test.csv' )
top20_consoles = pd.merge(consoles_df, item_df ).sort_values(by='revenue', ascending=False ).head(20 )<groupby>
index_f_data = np.load('.. /input/split-test-dataset/index_of_fake_data.npy')
 index_r_data = np.load('.. /input/split-test-dataset/index_of_real_data.npy' )
gr_date = train_df.groupby('date_block_num' )<create_dataframe>
fold_information = pd.read_csv(".. /input/fold-information/10_fold_information.csv" )
all_vals = pd.DataFrame()
for t in tqdm(t_d.values):
all_vals = pd.concat([all_vals, pd.DataFrame(t)], axis=0 )<rename_columns>
splits=[]
for i in fold_information.columns:
a = np.array(list(set(train_df.index)-set(list(fold_information[i].dropna()))))
a.sort()
splits.append(( np.array(fold_information[i].dropna().astype(int)) ,a))
all_vals.columns = ['date_block_num', 'shop_id', 'item_id']
all_vals.head()<create_dataframe>
del fold_information
del a
gc.collect()
test_vals = pd.DataFrame()
test_vals['shop_id'] = test_df['shop_id'].copy()
test_vals['item_id'] = test_df['item_id'].copy()
test_vals['date_block_num'] = 34<concatenate>
train_features = train_df.drop(['target','ID_code'], axis = 1)
 test_features = test_df.drop(['ID_code'],axis = 1)
 train_target = train_df['target']
all_vals = pd.concat([all_vals, test_vals], axis=0 )<groupby>
fake_data = test_features.iloc[index_f_data]
real_data = test_features.iloc[index_r_data]
gr_month = train_df.groupby(['shop_id', 'item_id', 'date_block_num'] )<rename_columns>
train_all_real = pd.concat([train_features,real_data], axis = 0 )
month_sum = month_sum.rename(columns={'item_cnt_day':'item_cnt_month'} )<feature_engineering>
for f in train_all_real.columns[0:200]:
train_all_real[f+'duplicate'] = train_all_real.duplicated(f,False ).astype(int )
month_sum['item_cnt_month'] = month_sum['item_cnt_month'].clip(0, 20 )<drop_column>
for f in train_all_real.columns[0:200]:
train_all_real[f+'duplicate_count'] = train_all_real.groupby([f])[f].transform('count' )
month_sum = month_sum.reset_index() [['shop_id', 'item_id',
'item_cnt_month', 'date_block_num',
'revenue', 'revenue_usd']]<merge>
for f in train_all_real.columns[0:200]:
train_all_real[f + '_duplicate_value_1'] = train_all_real[f]*(train_all_real[f + 'duplicate_count']>1 ).astype(int)
 train_all_real[f + '_duplicate_value_2'] = train_all_real[f]*(train_all_real[f + 'duplicate_count']>2 ).astype(int)
 train_all_real[f + '_duplicate_value_3'] = train_all_real[f]*(train_all_real[f + 'duplicate_count']>3 ).astype(int )
month_sum = pd.merge(all_vals, month_sum,
on=['shop_id','item_id', 'date_block_num'],
how='left' ).fillna(0 )<drop_column>
month_sum = reduce_mem_usage(month_sum)
 shop_df = reduce_mem_usage(shop_df)
 item_df = reduce_mem_usage(item_df )<merge>
train_all_real = train_all_real.drop(train_all_real.columns[400:600],axis = 1 )
month_sum = month_sum.merge(shop_df[['shop_id', 'city', 'shop_type']],
how='left', on=['shop_id'] )<merge>
train_features_real = train_all_real.iloc[:len(train_target)]
real_data = train_all_real.iloc[len(train_target):len(train_all_real)]
alles = month_sum.merge(item_df, how='left', on=['item_id'] )<feature_engineering>
del train_all_real
gc.collect()
exch_rate['month'] = exch_rate['date'].dt.month
exch_rate['year'] = exch_rate['date'].dt.year<groupby>
temp_mean = exch_rate.groupby(['year', 'month'] ).mean().reset_index()<feature_engineering>
for f in train_features_real.columns[0:200]:
train_features_real[f+'distance_of_mean'] = train_features_real[f]-train_features_real[f].mean()
real_data[f+'distance_of_mean'] = real_data[f]-real_data[f].mean()
train_features_real[f+'distance_of_mean'] = train_features_real[f+'distance_of_mean']*train_features_real[f+'duplicate']
real_data[f+'distance_of_mean'] = real_data[f+'distance_of_mean']*real_data[f+'duplicate']
temp_mean['date_block_num'] = range(35 )<feature_engineering>
del train_features
del test_features
gc.collect()
temp_mean = reduce_mem_usage(temp_mean )<merge>
n_splits = 10
alles = alles.merge(temp_mean, how='left', on=['date_block_num'] )<merge>
param = {
'bagging_freq': 5,
'bagging_fraction': 0.33,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.05,
'learning_rate': 0.0085,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 12,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1
}
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df<groupby>
gr = train_df.groupby(['item_id', 'shop_id', 'date_block_num'] )<compute_train_metric>
oof = np.zeros(len(train_features_real))
predictions = np.zeros(len(test_df))
feature_importance_df = pd.DataFrame()
features = [c for c in train_features_real.columns if c not in ['ID_code', 'target']]
for i,(train_idx, valid_idx)in enumerate(splits):
print(f'Fold {i + 1}')
 x_train = np.array(train_features_real)
 y_train = np.array(train_target)
 trn_data = lgb.Dataset(x_train[train_idx.astype(int)], label=y_train[train_idx.astype(int)])
 val_data = lgb.Dataset(x_train[valid_idx.astype(int)], label=y_train[valid_idx.astype(int)])
 num_round = 100000
clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 1000)
 oof[valid_idx] = clf.predict(x_train[valid_idx], num_iteration=clf.best_iteration)
 f
predictions[real_data.index] += clf.predict(real_data, num_iteration=clf.best_iteration)/ n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(train_target, oof)) )
m_price = gr.mean()<rename_columns>
predictions[fake_data.index] = 0.5
m_price = m_price.rename(columns={'revenue': 'av_revenue', 'revenue_usd': 'av_revenue_usd',
'item_cnt_day': 'av_item_cnt_day', 'item_price': 'av_item_price',
'price_usd': 'av_price_usd'} )<drop_column>
print('LightBGM auc = {:<8.5f}'.format(roc_auc_score(train_target, oof)))
m_price = m_price.drop(['rate'], axis=1 )<groupby>
id_code_test = test_df['ID_code']
id_code_train = train_df['ID_code']
m_price['item_purchase_days'] = gr.count() ['date']
m_price = m_price.reset_index()
m_price = reduce_mem_usage(m_price )<feature_engineering>
my_submission_lbgm = pd.DataFrame({"ID_code" : id_code_test, "target" : predictions})
 my_submission_train = pd.DataFrame({"ID_code" : id_code_train, "target" : oof})
<groupby><EOS> 
my_submission_lbgm.to_csv('submission_lbgm.csv', index = False, header = True)
 my_submission_train.to_csv('submission_lbgm_train.csv', index = False, header = True)