Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<groupby>
warnings.filterwarnings('ignore' )
full_train_monthly['item_cnt_next_month'] = full_train_monthly.sort_values('date_block_num' ).groupby(['shop_id','item_id'])['item_cnt'].shift(-1 )<groupby>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv" )
full_train_monthly_by_category['item_cnt_next_month'] = full_train_monthly_by_category.sort_values('date_block_num' ).groupby(['shop_id', 'item_category_id'])['item_cnt'].shift(-1 )<groupby>
x = train_df.drop(['ID_code', 'target'], axis=1)
 y = train_df['target']
x_test = test_df.drop(['ID_code'], axis =1 )
full_train_cat_no_shop['item_cnt_next_month'] = full_train_cat_no_shop.sort_values('date_block_num' ).groupby(['item_category_id'])['item_cnt'].shift(-1 )<categorify>
pd.DataFrame([mean_test, mean_train] )
full_train_monthly = pd.concat(
 [full_train_monthly, pd.get_dummies(full_train_monthly['year'], prefix='year')],axis=1 
)
 full_train_monthly = pd.concat(
 [full_train_monthly, pd.get_dummies(full_train_monthly['month'], prefix='month')],axis=1
)<categorify>
pd.DataFrame([std_test, std_train] )
full_train_monthly_by_category = pd.concat(
 [full_train_monthly_by_category, pd.get_dummies(full_train_monthly_by_category['year'], prefix='year')],axis=1 
)
 full_train_monthly_by_category = pd.concat(
 [full_train_monthly_by_category, pd.get_dummies(full_train_monthly_by_category['month'], prefix='month')],axis=1
)<categorify>
%%time
correlations = x[features].corr().abs().unstack().sort_values(kind="quicksort" ).reset_index()
correlations = correlations[correlations['level_0'] != correlations['level_1']]
full_train_cat_no_shop = pd.concat(
 [full_train_cat_no_shop, pd.get_dummies(full_train_cat_no_shop['year'], prefix='year')],axis=1 
)
 full_train_cat_no_shop = pd.concat(
 [full_train_cat_no_shop, pd.get_dummies(full_train_cat_no_shop['month'], prefix='month')],axis=1
)<merge>
gc.collect()
t = test.copy()
t = t.join(items, on='item_id', rsuffix='_' ).drop(['item_id_', 'item_name'], axis=1 )<create_dataframe>
x_test =(x_test - mean_test)+(mean_train)
 x_test = x_test /(std_train)
 x = x /(std_train )
t_train_cat = full_train_monthly_by_category.copy()<merge>
for df in [x, x_test]:
for feat in features:
df['r2_'+feat] = np.round(df[feat], 2)
 df['r2_'+feat] = np.round(df[feat], 2)
 df['sum'] = df[features].sum(axis=1)
 df['min'] = df[features].min(axis=1)
 df['max'] = df[features].max(axis=1)
 df['mean'] = df[features].mean(axis=1)
 df['std'] = df[features].std(axis=1)
 df['skew'] = df[features].skew(axis=1)
 df['kurt'] = df[features].kurtosis(axis=1)
 df['med'] = df[features].median(axis=1)
 print('Train and test shape:',x.shape, x_test.shape )
%%time
m_test = pd.merge(t, t_train[t_train.date_block_num==33], how = 'left', on=['shop_id', 'item_id'])
 m_test = m_test.rename(columns={'item_category_id_x':'item_category_id'})
 m_test = m_test.drop('item_category_id_y', axis=1 )<merge>
model = ExtraTreesClassifier()
model.fit(x,y )
%%time
null_test = pd.merge(m_test[m_test['item_cnt'].isnull() ][['ID', 'shop_id', 'item_id','item_category_id']],t_train_cat[t_train_cat.date_block_num==33],how = 'left',on = ['shop_id', 'item_category_id'])
 null_test.index = null_test['ID']
for i in m_test.columns:
m_test.loc[m_test.ID.isin(null_test.ID),i] = null_test[i]<merge>
impotrant_featues = {}
for i, j in enumerate(model.feature_importances_):
impotrant_featues[i] = j
<feature_engineering>
variables = [k for k in sorted(impotrant_featues, key=impotrant_featues.get, reverse=True)][:50]
m_test.loc[m_test['item_cnt'].isnull() ,'year'] = 2015
m_test.loc[m_test['item_cnt'].isnull() ,'month'] = 10
m_test.loc[m_test['item_cnt'].isnull() ,'month_10'] = 1
m_test.loc[m_test['item_cnt'].isnull() ,'date_block_num'] = 33<correct_missing_values>
imp_features = []
for var in variables:
if var <200:
imp_features.append("var_"+str(var))
m_test = m_test.fillna(0 )<drop_column>
def dist(a,b):
return distance.euclidean(a, b)
 for i in imp_features:
mean_1 = train_df[i][train_df['target']==1].mean()
mean_0 = train_df[i][train_df['target']==0].mean()
mean_1_col = i + "_mean_1_dist"
mean_0_col = i + "_mean_0_dist"
x[mean_1_col] = x.apply(lambda p: dist(p[i], mean_1), axis=1)
 x[mean_0_col] = x.apply(lambda p: dist(p[i], mean_0), axis=1)
 x_test[mean_1_col] = x_test.apply(lambda x: dist(x[i], mean_1), axis=1)
 x_test[mean_0_col] = x_test.apply(lambda x: dist(x[i], mean_0), axis=1)
 print("done ",i )
drop_cols = ['date_block_num',
'ID',
'shop_id',
'item_id',
'item_price_median',
'year',
'month',
'item_cnt_next_month',
'item_category_id']
X_test = m_test.drop(drop_cols, axis=1 )<save_to_csv>
param = {
'bagging_freq': 5,
'bagging_fraction': 0.38,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.045,
'learning_rate': 0.0095,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1,
'lambda_l1': 4.972,
'lambda_l2': 2.276,
'is_unbalance': True
}
X_test.to_csv('X_test.csv', index=False )<load_from_csv>
param = {'max_depth': 3,
'num_leaves': 6,
'min_child_samples': 200,
'scale_pos_weight': 1.0,
'subsample': 0.6,
'colsample_bytree': 0.6,
'metric': 'auc',
'nthread': 8,
'boosting_type': 'gbdt',
'objective': 'binary',
'learning_rate': 0.15,
'max_bin': 100,
'min_child_weight': 0,
'min_split_gain': 0,
'subsample_freq': 1,
'is_unbalance': True,
'bagging_freq': 5,
'bagging_fraction': 0.38,
'boost_from_average':'false',
'feature_fraction': 0.045,
'learning_rate': 0.0095,
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'tree_learner': 'serial'}
<load_from_csv>
param = {
'bagging_freq': 5,
'bagging_fraction': 0.335,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.041,
'learning_rate': 0.0083,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': -1
}
gc.collect()
<define_variables>
nfold = 5
k =StratifiedKFold(n_splits=nfold, shuffle=True, random_state=49)
 oof = np.zeros(len(y))
predictions = np.zeros(len(x_test))
i =1
for train_idx, val_idx in k.split(x, y.values):
print("
fold {}".format(i))
light_train = lightgbm.Dataset(x.iloc[train_idx].values,
label = y.iloc[train_idx].values,
free_raw_data = False)
 light_val = lightgbm.Dataset(x.iloc[val_idx].values,
label = y.iloc[val_idx].values,
free_raw_data = False)
 clf = lightgbm.train(param, light_train, 9000, valid_sets=[light_val], verbose_eval=200, early_stopping_rounds=100)
 oof[val_idx] = clf.predict(x.iloc[val_idx].values, num_iteration = clf.best_iteration)
 predictions += clf.predict(x_test.values, num_iteration= clf.best_iteration)/nfold
i+=1
%%time
train_set = t_train.query('date_block_num>=6 and date_block_num <33' ).copy()
train_set_cat = t_train_cat.query('date_block_num>=6 and date_block_num <33' ).copy()<train_model>
print("CV AUC: {:<0.2f}".format(metrics.roc_auc_score(y.values, oof)) )
for_fit = np.setxor1d(np.arange(rand_n), for_val)
 print(for_fit, for_fit.shape )<categorify>
sub_df = pd.DataFrame()
sub_df['ID_code'] = test_df['ID_code']
sub_df['target'] = predictions
sub_df.to_csv("sub1.csv", index = False )