Kaggle competition: Santander Customer Transaction Prediction
group3.iloc[0].avg_questions_seen<groupby>
warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 200)
 pd.set_option('display.max_rows', 200 )
results_u_final = train.loc[train.content_type_id == False, ['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_final.columns = ['answered_correctly_user']
results_u2_final = train.loc[train.content_type_id == False, ['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_final.columns = ['explanation_mean_user']<merge>
VERSION = 6
SEED = 42
train = pd.merge(train, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left' )<groupby>
test_df_src = pd.read_csv('.. /input/santander-customer-transaction-prediction/test.csv')
 train_df_src = pd.read_csv('.. /input/santander-customer-transaction-prediction/train.csv')
 with open('.. /input/data-separation/synthetic_samples_indexes.npy', 'rb')as f:
test_synth = np.load(f)
 test_synth = list(test_synth)
 with open('.. /input/data-separation/public_LB.npy', 'rb')as f:
test_pub = np.load(f)
 test_pub = list(test_pub.reshape(-1)[0])
 with open('.. /input/data-separation/private_LB.npy', 'rb')as f:
test_priv = np.load(f)
 test_priv = list(test_priv.reshape(-1)[0])
results_q_final = train.loc[train.content_type_id == False, ['question_id','answered_correctly']].groupby(['question_id'] ).agg(['mean'])
 results_q_final.columns = ['quest_pct']<groupby>
var_cols = [c for c in list(train_df_src.columns)if 'var_' in c]
results_q2_final = train.loc[train.content_type_id == False, ['question_id','part']].groupby(['question_id'] ).agg(['count'])
 results_q2_final.columns = ['count']<merge>
all_df_real = pd.concat([train_df_src, test_df_src.loc[test_pub + test_priv]],
axis=0, copy=False, sort=False ).set_index('ID_code' ).astype('float32')
 all_df_synth = pd.concat([train_df_src, test_df_src.loc[test_synth]],
axis=0, copy=False, sort=False ).set_index('ID_code' ).astype('float32' )
question2 = pd.merge(questions_df, results_q_final, left_on = 'question_id', right_on = 'question_id', how = 'left' )<merge>
def get_all_freq(df, columns, frqs_series=None):
freq_df = pd.DataFrame(index=df.index)
 f_s = frqs_series if frqs_series else {}
for col in tqdm(columns):
if not frqs_series:
f_s[col] = df[f'{col}'].value_counts()
freq_df[f'{col}_freq_N'] = df[f'{col}'].map(f_s[col])
 freq_df[f'{col}_freq_1'] =(freq_df[f'{col}_freq_N'] > 1 ).astype('category')
 freq_df[f'{col}_mul_freq'] = df[col]*freq_df[f'{col}_freq_N']
freq_df[f'{col}_div_freq'] = df[col]/freq_df[f'{col}_freq_N']
return freq_df, f_s
question2 = pd.merge(question2, results_q2_final, left_on = 'question_id', right_on = 'question_id', how = 'left' )<feature_engineering>
all_df_real_uflag, f_s = get_all_freq(all_df_real, var_cols)
 all_df_synth_uflag, f_s = get_all_freq(all_df_synth, var_cols, f_s)
 all_df_real = pd.concat([all_df_real, all_df_real_uflag],
axis=1, copy=False, sort=False)
 all_df_synth = pd.concat([all_df_synth, all_df_synth_uflag],
axis=1, copy=False, sort=False)
 train_df = all_df_real[all_df_real.index.str.contains('train')| all_df_real.index.str.contains('pred')].copy()
train_df = train_df.loc[train_df_src['ID_code']]
question2.quest_pct = round(question2.quest_pct,5 )<drop_column>
del all_df_real_uflag, all_df_synth_uflag, train_df_src
gc.collect()
train.drop(['timestamp', 'content_type_id', 'question_id', 'part'], axis=1, inplace=True )<filter>
test_df_real = all_df_real[all_df_real.index.str.contains('test')]
test_df_synth = all_df_synth[all_df_synth.index.str.contains('test')]
test_df = pd.concat([test_df_real, test_df_synth], axis=0, copy=True, sort=False)
 test_df = test_df.loc[test_df_src['ID_code']]
validation = train.groupby('user_id' ).tail(5)
 train = train[~train.index.isin(validation.index)]
len(train)+ len(validation )<groupby>
del all_df_real, all_df_synth, test_df_src
gc.collect()
results_u_val = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_val.columns = ['answered_correctly_user']
results_u2_val = train[['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_val.columns = ['explanation_mean_user']<groupby>
features = [c for c in train_df.columns if c not in ['ID_code', 'target']]
target = train_df['target']
X = train.groupby('user_id' ).tail(18)
 train = train[~train.index.isin(X.index)]
len(X)+ len(train)+ len(validation )<groupby>
def add_freq(df):
df_res = pd.DataFrame(index=df.index)
 for c in tqdm(var_cols):
df_res[f'{c}_freq_mean'] = df.groupby(f'{c}_freq_1')[c].transform(np.mean ).astype('float16')
 df_res[f'{c}_freq_std'] = df.groupby(f'{c}_freq_1')[c].transform(np.std ).astype('float16')
 df_res[f'{c}_freq_var1'] =(df_res[f'{c}_freq_mean'] - 2*df_res[f'{c}_freq_std'] ).astype('float16')
 df_res[f'{c}_freq_var2'] =(df_res[f'{c}_freq_mean'] + 2*df_res[f'{c}_freq_std'] ).astype('float16')
 df_res[f'{c}_freq_mean_unq'] = df[c]
idx = df[df[f'{c}_freq_1']==False].index
df_res.loc[idx, [f'{c}_freq_mean_unq']] = df_res.loc[idx][f'{c}_freq_mean_unq'].mean()
return df_res
def add_features(df):
df_count = add_freq(df)
 colums = [f'{x}_freq_N' for x in var_cols] + \
[f'{x}_freq_1' for x in var_cols] + \
[f'{x}_mul_freq' for x in var_cols] + \
[f'{x}_div_freq' for x in var_cols] + \
['target']
res = pd.concat([df[colums], df_count], axis=1, copy=False, sort=False)
 return res
results_u_X = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_X.columns = ['answered_correctly_user']
results_u2_X = train[['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_X.columns = ['explanation_mean_user']<drop_column>
param = {
'random_seed': SEED,
'gpu_ram_part': 0.95,
'iterations': 200000,
'learning_rate': 0.04,
'l2_leaf_reg': 5,
'depth': 1,
'thread_count': 4,
'custom_metric': ['Logloss', 'AUC:hints=skip_train~false'],
'od_type': 'Iter',
'od_wait': 500,
'task_type': 'GPU',
'eval_metric': 'AUC',
'use_best_model': True
}
del(train )<merge>
%%time
num_folds = 5
features = [c for c in train_df.columns if c not in ['ID_code', 'target']]
print('Training the Model:')
 val_list = []
predictions = []
for i in range(4):
param['depth'] = i%2 + 1
clf = ctb.CatBoostClassifier(**param)
 folds = StratifiedKFold(n_splits=num_folds, shuffle=True, random_state=SEED*i)
 oof = np.zeros(len(train_df))
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_df.values, target.values)) :
gc.collect()
print("Fold idx:{}".format(fold_ + 1))
trn_data = add_features(train_df.iloc[trn_idx])
 y_trn = trn_data['target']
features_ext = [c for c in trn_data.columns if c not in ['ID_code', 'target']]
trn_data = trn_data[features_ext]
val_data = add_features(train_df.iloc[val_idx])
 y_val = val_data['target']
val_data = val_data[features_ext]
clf.fit(X=trn_data, y=y_trn, eval_set=[(val_data, y_val)], verbose=1000, early_stopping_rounds = 1001)
 del trn_data, val_data
gc.collect()
train_data = add_features(train_df.iloc[val_idx])[features_ext]
oof[val_idx] = clf.predict_proba(train_data)[:,1]
del train_data
gc.collect()
test_data = add_features(test_df)[features_ext]
predictions.append(clf.predict_proba(test_data)[:,1])
 del test_data
del clf
val_list.append(oof)
 oof = gmean(val_list, 0)
 predictions_gmean = gmean(predictions, 0 )
X = pd.merge(X, group3, left_on=['task_container_id'], right_index= True, how="left")
 X = pd.merge(X, results_u_X, on=['user_id'], how="left")
 X = pd.merge(X, results_u2_X, on=['user_id'], how="left")
 X = pd.merge(X, user_lecture_stats_part, on=['user_id'], how="left" )<merge>
sub = pd.DataFrame({"ID_code": test_df.index.values})
 sub["target"] = predictions_gmean
sub.to_csv('submission_cb_{}_seed_v{}.csv'.format(SEED, VERSION), index=False )
validation = pd.merge(validation, group3, left_on=['task_container_id'], right_index= True, how="left")
 validation = pd.merge(validation, results_u_val, on=['user_id'], how="left")
 validation = pd.merge(validation, results_u2_val, on=['user_id'], how="left")
 validation = pd.merge(validation, user_lecture_stats_part, on=['user_id'], how="left" )<categorify>
roc = roc_auc_score(target, oof)
 print("CV score: {:<8.5f}".format(roc))
<load_from_csv><EOS> 
oof_train = pd.DataFrame()
oof_train[f'oof_cb_{num_folds}_{SEED}_{VERSION}'] = oof
oof_test = pd.DataFrame()
oof_test[f'oof_cb_{num_folds}_{SEED}_{VERSION}'] = predictions_gmean
oof_train.to_csv('train_cb_roc_{}_seed_{}_v{}.csv'.format(
 roc, SEED, VERSION), index=False)
 oof_test.to_csv('test_cb_roc_{}_seed_{}_v{}.csv'.format(
 roc, SEED, VERSION), index=False )