Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<feature_engineering>
warnings.simplefilter(action='ignore', category=FutureWarning)
 warnings.filterwarnings('ignore')
 plt.style.use('seaborn')
 sns.set(font_scale=1)
questions_df['num_tags'] = questions_df['tags'].apply(lambda x:len(x.split())if pd.notna(x)else 0)
<drop_column>
train = pd.read_csv('.. /input/santander-customer-transaction-prediction/train.csv')
 test = pd.read_csv('.. /input/santander-customer-transaction-prediction/test.csv')
 fake_ids = np.load('.. /input/fakeids/synthetic_samples_indexes.npy')
 ids = np.arange(test.shape[0])
 real_ids = list(set(ids)- set(fake_ids))
real_test = test.iloc[real_ids]
fake_test = test.iloc[fake_ids]
real_test_id = real_test.ID_code
questions_df = questions_df[['question_id','part','num_tags']]<load_from_csv>
features = [c for c in train.columns if c not in ['target', 'ID_code']]
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )<categorify>
df = pd.concat([train,real_test], axis = 0)
 for feat in tqdm_notebook(features):
df[feat+'_var'] = df.groupby([feat])[feat].transform('var')
 for feat in tqdm_notebook(features):
df[feat+'plus_'] = df[feat] + df[feat+'_var']
drop_features = [c for c in df.columns if '_var' in c]
df.drop(drop_features, axis=1, inplace=True )
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'] )<define_variables>
train = df.iloc[:train.shape[0]]
real_test = df.iloc[train.shape[0]:]
part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]<merge>
features = [c for c in train.columns if c not in ['ID_code', 'target']]
target = train['target']
train_lectures = train[train.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left' )<groupby>
param = {
'bagging_freq': 5, 'bagging_fraction': 0.4, 'boost_from_average':'false',
'boost': 'gbdt', 'feature_fraction': 0.04, 'learning_rate': 0.01,
'max_depth': -1, 'metric':'auc', 'min_data_in_leaf': 80,
'num_leaves': 13, 'num_threads': 8,
'tree_learner': 'serial', 'objective': 'binary', 'verbosity': 1
}
user_lecture_stats_part = train_lectures.groupby('user_id')[part_lectures_columns + types_of_lectures_columns].sum()<data_type_conversions>
folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=4242)
 oof = np.zeros(len(train))
predictions = np.zeros(len(real_test))
val_aucs = []
feature_importance_df = pd.DataFrame()
for column in user_lecture_stats_part.columns:
bool_column = column + '_boolean'
user_lecture_stats_part[bool_column] =(user_lecture_stats_part[column] > 0 ).astype(int )<drop_column>
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train.values, target.values)) :
print("Fold {}".format(fold_))
trn_data = lgb.Dataset(train.iloc[trn_idx][features], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(train.iloc[val_idx][features], label=target.iloc[val_idx])
 clf = lgb.train(param, trn_data, 100000, valid_sets = [trn_data, val_data], verbose_eval=5000, early_stopping_rounds = 3000)
 oof[val_idx] = clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 val_aucs.append(roc_auc_score(target[val_idx] , oof[val_idx]))
predictions += clf.predict(real_test[features], num_iteration=clf.best_iteration)/ folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
del(train_lectures )<sort_values>
mean_auc = np.mean(val_aucs)
 std_auc = np.std(val_aucs)
 all_auc = roc_auc_score(target, oof)
 print("Mean auc: %.9f, std: %.9f.All auc: %.9f." %(mean_auc, std_auc, all_auc))
train = train[train.content_type_id == False].sort_values('timestamp' ).reset_index(drop = True )<count_unique_values>
subreal = pd.DataFrame({"ID_code": real_test_id.values})
 subreal['target']=predictions
sub = pd.DataFrame({"ID_code": test.ID_code.values} )
train[(train.content_type_id == False)].task_container_id.nunique()<groupby>
finalsub = sub.set_index('ID_code' ).join(subreal.set_index('ID_code')).reset_index()
finalsub.fillna(0,inplace=True)
 finalsub.to_csv("submission.csv", index=False )
<groupby><EOS> 
def create_download_link(title = "Download CSV file", filename = "data.csv"):
html = '<a href={filename}>{title}</a>'
html = html.format(title=title,filename=filename)
 return HTML(html)
 create_download_link(filename='submission.csv' )