Kaggle competition: Santander Customer Transaction Prediction
env = riiideducation.make_env()
iter_test = env.iter_test()<merge>
def augment(x,y,t=2):
xs,xn = [],[]
for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(200):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
x1[:,c+200] = x1[ids][:,c+200]
x1[:,c+400] = x1[ids][:,c+400]
xs.append(x1)
 for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(200):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
x1[:,c+200] = x1[ids][:,c+200]
x1[:,c+400] = x1[ids][:,c+400]
xn.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x,xs,xn])
 y = np.concatenate([y,ys,yn])
 return x,y
for(test_df, sample_prediction_df)in iter_test:
test_df['task_container_id'] = test_df.task_container_id.mask(test_df.task_container_id > 9999, 9999)
 test_df = pd.merge(test_df, group3, left_on=['task_container_id'], right_index= True, how="left")
 test_df = pd.merge(test_df, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df = pd.merge(test_df, results_u_final, on=['user_id'], how="left")
 test_df = pd.merge(test_df, results_u2_final, on=['user_id'], how="left")
 test_df = pd.merge(test_df, user_lecture_stats_part, on=['user_id'], how="left")
 test_df['part_1'].fillna(0, inplace = True)
 test_df['part_2'].fillna(0, inplace = True)
 test_df['part_3'].fillna(0, inplace = True)
 test_df['part_4'].fillna(0, inplace = True)
 test_df['part_5'].fillna(0, inplace = True)
 test_df['part_6'].fillna(0, inplace = True)
 test_df['part_7'].fillna(0, inplace = True)
 test_df['type_of_concept'].fillna(0, inplace = True)
 test_df['type_of_intention'].fillna(0, inplace = True)
 test_df['type_of_solving_question'].fillna(0, inplace = True)
 test_df['type_of_starter'].fillna(0, inplace = True)
 test_df['part_1_boolean'].fillna(0, inplace = True)
 test_df['part_2_boolean'].fillna(0, inplace = True)
 test_df['part_3_boolean'].fillna(0, inplace = True)
 test_df['part_4_boolean'].fillna(0, inplace = True)
 test_df['part_5_boolean'].fillna(0, inplace = True)
 test_df['part_6_boolean'].fillna(0, inplace = True)
 test_df['part_7_boolean'].fillna(0, inplace = True)
 test_df['type_of_concept_boolean'].fillna(0, inplace = True)
 test_df['type_of_intention_boolean'].fillna(0, inplace = True)
 test_df['type_of_solving_question_boolean'].fillna(0, inplace = True)
 test_df['type_of_starter_boolean'].fillna(0, inplace = True)
 test_df['answered_correctly_user'].fillna(0.65, inplace=True)
 test_df['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 test_df['quest_pct'].fillna(content_mean, inplace=True)
 test_df['part'] = test_df.part - 1
test_df['part'].fillna(4, inplace = True)
 test_df['avg_questions_seen'].fillna(1, inplace = True)
 test_df['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df["prior_question_had_explanation_enc"] = lb_make.fit_transform(test_df["prior_question_had_explanation"])
 X = test_df[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'part_1_boolean', 'part_2_boolean', 'part_3_boolean', 'part_4_boolean', 'part_5_boolean', 'part_6_boolean', 'part_7_boolean',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean']]
X=scaler.transform(X)
 X = X.reshape(X.shape[0], X.shape[1], 1)
 X = keras.preprocessing.sequence.pad_sequences(X, maxlen=maxlen)
 test_df['answered_correctly'] = model.predict(X)
 env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<set_options>
train_data=pd.read_csv(path+"train.csv")
 test_data=pd.read_csv(path+"test.csv" )
env = riiideducation.make_env()<load_from_csv>
features = [x for x in train_data.columns if x not in ['ID_code', "target"]]
label = "target"
train = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',
usecols=[1, 2, 3, 4, 5, 7, 8, 9],
dtype={'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id': 'int8',
'task_container_id': 'int16',
'answered_correctly':'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'boolean'}
)<load_from_csv>
df_test = test_data.values
unique_samples = []
unique_count = np.zeros_like(df_test)
 for feature in tqdm_notebook(range(df_test.shape[1])) :
if feature in [0]:
print('ok')
 continue
_, index_, count_ = np.unique(df_test[:, feature], return_counts=True, return_index=True)
 unique_count[index_[count_ == 1], feature] += 1
real_samples_indexes = np.argwhere(np.sum(unique_count, axis=1)> 0)[:, 0]
synthetic_samples_indexes = np.argwhere(np.sum(unique_count, axis=1)== 0)[:, 0]
questions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 3],
dtype={'question_id': 'int16',
'part': 'int8'}
)<load_from_csv>
idx_ = list(real_samples_indexes)
 data=pd.concat([train_data,test_data.loc[idx_]])
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )<categorify>
for col in tqdm_notebook(features):
count=data[col].value_counts()
rank=len(data[col].unique())
 train_data["rank_"+col]=train_data[col].map(count.rank() /rank)
 test_data["rank_"+col]=test_data[col].map(count.rank() /rank )
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])
 part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]<merge>
for col in tqdm_notebook(features):
unique_value = train_data['rank_'+col].unique().tolist() +test_data['rank_'+col].unique().tolist()
uq_v = np.min(sorted(list(set(unique_value))))
train_data['cut_'+col] = train_data[col]
test_data['cut_'+col] = test_data[col]
mean1=train_data['cut_'+col].mean()
mean2=test_data['cut_'+col].mean()
train_data['cut_'+col][(train_data['rank_'+col]==uq_v)]=mean1
test_data['cut_'+col][(test_data['rank_'+col]==uq_v)]=mean2
train_lectures = train[train.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left' )<groupby>
train=train_data
test=test_data
del train_data,test_data,data
gc.collect()
user_lecture_stats_part = train_lectures.groupby('user_id')[part_lectures_columns + types_of_lectures_columns].sum()<data_type_conversions>
features = [x for x in train.columns if x not in ['ID_code', "target"]]
label = "target"
for column in user_lecture_stats_part.columns:
bool_column = column + '_boolean'
user_lecture_stats_part[bool_column] =(user_lecture_stats_part[column] > 0 ).astype(int )<drop_column>
"feature count:",len(features )
del(train_lectures )<sort_values>
skf = StratifiedKFold(n_splits=10, shuffle=True, random_state=99999)
 oof = train[['ID_code']]
oof['target'] = 0
predictions = test[['ID_code']]
val_aucs = []
train = train[train.content_type_id == False].sort_values('timestamp' ).reset_index(drop = True )<count_unique_values>
for fold,(trn_idx, val_idx)in enumerate(skf.split(train, train['target'])) :
X_train, y_train = train.iloc[trn_idx][features], train.iloc[trn_idx]['target']
X_valid, y_valid = train.iloc[val_idx][features], train.iloc[val_idx]['target']
N = 3
p_valid,yp = 0,0
for i in range(N):
X_t, y_t = augment(X_train.values, y_train.values,2)
 X_t = pd.DataFrame(X_t)
 X_t.columns = features
trn_data = Pool(X_t, label=y_t)
 val_data = Pool(X_valid, label=y_valid)
 evals_result = {}
model = CatBoostClassifier(iterations=10000000,
learning_rate = 0.03,
od_type='Iter',
od_wait=3500,
loss_function="Logloss",
eval_metric='AUC',
bagging_temperature=0.7,
random_seed = 2019,
task_type='GPU' 
)
 model.fit(trn_data,eval_set=val_data,
use_best_model=True,
verbose=3000 
)
 p_valid += model.predict_proba(X_valid)[:,1]
yp += model.predict_proba(test[features])[:,1]
gc.collect()
gc.collect()
oof['target'][val_idx] = p_valid/N
val_score = roc_auc_score(y_valid, p_valid)
 print("fold {}|5".format(fold+1),"auc score : ",val_score)
 val_aucs.append(val_score)
 predictions['fold{}'.format(fold+1)] = yp/N
train[(train.content_type_id == False)].task_container_id.nunique()<groupby>
print("oof auc")
 roc_auc_score(train['target'], oof['target'] )
group1 = train.loc[(train.content_type_id == False), ['task_container_id', 'user_id']].groupby(['task_container_id'] ).agg(['count'])
 group1.columns = ['avg_questions']
group2 = train.loc[(train.content_type_id == False), ['task_container_id', 'user_id']].groupby(['task_container_id'] ).agg(['nunique'])
 group2.columns = ['avg_questions']
group3 = group1 / group2<groupby>
"auc mean:",np.mean(val_aucs )
group3['avg_questions_seen'] = group3.avg_questions.cumsum()<set_options>
predictions['target'] = np.mean(predictions[[col for col in predictions.columns if col not in ['ID_code', 'target']]].values, axis=1)
 predictions.to_csv('lgb_all_predictions.csv', index=None)
 sub_df = pd.DataFrame({"ID_code":test["ID_code"].values})
 sub_df["target"] = predictions['target']
sub_df.to_csv("lgb_submission.csv", index=False)
 oof_all=pd.concat([oof,sub_df])
 oof_all.to_csv('augment_cat_oof.csv', index=False )