Kaggle competition: Santander Customer Transaction Prediction
validation = train.groupby('user_id' ).tail(5)
 train = train[~train.index.isin(validation.index)]
len(train)+ len(validation )<groupby>
warnings.filterwarnings('ignore' )
results_u_val = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_val.columns = ['answered_correctly_user']
results_u2_val = train[['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_val.columns = ['explanation_mean_user']<groupby>
df_train = pd.read_csv("/kaggle/input/santander-customer-transaction-prediction-dataset/train.csv")
 df_test = pd.read_csv("/kaggle/input/santander-customer-transaction-prediction-dataset/test.csv" )
X = train.groupby('user_id' ).tail(18)
 train = train[~train.index.isin(X.index)]
len(X)+ len(train)+ len(validation )<groupby>
sample_submission = pd.read_csv("/kaggle/input/santander-customer-transaction-prediction/sample_submission.csv" )
results_u_X = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_X.columns = ['answered_correctly_user']
results_u2_X = train[['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_X.columns = ['explanation_mean_user']<drop_column>
df_train.isnull().sum().sum()
del(train )<merge>
df_test.isnull().sum().sum()
X = pd.merge(X, group3, left_on=['task_container_id'], right_index= True, how="left")
 X = pd.merge(X, results_u_X, on=['user_id'], how="left")
 X = pd.merge(X, results_u2_X, on=['user_id'], how="left")
 X = pd.merge(X, user_lecture_stats_part, on=['user_id'], how="left" )<merge>
df_train.target.value_counts(normalize=True )
validation = pd.merge(validation, group3, left_on=['task_container_id'], right_index= True, how="left")
 validation = pd.merge(validation, results_u_val, on=['user_id'], how="left")
 validation = pd.merge(validation, results_u2_val, on=['user_id'], how="left")
 validation = pd.merge(validation, user_lecture_stats_part, on=['user_id'], how="left" )<categorify>
df_train.loc[df_train.target == 0][feat].mean(axis=1 )
lb_make = LabelEncoder()
X.prior_question_had_explanation.fillna(False, inplace = True)
 validation.prior_question_had_explanation.fillna(False, inplace = True)
 validation["prior_question_had_explanation_enc"] = lb_make.fit_transform(validation["prior_question_had_explanation"])
 X["prior_question_had_explanation_enc"] = lb_make.fit_transform(X["prior_question_had_explanation"] )<load_from_csv>
df_train.loc[df_train.target == 1][feat].mean(axis=1 )
<categorify>
df_train.loc[df_train.target == 0][feat].mean()
question2.quest_pct = question2.quest_pct.mask(( question2['count'] < 3),.65)
 question2.quest_pct = question2.quest_pct.mask(( question2.quest_pct <.2)&(question2['count'] < 21),.2)
 question2.quest_pct = question2.quest_pct.mask(( question2.quest_pct >.95)&(question2['count'] < 21),.95 )<merge>
df_train.loc[df_train.target == 1][feat].mean()
X = pd.merge(X, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 validation = pd.merge(validation, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 X.part = X.part - 1
validation.part = validation.part - 1<prepare_x_and_y>
import lightgbm as lgb
from sklearn.metrics import f1_score, classification_report
from sklearn.metrics import confusion_matrix
from sklearn.metrics import roc_auc_score, roc_curve
from sklearn.model_selection import StratifiedKFold
y = X['answered_correctly']
X = X.drop(['answered_correctly'], axis=1)
 X.head()
y_val = validation['answered_correctly']
X_valdf = validation.drop(['answered_correctly'], axis=1 )<drop_column>
X= df_train.drop(["target","ID_code"],axis=1)
 X.head()
X_train = X[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean', 'num_tags',]]
X_val = X_valdf[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean', 'num_tags',]] 
<categorify>
Y=df_train.target
Y.head()
X_train['answered_correctly_user'].fillna(0.65, inplace=True)
 X_train['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 X_train['quest_pct'].fillna(content_mean, inplace=True)
 X_train['num_tags'].fillna(0, inplace=True)
 X_train['part'].fillna(4, inplace = True)
 X_train['avg_questions_seen'].fillna(1, inplace = True)
 X_train['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 X_train['prior_question_had_explanation_enc'].fillna(0, inplace = True)
 X_train['part_1'].fillna(0, inplace = True)
 X_train['part_2'].fillna(0, inplace = True)
 X_train['part_3'].fillna(0, inplace = True)
 X_train['part_4'].fillna(0, inplace = True)
 X_train['part_5'].fillna(0, inplace = True)
 X_train['part_6'].fillna(0, inplace = True)
 X_train['part_7'].fillna(0, inplace = True)
 X_train['type_of_concept'].fillna(0, inplace = True)
 X_train['type_of_intention'].fillna(0, inplace = True)
 X_train['type_of_solving_question'].fillna(0, inplace = True)
 X_train['type_of_starter'].fillna(0, inplace = True)
 X_train['type_of_concept_boolean'].fillna(0, inplace = True)
 X_train['type_of_intention_boolean'].fillna(0, inplace = True)
 X_train['type_of_solving_question_boolean'].fillna(0, inplace = True)
 X_train['type_of_starter_boolean'].fillna(0, inplace = True )<data_type_conversions>
X_test = df_test.drop("ID_code",axis=1)
 X_test.head()
X_val['answered_correctly_user'].fillna(0.65, inplace=True)
 X_val['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 X_val['quest_pct'].fillna(content_mean, inplace=True)
 X_val['part'].fillna(4, inplace = True)
 X_val['avg_questions_seen'].fillna(1, inplace = True)
 X_val['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 X_val['prior_question_had_explanation_enc'].fillna(0, inplace = True)
 X_val['num_tags'].fillna(0, inplace=True)
 X_val['part_1'].fillna(0, inplace = True)
 X_val['part_2'].fillna(0, inplace = True)
 X_val['part_3'].fillna(0, inplace = True)
 X_val['part_4'].fillna(0, inplace = True)
 X_val['part_5'].fillna(0, inplace = True)
 X_val['part_6'].fillna(0, inplace = True)
 X_val['part_7'].fillna(0, inplace = True)
 X_val['type_of_concept'].fillna(0, inplace = True)
 X_val['type_of_intention'].fillna(0, inplace = True)
 X_val['type_of_solving_question'].fillna(0, inplace = True)
 X_val['type_of_starter'].fillna(0, inplace = True)
 X_val['type_of_concept_boolean'].fillna(0, inplace = True)
 X_val['type_of_intention_boolean'].fillna(0, inplace = True)
 X_val['type_of_solving_question_boolean'].fillna(0, inplace = True)
 X_val['type_of_starter_boolean'].fillna(0, inplace = True )<normalization>
params = {'objective' : "binary",
'boost':"gbdt",
'metric':"auc",
'boost_from_average':"false",
'num_threads':8,
'learning_rate' : 0.01,
'num_leaves' : 13,
'max_depth':-1,
'tree_learner' : "serial",
'feature_fraction' : 0.05,
'bagging_freq' : 5,
'bagging_fraction' : 0.4,
'min_data_in_leaf' : 80,
'min_sum_hessian_in_leaf' : 10.0,
'verbosity' : 1}
X_train_values = X_train.values
min_max_scaler = preprocessing.MinMaxScaler()
x_train_scaled = min_max_scaler.fit_transform(X_train_values)
 x_train_scaled<normalization>
num_folds = 10
folds = StratifiedKFold(n_splits=num_folds, shuffle=False, random_state=42)
 oof = np.zeros(len(df_train))
y_pred_lgbm = np.zeros(len(df_train.target))
print('Light GBM Model')
 for fold_,(trn_idx, val_idx)in enumerate(folds.split(df_train.values,df_train.target.values)) :
print('Fold', fold_)
 X_train, Y_train = df_train.iloc[trn_idx][feat], df_train.target.iloc[trn_idx]
X_valid, Y_valid = df_train.iloc[val_idx][feat], df_train.target.iloc[val_idx]
train_data = lgb.Dataset(X_train, label=Y_train)
 valid_data = lgb.Dataset(X_valid, label=Y_valid)
 lgb_model = lgb.train(params, train_data, 1000000, valid_sets = [train_data, valid_data], verbose_eval=5000, early_stopping_rounds = 4000)
 oof[val_idx] = lgb_model.predict(df_train.iloc[val_idx][feat], num_iteration=lgb_model.best_iteration)
 y_pred_lgbm += lgb_model.predict(df_test[feat], num_iteration=lgb_model.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(df_train.target, oof)) )
X_val_values = X_val.values
min_max_scaler = preprocessing.MinMaxScaler()
x_val_scaled = min_max_scaler.fit_transform(X_val_values)
 x_val_scaled<split>
from sklearn.model_selection import train_test_split
y_train = y.values
y_val = y_val.values<import_modules>
from sklearn.model_selection import train_test_split
from tensorflow.keras import models, Sequential
from tensorflow.keras.layers import Embedding<define_variables>
train_X, val_X, train_y, val_y = train_test_split(X, Y, random_state=42)
 rfc_mod = RandomForestClassifier(random_state=42 ).fit(train_X, train_y )
dim = x_train_scaled.shape[1] 
<choose_model_class>
y_pred_rfc = rfc_mod.predict(X_test )
dim = x_train_scaled.shape[1]
inputs = tf.keras.Input(shape=(dim,))
x = tf.keras.layers.Dense(16, activation='relu' )(inputs)
 x = tf.keras.layers.Dropout(0.3 )(x)
 x = tf.keras.layers.Dense(16, activation='relu' )(x)
 x = tf.keras.layers.Dropout(0.3 )(x)
 out = tf.keras.layers.Dense(1, activation="sigmoid" )(x)
 model_DNN = tf.keras.Model(inputs=inputs, outputs = out)
 model_DNN.compile(optimizer=tf.keras.optimizers.Adam(lr=0.0002), loss="binary_crossentropy", metrics=[tf.keras.metrics.Precision() ])
 model_DNN.summary()<train_model>
print("F1 Score :",f1_score(y_pred_rfc,Y,average = "weighted"))
print('Report:
',classification_report(Y, y_pred_rfc))
print('Confusion Matrix:
',confusion_matrix(Y, y_pred_rfc))
epoch_num = 20
history = model.fit(x_train_scaled, np.array(y_train), validation_data=(x_val_scaled, np.array(y_val)) , batch_size=64, epochs=epoch_num)
<compute_train_metric>
train_X, val_X, train_y, val_y = train_test_split(X, Y, random_state=42)
 dectre_mod = DecisionTreeClassifier(random_state=42, max_depth=5, min_samples_split=5 ).fit(train_X, train_y )
y_pred = model_DNN.predict(x_val_scaled)
 y_true = np.array(y_val)
 roc_auc_score(y_true, y_pred )<split>
y_pred_dec_tree = dectre_mod.predict(X_test )
iter_test = env.iter_test()<merge>
print("F1 Score :",f1_score(y_pred_dec_tree,Y,average = "weighted"))
print('Report:
',classification_report(Y, y_pred_dec_tree))
print('Confusion Matrix:
',confusion_matrix(Y, y_pred_dec_tree))
for(test_df, sample_prediction_df)in iter_test:
test_df['task_container_id'] = test_df.task_container_id.mask(test_df.task_container_id > 9999, 9999)
 test_df = pd.merge(test_df, group3, left_on=['task_container_id'], right_index= True, how="left")
 test_df = pd.merge(test_df, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df = pd.merge(test_df, results_u_final, on=['user_id'], how="left")
 test_df = pd.merge(test_df, results_u2_final, on=['user_id'], how="left")
 test_df = pd.merge(test_df, user_lecture_stats_part, on=['user_id'], how="left")
 test_df['part_1'].fillna(0, inplace = True)
 test_df['part_2'].fillna(0, inplace = True)
 test_df['part_3'].fillna(0, inplace = True)
 test_df['part_4'].fillna(0, inplace = True)
 test_df['part_5'].fillna(0, inplace = True)
 test_df['part_6'].fillna(0, inplace = True)
 test_df['part_7'].fillna(0, inplace = True)
 test_df['type_of_concept'].fillna(0, inplace = True)
 test_df['type_of_intention'].fillna(0, inplace = True)
 test_df['type_of_solving_question'].fillna(0, inplace = True)
 test_df['type_of_starter'].fillna(0, inplace = True)
 test_df['type_of_concept_boolean'].fillna(0, inplace = True)
 test_df['type_of_intention_boolean'].fillna(0, inplace = True)
 test_df['type_of_solving_question_boolean'].fillna(0, inplace = True)
 test_df['type_of_starter_boolean'].fillna(0, inplace = True)
 test_df['num_tags'].fillna(0, inplace = True)
 test_df['answered_correctly_user'].fillna(0.65, inplace=True)
 test_df['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 test_df['quest_pct'].fillna(content_mean, inplace=True)
 test_df['part'] = test_df.part - 1
test_df['part'].fillna(4, inplace = True)
 test_df['avg_questions_seen'].fillna(1, inplace = True)
 test_df['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df["prior_question_had_explanation_enc"] = lb_make.fit_transform(test_df["prior_question_had_explanation"])
 test_df_train = test_df[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean','num_tags',]]
test_df_values = test_df_train.values
min_max_scaler = preprocessing.MinMaxScaler()
test_df_values = min_max_scaler.fit_transform(test_df_values)
 test_df['answered_correctly'] = model_DNN.predict(test_df_values)
 env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<import_modules>
train_X, val_X, train_y, val_y = train_test_split(X,Y,random_state=42)
 logreg_mod = LogisticRegression(random_state=42 ).fit(train_X, train_y )
import pandas as pd
import numpy as np
import gc
from sklearn.metrics import roc_auc_score
from collections import defaultdict
from tqdm.notebook import tqdm
import lightgbm as lgb<define_variables>
y_pred_logreg = logreg_mod.predict(X_test )
train_pickle = '.. /input/riiid-cross-validation-files/cv1_train.pickle'
valid_pickle = '.. /input/riiid-cross-validation-files/cv1_valid.pickle'
question_file = '.. /input/riiid-test-answer-prediction/questions.csv'
debug = False
validaten_flg = False<data_type_conversions>
print("F1 Score :",f1_score(y_pred_logreg,Y,average = "weighted"))
print('Report:
',classification_report(Y, y_pred_logreg))
print('Confusion Matrix:
',confusion_matrix(Y, y_pred_logreg))
def add_user_feats(df, answered_correctly_sum_u_dict, count_u_dict):
acsu = np.zeros(len(df), dtype=np.int32)
 cu = np.zeros(len(df), dtype=np.int32)
 for cnt,row in enumerate(tqdm(df[['user_id','answered_correctly']].values)) :
acsu[cnt] = answered_correctly_sum_u_dict[row[0]]
cu[cnt] = count_u_dict[row[0]]
answered_correctly_sum_u_dict[row[0]] += row[1]
count_u_dict[row[0]] += 1
user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu})
 user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']
df = pd.concat([df, user_feats_df], axis=1)
 return df
def add_user_feats_without_update(df, answered_correctly_sum_u_dict, count_u_dict):
acsu = np.zeros(len(df), dtype=np.int32)
 cu = np.zeros(len(df), dtype=np.int32)
 for cnt,row in enumerate(df[['user_id']].values):
acsu[cnt] = answered_correctly_sum_u_dict[row[0]]
cu[cnt] = count_u_dict[row[0]]
user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu})
 user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']
df = pd.concat([df, user_feats_df], axis=1)
 return df
def update_user_feats(df, answered_correctly_sum_u_dict, count_u_dict):
for row in df[['user_id','answered_correctly','content_type_id']].values:
if row[2] == 0:
answered_correctly_sum_u_dict[row[0]] += row[1]
count_u_dict[row[0]] += 1<categorify>
submission_lgbm = pd.DataFrame({
"ID_code": df_test["ID_code"],
"target": y_pred_lgbm
})
 submission_lgbm.to_csv('submission_lgbm.csv', index=False )
feld_needed = ['row_id', 'user_id', 'content_id', 'content_type_id', 'answered_correctly', 'prior_question_elapsed_time', 'prior_question_had_explanation']
train = pd.read_pickle(train_pickle)[feld_needed]
valid = pd.read_pickle(valid_pickle)[feld_needed]
if debug:
train = train[:1000000]
valid = valid[:10000]
train = train.loc[train.content_type_id == False].reset_index(drop=True)
 valid = valid.loc[valid.content_type_id == False].reset_index(drop=True)
 content_df = train[['content_id','answered_correctly']].groupby(['content_id'] ).agg(['mean'] ).reset_index()
content_df.columns = ['content_id', 'answered_correctly_avg_c']
train = pd.merge(train, content_df, on=['content_id'], how="left")
 valid = pd.merge(valid, content_df, on=['content_id'], how="left")
 answered_correctly_sum_u_dict = defaultdict(int)
 count_u_dict = defaultdict(int)
 train = add_user_feats(train, answered_correctly_sum_u_dict, count_u_dict)
 valid = add_user_feats(valid, answered_correctly_sum_u_dict, count_u_dict)
 prior_question_elapsed_time_mean = train.prior_question_elapsed_time.dropna().values.mean()
train['prior_question_elapsed_time_mean'] = train.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 valid['prior_question_elapsed_time_mean'] = valid.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 questions_df = pd.read_csv(question_file)
 train = pd.merge(train, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')
 valid = pd.merge(valid, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')
 train['prior_question_had_explanation'] = train.prior_question_had_explanation.fillna(False ).astype('int8')
 valid['prior_question_had_explanation'] = valid.prior_question_had_explanation.fillna(False ).astype('int8' )<prepare_x_and_y>
submission_rfc = pd.DataFrame({
"ID_code": df_test["ID_code"],
"target": y_pred_rfc
})
 submission_rfc.to_csv('submission_rfc.csv', index=False )
TARGET = 'answered_correctly'
FEATS = ['answered_correctly_avg_u', 'answered_correctly_sum_u', 'count_u', 'answered_correctly_avg_c', 'part', 'prior_question_had_explanation', 'prior_question_elapsed_time']
dro_cols = list(set(train.columns)- set(FEATS))
y_tr = train[TARGET]
y_va = valid[TARGET]
train.drop(dro_cols, axis=1, inplace=True)
 valid.drop(dro_cols, axis=1, inplace=True)
 _=gc.collect()<create_dataframe>
submission_dec_tree = pd.DataFrame({
"ID_code": df_test["ID_code"],
"target": y_pred_dec_tree
})
 submission_dec_tree.to_csv('submission_dec_tree.csv', index=False )
<feature_engineering><EOS> 
submission_logreg = pd.DataFrame({
"ID_code": df_test["ID_code"],
"target": y_pred_logreg
})
 submission_logreg.to_csv('submission_logreg.csv', index=False )