Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<choose_model_class>
env = riiideducation.make_env()
input_shape =(28,28,1)
 model = Sequential()
model.add(Conv2D(filters = 96, kernel_size =(3, 3), padding='same', activation='relu', input_shape=input_shape))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 96,kernel_size =(3, 3), padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(filters = 192, kernel_size =(3,3), activation='relu', padding='same'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 256, kernel_size =(3,3), activation='relu', padding='same'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Conv2D(filters = 256, kernel_size =(3,3), activation='relu', padding='same'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 256, kernel_size =(3,3), activation='relu', padding='same'))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.25))
model.add(Dense(units=10, activation='softmax'))
model.compile(optimizer=OPTIMIZER, loss="categorical_crossentropy", metrics=["accuracy"] )<choose_model_class>
train = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',
usecols=[1, 2, 3, 4, 5, 7, 8, 9],
dtype={'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id': 'int8',
'task_container_id': 'int16',
'answered_correctly':'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'boolean'} 
)
 train.info()
learning_rate = ReduceLROnPlateau(monitor='val_acc', patience=2, verbose=1, factor=0.5, min_lr=1.0e-5 )<choose_model_class>
questions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 3],
dtype={'question_id': 'int16',
'part': 'int8'} 
)
 questions_df
EarlyStopping_cb = keras.callbacks.EarlyStopping(monitor='val_acc', patience=5, verbose=1, mode='auto' )<set_options>
train = train[train.content_type_id == False].sort_values('timestamp' ).reset_index(drop = True )
csv_logger = CSVLogger('model.log' )<split>
group1 = train.loc[(train.content_type_id == False), ['task_container_id', 'user_id']].groupby(['task_container_id'] ).agg(['count'])
 group1.columns = ['avg_questions']
group2 = train.loc[(train.content_type_id == False), ['task_container_id', 'user_id']].groupby(['task_container_id'] ).agg(['nunique'])
 group2.columns = ['avg_questions']
group3 = group1 / group2
group3['avg_questions_seen'] = group3.avg_questions.cumsum()
print('The amount of questions seen by the average user:')
 group3.iloc[0].avg_questions_seen
x_train2, x_val2, y_train2, y_val2 = train_test_split(x_train, y_train, test_size=VALIDATION_SPLIT, random_state=0 )<train_model>
train = pd.merge(train, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left' )
def fit_model() :
model_file = './log_dir/model_file' + '.hdf5'
checkpointer = ModelCheckpoint(filepath=model_file, verbose=1, monitor='val_acc', save_best_only=True)
 history = model.fit_generator(datagen.flow(x_train2, y_train2, batch_size=BATCH_SIZE),
validation_data=(x_val2, y_val2),
epochs = NB_EPOCH,
verbose = 2,
steps_per_epoch=x_train2.shape[0] // BATCH_SIZE ,
callbacks=[learning_rate, checkpointer, csv_logger, EarlyStopping_cb])
 return(history )<train_model>
results_q2_final = train.loc[train.content_type_id == False, ['question_id','part']].groupby(['question_id'] ).agg(['count'])
 results_q2_final.columns = ['count']
history = fit_model()<save_to_csv>
question2 = pd.merge(questions_df, results_q_final, left_on = 'question_id', right_on = 'question_id', how = 'left')
 question2 = pd.merge(question2, results_q2_final, left_on = 'question_id', right_on = 'question_id', how = 'left')
 question2.quest_pct = round(question2.quest_pct,5)
 question2
results = np.zeros(( x_test.shape[0], NB_CLASSES))
model_file = './log_dir/model_file' + '.hdf5'
model.load_weights(model_file)
 results = results + model.predict(x_test)
 results = np.argmax(results, axis=1)
 results = pd.Series(results, name="Label")
 submission = pd.concat([pd.Series(range(1, 28001), name="ImageId"), results], axis=1)
 submission.to_csv("submission.csv", index=False )<load_from_csv>
prior_mean_user = results_u2_final.explanation_mean_user.mean()
prior_mean_user
!wc -l./submission.csv<import_modules>
train.drop(['timestamp', 'content_type_id', 'question_id', 'part'], axis=1, inplace=True )
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
print('The old length of the training set:')
 print(len(train))
validation = train.groupby('user_id' ).tail(10)
 train = train[~train.index.isin(validation.index)]
print('The length of the training set plus the length of the validation set:')
 print(len(train)+ len(validation))
dataset = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 dataset<count_missing_values>
results_u_val = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_val.columns = ['answered_correctly_user']
results_u2_val = train[['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_val.columns = ['explanation_mean_user']
np.any(dataset.isnull().sum() )<normalization>
X = train.groupby('user_id' ).tail(30)
 train = train[~train.index.isin(X.index)]
print('The length of the training set plus the length of the validation set plus the length of the set to be discarded:')
 print(len(X)+ len(validation)+ len(train))
scaler = StandardScaler()
X = scaler.fit_transform(X)
 X<load_from_csv>
results_u_X = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_X.columns = ['answered_correctly_user']
results_u2_X = train[['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_X.columns = ['explanation_mean_user']
test_data = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
 X_test = test_data.values
X_test.shape<normalization>
del(train )
X_test_scaled = scaler.fit_transform(X_test)
 X_test_scaled<train_model>
X = pd.merge(X, group3, left_on=['task_container_id'], right_index= True, how="left")
 X = pd.merge(X, results_u_X, on=['user_id'], how="left")
 X = pd.merge(X, results_u2_X, on=['user_id'], how="left" )
datagen = ImageDataGenerator(rotation_range=10, zoom_range = 0.1, width_shift_range=0.1, height_shift_range=0.1)
<define_variables>
validation = pd.merge(validation, group3, left_on=['task_container_id'], right_index= True, how="left")
 validation = pd.merge(validation, results_u_val, on=['user_id'], how="left")
 validation = pd.merge(validation, results_u2_val, on=['user_id'], how="left" )
model_list = []<choose_model_class>
lb_make = LabelEncoder()
X.prior_question_had_explanation.fillna(False, inplace = True)
 validation.prior_question_had_explanation.fillna(False, inplace = True)
 validation["prior_question_had_explanation_enc"] = lb_make.fit_transform(validation["prior_question_had_explanation"])
 X["prior_question_had_explanation_enc"] = lb_make.fit_transform(X["prior_question_had_explanation"] )
reduce_lr = ReduceLROnPlateau(monitor='val_accuracy',
patience=3,
verbose=1,
factor=0.2,
min_lr=1e-6 )<choose_model_class>
question2.quest_pct = question2.quest_pct.mask(( question2['count'] < 3),.65)
 question2.quest_pct = question2.quest_pct.mask(( question2.quest_pct <.2)&(question2['count'] < 21),.2)
 question2.quest_pct = question2.quest_pct.mask(( question2.quest_pct >.95)&(question2['count'] < 21),.95 )
for i in range(7):
model = keras.models.Sequential([
keras.layers.Conv2D(32, kernel_size = 3, activation = 'relu', padding = 'same', input_shape = [28, 28, 1]),
keras.layers.BatchNormalization() ,
keras.layers.Conv2D(64, kernel_size = 3, activation = 'relu', padding = 'same'),
keras.layers.MaxPool2D() ,
keras.layers.Conv2D(32, kernel_size = 3, activation = 'relu', padding = 'same'),
keras.layers.BatchNormalization() ,
keras.layers.Conv2D(64, kernel_size = 3, activation = 'relu', padding = 'same'),
keras.layers.MaxPool2D() ,
keras.layers.Flatten() ,
keras.layers.Dropout(0.25),
keras.layers.Dense(256, activation = 'relu'),
keras.layers.Dropout(0.5),
keras.layers.Dense(10, activation = 'softmax')])
 model.compile(loss = 'sparse_categorical_crossentropy', optimizer = 'nadam', metrics = ['accuracy'])
 model_list.append(model)
<train_model>
X = pd.merge(X, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 validation = pd.merge(validation, question2, left_on = 'content_id', right_on = 'question_id', how = 'left' )
history = [0] * 7
for i in range(7):
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size = 0.08)
 history[i] = model_list[i].fit_generator(datagen.flow(X, y, batch_size = 64),
epochs = 20, validation_data =(X_valid, y_valid), callbacks = [reduce_lr])
 print("CNN : {} Maximum Train Accuracy : {} Maximum Validation Accuracy : {}".format(i+1, max(history[i].history['accuracy']), max(history[i].history['val_accuracy'])) )<train_model>
y = X['answered_correctly']
X = X.drop(['answered_correctly'], axis=1)
 X.head()
y_val = validation['answered_correctly']
X_val = validation.drop(['answered_correctly'], axis=1 )
X_test_scaled = X_test_scaled.reshape(( 28000, 28, 28, 1))
X_test_scaled.shape<predict_on_test>
X = X[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part']]
X_val = X_val[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part']]
for i in range(7):
ensemble_cnn_pred = ensemble_cnn_pred + model_list[i].predict(X_test_scaled )<count_values>
X['answered_correctly_user'].fillna(0.65, inplace=True)
 X['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 X['quest_pct'].fillna(content_mean, inplace=True)
 X['part'].fillna(4, inplace = True)
 X['avg_questions_seen'].fillna(1, inplace = True)
 X['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 X['prior_question_had_explanation_enc'].fillna(0, inplace = True )
np.sum(ensemble_cnn_pred[0] )<prepare_output>
X_val['answered_correctly_user'].fillna(0.65, inplace=True)
 X_val['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 X_val['quest_pct'].fillna(content_mean, inplace=True)
 X_val['part'].fillna(4, inplace = True)
 X['avg_questions_seen'].fillna(1, inplace = True)
 X_val['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 X_val['prior_question_had_explanation_enc'].fillna(0, inplace = True )
ensemble_cnn_pred = np.argmax(ensemble_cnn_pred, axis = 1)
 ensemble_cnn_pred[0]<prepare_output>
lgb_train = lgb.Dataset(X, y, categorical_feature = ['part', 'prior_question_had_explanation_enc'],free_raw_data=False)
 lgb_eval = lgb.Dataset(X_val, y_val, categorical_feature = ['part', 'prior_question_had_explanation_enc'], reference=lgb_train, free_raw_data=False )
pred_df_ensemble_cnn = pd.DataFrame(columns = ['ImageId', 'Label'])
 pred_df_ensemble_cnn['ImageId'] = np.arange(1, 28001)
 pred_df_ensemble_cnn['Label'] = ensemble_cnn_pred
pred_df_ensemble_cnn<save_to_csv>
y_val_array = np.array(y_val)
 def obj_and_bounds(trial):
params = {
'num_leaves': trial.suggest_int('num_leaves', 160, 162),
'boosting_type': 'gbdt',
'max_bin': trial.suggest_int('max_bin', 889, 891),
'objective': 'binary',
'metric': 'auc',
'max_depth': trial.suggest_int('max_depth', 10, 12),
'min_child_weight': trial.suggest_int('min_child_weight', 10, 12),
'feature_fraction': trial.suggest_uniform('feature_fraction', 0.69, 0.7),
'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.92, 0.93),
'bagging_freq': trial.suggest_int('bagging_freq', 6, 8),
'min_child_samples': trial.suggest_int('min_child_samples', 76, 78),
'lambda_l1': trial.suggest_loguniform('lambda_l1', 2e-2, 3e-2),
'lambda_l2': trial.suggest_loguniform('lambda_l2', 9e-8, 1e-7),
'early_stopping_rounds': 10
}
model = lgb.train(params, lgb_train, valid_sets=[lgb_train, lgb_eval], verbose_eval=1000, num_boost_round=300)
 val_pred = model.predict(X_val)
 score = roc_auc_score(y_val_array, val_pred)
 print("ROC/AUC = ")
 print(score)
 return score
pred_df_ensemble_cnn.to_csv('ens_cnn_with_aug_sub.csv', index = False )<import_modules>
study = optuna.create_study(direction='maximize')
 study.optimize(obj_and_bounds, n_trials=2)
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import tensorflow as tf<load_from_csv>
print('Number of finished trials: {}'.format(len(study.trials)))
 trial = study.best_trial
print('ROC/AUC of the best trial: {}'.format(trial.value))
print('Parameters of the best trial: ')
 for key, value in trial.params.items() :
print(' {}: {}'.format(key, value))
df=pd.read_csv(".. /input/digit-recognizer/train.csv")
 df.head()<load_from_csv>
model = lgb.train(
 trial.params,
lgb_train,
valid_sets=[lgb_train, lgb_eval],
verbose_eval=1000,
num_boost_round=2000
)
df_test=pd.read_csv(".. /input/digit-recognizer/test.csv" ).values
df_test[0:5]<prepare_x_and_y>
y_pred = model.predict(X_val)
 roc_auc_score(y_val_array, y_pred )
<split><EOS> 
iter_test = env.iter_test()
for(test_df, sample_prediction_df)in iter_test:
test_df['task_container_id'] = test_df.task_container_id.mask(test_df.task_container_id > 9999, 9999)
 test_df = pd.merge(test_df, group3, left_on=['task_container_id'], right_index= True, how="left")
 test_df = pd.merge(test_df, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df = pd.merge(test_df, results_u_final, on=['user_id'], how="left")
 test_df = pd.merge(test_df, results_u2_final, on=['user_id'], how="left")
 test_df['answered_correctly_user'].fillna(0.65, inplace=True)
 test_df['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 test_df['quest_pct'].fillna(content_mean, inplace=True)
 test_df['part'].fillna(4, inplace = True)
 test_df['avg_questions_seen'].fillna(1, inplace = True)
 test_df['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df["prior_question_had_explanation_enc"] = lb_make.fit_transform(test_df["prior_question_had_explanation"])
 test_df['answered_correctly'] = model.predict(test_df[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part']])
 env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )