Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<choose_model_class>
import numpy as np
import pandas as pd
import lightgbm as lgb
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.metrics import roc_auc_score
from multiprocessing import cpu_count
from tqdm.notebook import tqdm
from cairosvg import svg2png
from PIL import Image
from io import BytesIO
import gc
import os
import sys
layers = [
tf.keras.layers.Conv2D(16, kernel_size=3, strides=1, padding='same', input_shape=(28,28,1)) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.ReLU() ,
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Conv2D(32, kernel_size=3, strides=1, padding='valid'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.ReLU() ,
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Conv2D(64, kernel_size=5, strides=1, padding='valid'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.ReLU() ,
tf.keras.layers.Dropout(0.25),
tf.keras.layers.MaxPool2D(strides=2),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(512, activation='relu'),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Dense(256, activation='relu'),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Dense(10, activation='softmax')
 ]
model = tf.keras.Sequential(layers=layers)
 lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(1e-3, decay_steps=3000, decay_rate=0.95)
 optimizer = tf.keras.optimizers.Adam(learning_rate=lr_schedule)
 model.compile(optimizer=optimizer, loss='sparse_categorical_crossentropy', metrics=['accuracy'])
 history = model.fit(train_ds, epochs=40, validation_data=val_ds )<save_to_csv>
VERSION = 'V1E'
NUM_BOOST_ROUND = 5000
VERBOSE_EVAL = 10
METRICS = ['auc']
N_ROWS = 99271300
def get_index_np() :
return np.arange(N_ROWS )
pred_test = np.argmax(model.predict(test_ds), axis=1)
 out_df = pd.DataFrame({'ImageId': np.arange(1, len(pred_test)+1), 'Label': pred_test})
 out_df.to_csv('output.csv', index=False )<import_modules>
FEATURES = np.load(f'/kaggle/input/riiid-training-and-prediction-using-a-state-data/train_features_{VERSION}.npz', allow_pickle=True )
import itertools
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Flatten, Dense, Conv2D, Lambda, MaxPooling2D, Dropout, BatchNormalization
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import train_test_split<load_from_csv>
given_features = [
'prior_question_elapsed_time',
]
deduced_features = [
'mean_user_accuracy',
'answered_correctly_user',
'answered_user',
'mean_content_accuracy',
'part',
'hmean_user_content_accuracy',
'attempt',
]
features = given_features + deduced_features
target = 'answered_correctly'
categorical_feature = ['part', 'tags', 'tags_label', 'prior_question_had_explanation']
categorical_feature_idxs = []
for v in categorical_feature:
try:
categorical_feature_idxs.append(features.index(v))
except:
pass
train_df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test_df = pd.read_csv('/kaggle/input/digit-recognizer/test.csv' )<split>
def get_train_val_idxs(TRAIN_SIZE, VAL_SIZE):
train_idxs = []
val_idxs = []
NEW_USER_FRAC = 1/4
np.random.seed(42)
 df = pd.DataFrame(index=get_index_np())
 for col in ['user_id']:
df[col] = FEATURES[col]
df['index'] = df.index.values.astype(np.uint32)
 user_id_index = df.groupby('user_id')['index'].apply(np.array)
 for indices in user_id_index.sample(user_id_index.size, random_state=42):
if len(train_idxs)> TRAIN_SIZE:
break
if len(val_idxs)< VAL_SIZE:
if np.random.rand() < NEW_USER_FRAC:
val_idxs += list(indices)
 else:
offset = np.random.randint(0, indices.size)
 train_idxs += list(indices[:offset])
 val_idxs += list(indices[offset:])
 else:
train_idxs += list(indices)
 return train_idxs, val_idxs
train_idxs, val_idxs = get_train_val_idxs(int(50e6), 2.5e6)
 print(f'len train_idxs: {len(train_idxs)}, len validation_idxs: {len(val_idxs)}' )
train_df.sample(frac=1)
 cols = list(train_df.columns)
 cols.remove('label')
 X = train_df[cols]
Y = train_df['label']
X_train, X_dev, Y_train, Y_dev = train_test_split(X, Y, test_size=0.1, random_state=0)
 X_test = test_df[cols]<categorify>
def make_x_y(FEATURES, train_idxs, val_idxs):
X_train = np.ndarray(shape=(len(train_idxs), len(features)) , dtype=np.float32)
 X_val = np.ndarray(shape=(len(val_idxs), len(features)) , dtype=np.float32)
 for idx, feature in enumerate(tqdm(features)) :
X_train[:,idx] = FEATURES[feature][train_idxs].astype(np.float32)
 X_val[:,idx] = FEATURES[feature][val_idxs].astype(np.float32)
 y_train = FEATURES[target][train_idxs].astype(np.int8)
 y_val = FEATURES[target][val_idxs].astype(np.int8)
 return X_train, y_train, X_val, y_val
X_train, y_train, X_val, y_val = make_x_y(FEATURES, train_idxs, val_idxs )
X_train = X_train.values.reshape(-1, 28, 28)
 X_dev = X_dev.values.reshape(-1, 28, 28)
 X_test = X_test.values.reshape(-1, 28, 28 )<normalization>
pd.DataFrame(X_train[:10], columns=features )
X_train = np.expand_dims(X_train, axis=-1)/ 255
X_dev = np.expand_dims(X_dev, axis=-1)/ 255
X_test = np.expand_dims(X_test, axis=-1)/ 255<normalization>
train_data = lgb.Dataset(
 data = X_train,
label = y_train,
categorical_feature = None, 
)
 val_data = lgb.Dataset(
 data = X_val,
label = y_val,
categorical_feature = None,
)
def data_augmentation(x_data, y_data, batch_size):
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False 
)
 datagen.fit(x_data)
 train_data = datagen.flow(x_data, y_data, batch_size=batch_size, shuffle=True)
 return train_data<normalization>
del X_train, y_train, X_val, y_val
gc.collect()
BATCH_SIZE = 64
aug_train_data = data_augmentation(X_train, Y_train, BATCH_SIZE )<choose_model_class>
lgbm_params = {
'objective': 'binary',
'metric': METRICS,
}
layers = [
Conv2D(filters=96, kernel_size=(11, 11), strides=2, activation='relu', input_shape=(28, 28, 1)) ,
MaxPooling2D(pool_size=(3, 3), strides=2),
Conv2D(filters=256, kernel_size=(5, 5), padding='same', activation='relu'),
Flatten() ,
Dense(9216, activation='relu'),
Dense(4096, activation='relu'),
Dense(4096, activation='relu'),
Dense(10, activation='softmax'),
]<train_model>
%%time
def train() :
evals_result = {}
model = lgb.train(
 params = lgbm_params,
train_set = train_data,
valid_sets = [val_data],
num_boost_round = NUM_BOOST_ROUND,
verbose_eval = VERBOSE_EVAL,
evals_result = evals_result,
early_stopping_rounds = 10,
categorical_feature = categorical_feature_idxs,
feature_name = features, 
)
 model.save_model(f'model_{VERSION}_{NUM_BOOST_ROUND}.lgb')
 return model, evals_result
model, evals_result = train()
model = Sequential(layers)
 optimizer = tf.keras.optimizers.Adam(learning_rate=0.0001)
 model.compile(
 optimizer=optimizer,
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] 
)
 callbacks = [
tf.keras.callbacks.ReduceLROnPlateau(monitor="loss",factor=0.1, patience=2, min_lr=0.000001, verbose=1),
]
hist = model.fit(
 aug_train_data,
steps_per_epoch=X_train.shape[0] // BATCH_SIZE,
batch_size=BATCH_SIZE,
validation_data=(X_dev, Y_dev),
epochs=50,
callbacks=callbacks
)<train_model>
def show_feature_importances(model, importance_type, max_num_features=10**10):
feature_importances = pd.DataFrame()
feature_importances['feature'] = features
feature_importances['value'] = pd.DataFrame(model.feature_importance(importance_type))
feature_importances = feature_importances.sort_values(by='value', ascending=False)
 feature_importances.to_csv(f'feature_importances_{importance_type}.csv')
 feature_importances = feature_importances[:max_num_features]
plt.figure(figsize=(20, 8))
plt.xlim([0, feature_importances.value.max() *1.1])
 plt.title(f'Feature {importance_type}', fontsize=18);
sns.barplot(data=feature_importances, x='value', y='feature', palette='rocket');
for idx, v in enumerate(feature_importances.value):
plt.text(v, idx, " {:.2e}".format(v))
show_feature_importances(model, 'gain')
 show_feature_importances(model, 'split' )
predictions = Y_pred
print(predictions[0])
 print('Predicted digit is: ' + str(np.argmax(predictions[0])))
 print('Accuracy is: ' + str(np.max(predictions[0] * 100)) + '%')
 plt.imshow(X_dev[0].reshape(( 28, 28)) , cmap=plt.cm.binary )<predict_on_test>
del train_data
gc.collect()
predictions = model.predict(X_test )<load_from_csv>
def get_features_questions_df() :
features_df = pd.DataFrame(index=get_index_np())
 for col in tqdm(['content_id', 'part', 'tags', 'tags_label', 'mean_content_accuracy']):
features_df[col] = FEATURES[col]
features_questions_df = features_df.groupby('content_id')[[
'content_id',
'part',
'tags',
'tags_label',
'mean_content_accuracy',
]].first().reset_index(drop=True ).sort_values('content_id')
 return features_questions_df
features_questions_df = get_features_questions_df()
print(f'features_questions_df, rows: {features_questions_df.shape[0]}')
 display(features_questions_df.head() )
submission = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')
 submission.head()<prepare_output>
def get_state() :
features_df = pd.DataFrame(index=get_index_np())
 for col in tqdm(['user_id', 'content_id', 'answered_correctly']):
features_df[col] = FEATURES[col]
mean_user_accuracy = features_df.groupby('user_id')['answered_correctly'].mean().values
answered_correctly_user = features_df.groupby('user_id')['answered_correctly'].sum().values
answered_user = features_df.groupby('user_id')['answered_correctly'].count().values
state = dict()
for user_id in features_df['user_id'].unique() :
state[user_id] = {}
total = len(state.keys())
 user_content = features_df.groupby('user_id')['content_id'].apply(np.array ).apply(np.sort ).apply(np.unique)
 user_attempts = features_df.groupby(['user_id', 'content_id'])['content_id'].count().astype(np.uint8 ).groupby('user_id' ).apply(np.array ).values
user_attempts -= 1
for user_id, content, attempt in tqdm(zip(state.keys() , user_content, user_attempts),total=total):
state[user_id]['user_content_attempts'] = dict(zip(content, attempt))
del user_content, user_attempts
gc.collect()
for idx, user_id in enumerate(state.keys()):
state[user_id]['mean_user_accuracy'] = mean_user_accuracy[idx]
state[user_id]['answered_correctly_user'] = answered_correctly_user[idx]
state[user_id]['answered_user'] = answered_user[idx]
return state
state = get_state()
print('Example of the state for user 2746, attempt counting starts at 0 as the pandas cumcount function is used to create the attempt feature')
 display(state[2746] )
for i in submission.index:
submission['Label'][i] = np.argmax(predictions[i] )<save_to_csv>
def get_user_data(state, test_df):
attempt, mean_user_accuracy, answered_correctly_user, answered_user = [], [], [], []
for idx,(user_id, content_id)in test_df[['user_id', 'content_id']].iterrows() :
if user_id in state:
if content_id in state[user_id]['user_content_attempts']:
state[user_id]['user_content_attempts'][content_id] = min(4, state[user_id]['user_content_attempts'][content_id] + 1)
 else:
state[user_id]['user_content_attempts'][content_id] = 0
else:
dict_keys = ['mean_user_accuracy', 'answered_correctly_user', 'answered_user', 'user_content_attempts']
dict_default_vals = [0.680, 0, 0, dict(zip([content_id],[0])) ]
state[user_id] = dict(zip(dict_keys, dict_default_vals))
attempt.append(state[user_id]['user_content_attempts'][content_id])
 mean_user_accuracy.append(state[user_id]['mean_user_accuracy'])
 answered_correctly_user.append(state[user_id]['answered_correctly_user'])
 answered_user.append(state[user_id]['answered_user'])
 return attempt, mean_user_accuracy, answered_correctly_user, answered_user
submission.to_csv("sample_submission.csv", index=False )<save_model>
def update_user_data(state, features_questions_df, prev_test_df):
for user_id, content_id, answered_correctly in prev_test_df[['user_id', 'content_id', 'answered_correctly']].values:
state[user_id]['answered_correctly_user'] += answered_correctly
state[user_id]['answered_user'] += 1
state[user_id]['mean_user_accuracy'] = state[user_id]['answered_correctly_user'] / state[user_id]['answered_user']
model.save('model')
 model.save('model.h5' )<install_modules>
env = riiideducation.make_env()
iter_test = env.iter_test()
!pip install -q efficientnet_pytorch > /dev/null<set_options>
prev_test_df = None
mean_attempt_acc_factor = FEATURES['mean_attempt_acc_factor']
for idx,(test_df, _)in tqdm(enumerate(iter_test)) :
if prev_test_df is not None:
prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])
 update_user_data(state, features_questions_df, prev_test_df.loc[prev_test_df['content_type_id'] == 0])
 if idx is 1:
display(test_df)
 display(prev_test_df)
 attempt, mean_user_accuracy, answered_correctly_user, answered_user = get_user_data(state, test_df)
 test_df['attempt'] = attempt
test_df['mean_user_accuracy'] = mean_user_accuracy
test_df['answered_correctly_user'] = answered_correctly_user
test_df['answered_user'] = answered_user
test_df = features_questions_df.merge(test_df, how='right', on='content_id')
 test_df['prior_question_elapsed_time'].fillna(23916, inplace=True)
 test_df['hmean_user_content_accuracy'] = 2 *(
(test_df['mean_user_accuracy'] * test_df['mean_content_accuracy'])/ 
(test_df['mean_user_accuracy'] + test_df['mean_content_accuracy'])
)
 test_df['answered_correctly'] = model.predict(test_df[features])
 env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])
 prev_test_df = test_df.copy()
<load_from_csv><EOS> 
submission = pd.read_csv('./submission.csv' )