Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<prepare_x_and_y>
import pandas as pd
import numpy as np
import gc
from sklearn.metrics import roc_auc_score
from collections import defaultdict
from tqdm.notebook import tqdm
import lightgbm as lgb
import riiideducation
import matplotlib.pyplot as plt
import seaborn as sns
import random
import os
target_train = data_train.iloc[:, 0].values
image_train = data_train.iloc[:, 1:].values
image_test = data_test.values<data_type_conversions>
class UserFeats(object):
def __init__(
 self 
):
self._ans_cnt = 0
self._ans_corr_cnt = 0
self._questions_seen_id = dict()
self._part_info = dict()
self._last_container = -1
self._last_timestamp = [np.nan for i in range(4)]
self._last_correct_timestamp = [np.nan, np.nan]
self._last_incorrect_timestamp = [np.nan, np.nan]
self._total_prequestions_time_count = 0
self._total_prexlpan_count = 0
self._difficult_solved_count = 0
self._simple_mistake_count = 0
self._lecture_worked = 0
self._past_lecture_time = np.nan
def update_lc(self, time):
self._lecture_worked += 1
self._past_lecture_time = time
X = image_train.reshape(-1,28,28,1 ).astype('float32')
 X_test = image_test.reshape(-1,28,28,1 ).astype('float32')
 y = to_categorical(target_train.reshape(-1,1), num_classes=10)
 X /= 255
X_test /= 255
plt.imshow(image_train[0].reshape(28,28))<train_model>
class QuesFeats(object):
def __init__(self, feats_tuple):
self.part = feats_tuple[0]
self.crr_cnt = feats_tuple[1]
self.total_cnt = feats_tuple[2]
self.explan_false_mean = feats_tuple[4]
self.explan_true_mean = feats_tuple[5]
self.var = feats_tuple[6]
self.bundle_num = feats_tuple[7]
self.part_mean_correct = feats_tuple[8]
self.part_total_correct = feats_tuple[9]
self.part_total_count = feats_tuple[13]
self.part_var = feats_tuple[10]
self.part_bundle_id = feats_tuple[11]
self.content_explan_sum = feats_tuple[12]
self.tags_lsi = feats_tuple[15]
def get_acc(self, c_id):
return self.crr_cnt[c_id]/self.total_cnt[c_id]
def get_var(self, c_id):
return self.var[c_id]
datagen = ImageDataGenerator(rotation_range=10, width_shift_range=0.1, height_shift_range=0.1, zoom_range=0.1)
 datagen.fit(X )<choose_model_class>
def df_preprocessing(df, q_tmp):
df['prior_question_had_explanation'] = df.prior_question_had_explanation.fillna(False ).astype('int8')
 df['timestamp'] = df['timestamp']/(1000*1000)
 df['prior_question_elapsed_time'] = df['prior_question_elapsed_time']/(1000*1000)
 df['prior_question_elapsed_time'].fillna(0.013, inplace = True)
 df = pd.merge(df, q_tmp[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')
 del q_tmp
return df
num_models = 15
model = [None] * num_models
for i in range(num_models):
model[i] = Sequential()
model[i].add(Conv2D(32,(3, 3), activation='relu', input_shape=(28,28,1)))
 model[i].add(BatchNormalization())
 model[i].add(Conv2D(32,(3, 3), activation='relu'))
model[i].add(BatchNormalization())
 model[i].add(MaxPooling2D(( 2, 2), strides=2))
model[i].add(BatchNormalization())
 model[i].add(Conv2D(64,(3, 3), activation='relu'))
model[i].add(BatchNormalization())
 model[i].add(Conv2D(64,(3, 3), activation='relu'))
model[i].add(BatchNormalization())
 model[i].add(MaxPooling2D(( 2, 2), strides=2))
model[i].add(Flatten())
 model[i].add(BatchNormalization())
 model[i].add(Dense(256, activation='relu'))
model[i].add(BatchNormalization())
 model[i].add(Dropout(0.5))
model[i].add(Dense(10, activation='softmax'))
model[i].compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
 for i in range(num_models):
t = time()
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.1, random_state=i)
 history = model[i].fit_generator(datagen.flow(X_train, y_train, batch_size=512), epochs=50, verbose=0,
validation_data=(X_val, y_val))
print(history.history['val_accuracy'][-1], time() -t )<save_to_csv>
def handle_features(df,user_dict,content_f, add):
if add:
row = df[['user_id', 'answered_correctly', 'task_container_id', 'content_id', 'part', 'prior_question_elapsed_time', 'prior_question_had_explanation','timestamp']]
res = dict()
total = user_dict[row[0]]._ans_cnt if user_dict[row[0]]._ans_cnt !=0 else 1
if row[4] in user_dict[row[0]]._part_info:
if row[3] not in user_dict[row[0]]._questions_seen_id:
user_dict[row[0]]._questions_seen_id[row[3]] = 0
else:
user_dict[row[0]]._part_info[row[4]] = [0,0]
user_dict[row[0]]._questions_seen_id[row[3]] = 0
user_dict[row[0]]._part_info[row[4]] = [0,0]
res['task_container_id'] = row[2]
res['content_id'] = row[3]
res['part'] = row[4]
res['prior_question_elapsed_time'] = row[5]
res['prior_question_had_explanation'] = row[6]
res['timestamp'] = row[7]
res['u_answered_correctly_count'] = user_dict[row[0]]._ans_corr_cnt
res['u_answered_incorrectly_count'] = user_dict[row[0]]._ans_cnt - user_dict[row[0]]._ans_corr_cnt
res['u_answered_correctly_avg'] = user_dict[row[0]]._ans_corr_cnt/total
res['u_elapsed_time_avg'] = user_dict[row[0]]._total_prequestions_time_count/total
res['u_explanation_avg'] = user_dict[row[0]]._total_prexlpan_count/total
if row[2] == user_dict[row[0]]._last_container:
res['timestamp_u_recency_1'] = row[7] - user_dict[row[0]]._last_timestamp[0]
res['timestamp_u_recency_2'] = row[7] - user_dict[row[0]]._last_timestamp[1]
res['timestamp_u_recency_3'] = row[7] - user_dict[row[0]]._last_timestamp[2]
res['u_timestamp_correct_recency'] = row[7] - user_dict[row[0]]._last_correct_timestamp[0]
res['u_timestamp_incorrect_recency'] = row[7] - user_dict[row[0]]._last_incorrect_timestamp[0]
else:
res['timestamp_u_recency_1'] = row[7] - user_dict[row[0]]._last_timestamp[1]
res['timestamp_u_recency_2'] = row[7] - user_dict[row[0]]._last_timestamp[2]
res['timestamp_u_recency_3'] = row[7] - user_dict[row[0]]._last_timestamp[3]
res['u_timestamp_correct_recency'] = row[7] - user_dict[row[0]]._last_correct_timestamp[1]
res['u_timestamp_incorrect_recency'] = row[7] - user_dict[row[0]]._last_incorrect_timestamp[1]
res['upart_answered_incorrect'] = user_dict[row[0]]._part_info[row[4]][1] - user_dict[row[0]]._part_info[row[4]][0]
res['upart_answered_correct'] = user_dict[row[0]]._part_info[row[4]][0]
res['upart_answered_crr_mean'] = user_dict[row[0]]._part_info[row[4]][0] /(user_dict[row[0]]._part_info[row[4]][1] if user_dict[row[0]]._part_info[row[4]][1] !=0 else 1)
 res['upart_answered_incorrect'] = 0
res['upart_answered_correct'] = 0
res['upart_answered_crr_mean'] = 0
res['u_lecture_reviewed'] = user_dict[row[0]]._lecture_worked
res['u_lecture_avg'] = user_dict[row[0]]._lecture_worked/(user_dict[row[0]]._lecture_worked + total)
 res['u_last_lecture_timedif'] = row[7] - user_dict[row[0]]._past_lecture_time
res['u_solved_diffcult_questions_num'] = user_dict[row[0]]._difficult_solved_count
res['u_failed_simple_questions_num'] = user_dict[row[0]]._simple_mistake_count
res['u_questions_seen_time'] = 0.5
res['explan_false_mean'] = content_f.explan_false_mean[int(row[3])]
res['explan_true_mean'] = content_f.explan_true_mean[int(row[3])]
res['crr_cnt'] = content_f.crr_cnt[int(row[3])]
res['incrr_cnt'] = content_f.total_cnt[int(row[3])] - content_f.crr_cnt[int(row[3])]
res['var'] = content_f.var[int(row[3])]
res['bundle_num'] = content_f.bundle_num[int(row[3])]
res['part_mean_correct'] = content_f.part_mean_correct[int(row[3])]
res['part_total_correct'] = content_f.part_total_correct[int(row[3])]
res['part_total_incorrect'] = content_f.part_total_count[int(row[3])] - content_f.part_total_correct[int(row[3])]
res['part_var'] = content_f.part_var[int(row[3])]
res['part_bundle_id'] = content_f.part_bundle_id[int(row[3])]
res['content_explan_sum'] = content_f.content_explan_sum[int(row[3])]
res['tags_lsi'] = content_f.tags_lsi[int(row[3])]
return res
for num, row in tqdm(enumerate(df[['user_id', 'answered_correctly', 'task_container_id', 'content_id', 'part', 'prior_question_elapsed_time', 'prior_question_had_explanation','timestamp']].values)) :
if row[0] not in user_dict:
user_dict[row[0]] = UserFeats()
if row[4] in user_dict[row[0]]._part_info:
if row[3] not in user_dict[row[0]]._questions_seen_id:
user_dict[row[0]]._questions_seen_id[row[3]] = 0
else:
user_dict[row[0]]._part_info[row[4]] = [0,0]
user_dict[row[0]]._questions_seen_id[row[3]] = 0
user_dict[row[0]]._part_info[row[4]] = [0,0]
if row[1] == -1:
user_dict[row[0]]._past_lecture_time = row[7]
user_dict[row[0]]._lecture_worked += 1
continue
user_dict[row[0]]._ans_cnt += 1
user_dict[row[0]]._ans_corr_cnt += row[1]
user_dict[row[0]]._part_info[row[4]][0] += row[1]
user_dict[row[0]]._part_info[row[4]][1] += 1
if user_dict[row[0]]._questions_seen_id[row[3]] < 8:
user_dict[row[0]]._questions_seen_id[row[3]] += 1
if row[1] == 1:
if user_dict[row[0]]._last_correct_timestamp[1] != row[7]:
user_dict[row[0]]._last_correct_timestamp.pop(0)
 user_dict[row[0]]._last_correct_timestamp.append(row[7])
 else:
if user_dict[row[0]]._last_incorrect_timestamp[1] != row[7]:
user_dict[row[0]]._last_incorrect_timestamp.pop(0)
 user_dict[row[0]]._last_incorrect_timestamp.append(row[7])
 user_dict[row[0]]._total_prequestions_time_count += row[5]
user_dict[row[0]]._total_prexlpan_count += row[6]
if row[2] != user_dict[row[0]]._last_container:
user_dict[row[0]]._last_container = row[2]
user_dict[row[0]]._last_timestamp.pop(0)
 user_dict[row[0]]._last_timestamp.append(row[7])
 if content_f.get_acc(int(row[3])) < 0.574455 and row[1] == 1:
user_dict[row[0]]._difficult_solved_count += 1
if content_f.get_acc(int(row[3])) > 0.856363 and row[1] == 0:
user_dict[row[0]]._simple_mistake_count += 1
y_pred = np.zeros(( X_test.shape[0], 10))
for i in range(num_models):
y_pred += model[i].predict(X_test, batch_size=512, verbose=1)
 y_pred = np.argmax(y_pred, axis=1)
 submissions = pd.DataFrame({'ImageId': list(range(1,len(y_pred)+1)) ,
'Label': y_pred})
 submissions.to_csv('submission.csv', index=False, header=True )<predict_on_test>
def process_questions(filepath):
default_list = []
q_features = [
'part',
'content_correct_num',
'content_total_num',
'content_correct_mean',
'content_explation_false_mean',
'content_explation_true_mean',
'var',
'bundle_num',
'part_mean_correct',
'part_total_correct',
'part_var',
'part_bundle_id',
'content_explan_sum',
'part_num_user',
'part_num_c',
'tags_lsi',
]
q = pd.read_pickle(filepath)[q_features]
part = tuple(q['part'].values)
 for i in q.keys() :
default_list.append(tuple(q[i].values))
q_list = tuple(default_list)
 q_feats = QuesFeats(q_list)
 del default_list, q_list, q
gc.collect()
return q_feats
y_pred_val = np.zeros(( X_val.shape[0], 10))
for i in range(num_models):
y_pred_val += model[i].predict(X_val, batch_size=512, verbose=1)
 y_pred_val = np.argmax(y_pred_val, axis=1)
 y_true_val = np.argmax(y_val, axis=1)
 print(classification_report(y_true_val, y_pred_val, digits=4)) 
<set_options>
def read_and_preprocess(feature_engineering = False):
train = '.. /input/riiid-test-answer-prediction/train.csv'
question_file = '.. /input/qdataset/questions_features.pkl'
feld_needed = ['timestamp', 'user_id', 'answered_correctly', 'content_id', 'content_type_id', 'prior_question_elapsed_time', 'prior_question_had_explanation']
q_tmp= pd.read_csv('.. /input/riiid-test-answer-prediction/questions.csv')
 q_tmp['part'] = q_tmp['part'].astype(np.int32)
 print('User feature calculation started...')
 print('
')
 user_dict = dict()
content_f = process_questions(question_file)
 chunk_round = 0
reader = pd.read_csv(train, iterator=True, chunksize=10000000)
 for chunk in reader:
print('chunk', chunk_round, 'started.... ')
 chunk = df_preprocessing(chunk, q_tmp)
 handle_features(chunk, user_dict, content_f, False)
 del chunk
gc.collect()
chunk_round += 1
print('User feature calculation completed...')
 print('
')
 return user_dict,content_f
%matplotlib inline 
<load_from_csv>
user_dict,content_f = read_and_preprocess()
model = lgb.Booster(model_file='.. /input/trainmodel/model.txt')
 print('model load done.... ' )
<load_from_csv><EOS> 
TARGET = ['answered_correctly']
FEATURES = [
'timestamp',
'content_id',
'task_container_id',
'prior_question_elapsed_time',
'prior_question_had_explanation',
'part',
'u_answered_correctly_count',
'u_answered_correctly_avg',
'u_elapsed_time_avg',
'u_explanation_avg',
'timestamp_u_recency_1',
'timestamp_u_recency_2',
'timestamp_u_recency_3',
'u_timestamp_correct_recency',
'u_timestamp_incorrect_recency',
'upart_answered_incorrect',
'upart_answered_correct',
'upart_answered_crr_mean',
'u_lecture_reviewed',
'u_lecture_avg',
'u_last_lecture_timedif',
'u_solved_diffcult_questions_num',
'u_failed_simple_questions_num',
'u_questions_seen_time',
'explan_false_mean',
'explan_true_mean',
'crr_cnt',
'incrr_cnt',
'var',
'bundle_num',
'part_mean_correct',
'part_total_correct',
'part_total_incorrect',
'part_var',
'part_bundle_id',
'content_explan_sum',
'tags_lsi',
]
q_tmp= pd.read_csv('.. /input/riiid-test-answer-prediction/questions.csv')
 q_tmp['part'] = q_tmp['part'].astype(np.int32)
 env = riiideducation.make_env()
iter_test = env.iter_test()
previous_test_df = None
for(test_df, sample_prediction_df)in iter_test:
if previous_test_df is not None:
previous_test_df[TARGET] = eval(test_df["prior_group_answers_correct"].iloc[0])
 handle_features(previous_test_df, user_dict, content_f, False)
 test_df = df_preprocessing(test_df, q_tmp)
 previous_test_df = test_df.copy()
test_df[TARGET] = -2
test_df[TARGET] = test_df[TARGET].astype('float32')
 print(test_df.keys())
 for index, row in test_df.iterrows() :
try:
if row['user_id'] not in user_dict:
user_dict[row['user_id']] = UserFeats()
if row['content_type_id'] == 1:
user_dict[row['user_id']].update_lc(row['timestamp'])
 continue
if row['content_type_id'] == 0:
predict_dict = handle_features(row, user_dict, content_f, True)
 predict_list = []
for i in FEATURES:
predict_list.append(predict_dict[i])
 prd_value = float(model.predict([predict_list])[0])
 print('prd_value', prd_value)
 test_df.loc[test_df.row_id == row.row_id, TARGET] = prd_value
except Exception as e:
print(e)
 pass
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )