Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<find_best_model_class>
import psutil
import joblib
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
pipe_xt = Pipeline([ 
("scaler", StandardScaler()), 
("pca", PCA()), 
("xt", ExtraTreesClassifier(criterion = "gini",
max_features = "auto")) ])
 params_xt = {
"pca__n_components" : [2, 6, 10, 18],
"xt__n_estimators" : [300, 500, 700],
"xt__max_depth" : [5, 7, 9]}
xt_score, grid_xt = model_builder(X, Y, pipe_xt, params_xt, 10, "Extra Trees")
 Y_predicted_xt = grid_xt.predict_proba(X)[:, 1]<find_best_params>
env = riiideducation.make_env()
iter_test = env.iter_test()
pipe_knn = Pipeline([ 
("scaler", StandardScaler()), 
("knn", KNeighborsClassifier(algorithm = "auto")) ])
 params_knn = {
"knn__n_neighbors" : [2, 3, 5, 7, 9],
"knn__leaf_size" : [10, 20, 30, 40]}
knn_score, grid_knn = model_builder(X, Y, pipe_knn, params_knn, 10, "KNN")
 Y_predicted_knn = grid_knn.predict_proba(X)[:, 1]<define_variables>
MAX_SEQ = 100
all_results = [xgb_score, boost_score, log_score, rf_score,
svm_score, bag_score, xt_score, knn_score, cat_score, gbm_score]
result_names = ["XGBoost",
"Gradient Boosting",
"Logistic Regression",
"Random Forest",
"SVM",
"Bagging Classifier",
"Extra Trees Classifier",
"KNN",
"CatBoost",
"LightGBM"]
fig = plt.figure(figsize =(14,7))
fig.suptitle("Algorithm Comparison")
 ax = fig.add_subplot(111)
 plt.boxplot(all_results)
 ax.set_xticklabels(result_names)
 plt.show()<install_modules>
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128):
super(SAKTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.embedding = nn.Embedding(2*n_skill+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = x + pos_x
e = self.e_embedding(question_ids)
 x = x.permute(1, 0, 2)
 e = e.permute(1, 0, 2)
 att_mask = future_mask(x.size(0)).to(device)
 att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
 att_output = self.layer_normal(att_output + e)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
%%capture
!pip install keras-tuner<import_modules>
skills = joblib.load("/kaggle/input/riiid-sakt-model-dataset-public/skills.pkl.zip")
 n_skill = len(skills)
 group = joblib.load("/kaggle/input/riiid-sakt-model-dataset-public/group.pkl.zip" )
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout
from kerastuner.tuners import RandomSearch<split>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model = SAKTModel(n_skill, embed_dim=128)
 try:
model.load_state_dict(torch.load("/kaggle/input/riiid-sakt-model-dataset-public/sakt_model.pt"))
except:
model.load_state_dict(torch.load("/kaggle/input/riiid-sakt-model-dataset-public/sakt_model.pt", map_location='cpu'))
model.to(device)
 model.eval()
X_train, X_test, Y_train, Y_test = train_test_split(X,
Y,
random_state = 0,
test_size = 0.25 )<choose_model_class>
prev_test_df = None
for(test_df, sample_prediction_df)in iter_test:
if(prev_test_df is not None)&(psutil.virtual_memory().percent < 90):
prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prev_test_df = prev_test_df[prev_test_df.content_type_id == False]
prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
if prev_user_id in group.index:
group[prev_user_id] =(
 np.append(group[prev_user_id][0], prev_group[prev_user_id][0])[-MAX_SEQ:],
np.append(group[prev_user_id][1], prev_group[prev_user_id][1])[-MAX_SEQ:] 
)
 else:
group[prev_user_id] =(
 prev_group[prev_user_id][0],
prev_group[prev_user_id][1] 
)
 prev_test_df = test_df.copy()
test_df = test_df[test_df.content_type_id == False]
test_dataset = TestDataset(group, test_df, skills)
 test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)
 outs = []
for item in test_dataloader:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, att_weight = model(x, target_id)
 outs.extend(torch.sigmoid(output)[:, -1].view(-1 ).data.cpu().numpy())
 test_df['answered_correctly'] = outs
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )