Kaggle competition: Riiid Answer Correctness Prediction
%%capture
def tune_model(hp):
nn = Sequential()
nn.add(Dense(hp.Choice("units_1", [512, 1024, 2048]), activation = "relu", input_shape =(18,)))
 nn.add(Dropout(hp.Choice("dropout_1", [0.0, 0.15, 0.2, 0.25])))
 nn.add(Dense(hp.Choice("units_2", [128, 256, 512]), activation = "relu"))
nn.add(Dropout(hp.Choice("dropout_2", [0.0, 0.15, 0.2, 0.25])))
 nn.add(Dense(hp.Choice("units_3", [32, 64, 128]), activation = "relu"))
nn.add(Dropout(hp.Choice("dropout_3", [0.0, 0.15, 0.2, 0.25])))
 nn.add(Dense(hp.Choice("units_4", [8, 16, 32]), activation = "relu"))
nn.add(Dropout(hp.Choice("dropout_4", [0.0, 0.15, 0.2, 0.25])))
 nn.add(Dense(1, activation = "relu"))
lr = hp.Choice("learning_rate", [0.0001, 0.001, 0.01, 0.1])
 mm = hp.Choice("momentum", [0.0, 0.2, 0.4, 0.6, 0.8])
 nn.compile(optimizer = keras.optimizers.RMSprop(learning_rate = lr, momentum = mm),
loss = "binary_crossentropy",
metrics = ["accuracy"])
 return nn
tuner = RandomSearch(tune_model,
objective = "val_accuracy",
max_trials = 3)
 tuner.search(x = X_train,
y = Y_train,
verbose = 3,
epochs = 400,
batch_size = 25,
validation_data =(X_test, Y_test))<train_model>
import gc
import random
from tqdm import tqdm
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split
import seaborn as sns
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.nn.utils.rnn as rnn_utils
from torch.autograd import Variable
from torch.utils.data import Dataset, DataLoader
print("Best Params: " + str(tuner.oracle.get_best_trials(num_trials = 1)[0].hyperparameters.values))
nn = tuner.get_best_models() [0]
print("")
 print("Test score: " + str(nn.evaluate(X_test, Y_test)))
 print("")
 print("Train score: " + str(nn.evaluate(X_train, Y_train)) )<predict_on_test>
MAX_SEQ = 160
Y_predicted_nn = nn.predict(test.drop(["PassengerId"], axis = 1))
Y_predicted_nn =(Y_predicted_nn.ravel() > 0.5 ).astype(int )<import_modules>
%%time
dtype = {'timestamp':'int64',
'user_id':'int32' ,
'content_id':'int16',
'content_type_id':'int8',
'answered_correctly':'int8'}
train_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', usecols=[1, 2, 3, 4, 7], dtype=dtype)
 train_df.head()
from sklearn.metrics import roc_curve, roc_auc_score<find_best_params>
train_df = train_df[train_df.content_type_id == False]
train_df = train_df.sort_values(['timestamp'], ascending=True ).reset_index(drop = True )
log_fpr, log_tpr, log_treshholds = roc_curve(Y, Y_predicted_log)
 boost_fpr, boost_tpr, boost_treshholds = roc_curve(Y, Y_predicted_boost)
 svm_fpr, svm_tpr, svm_treshholds = roc_curve(Y, Y_predicted_svm)
 rf_fpr, rf_tpr, rf_treshholds = roc_curve(Y, Y_predicted_rf)
 xgb_fpr, xgb_tpr, xgb_treshholds = roc_curve(Y, Y_predicted_xgb)
 bag_fpr, bag_tpr, bag_treshholds = roc_curve(Y, Y_predicted_bag)
 xt_fpr, xt_tpr, xt_treshholds = roc_curve(Y, Y_predicted_xt)
 knn_fpr, knn_tpr, knn_treshholds = roc_curve(Y, Y_predicted_knn)
 auc_score_log = roc_auc_score(Y, Y_predicted_log)
 auc_score_boost = roc_auc_score(Y, Y_predicted_boost)
 auc_score_svm = roc_auc_score(Y, Y_predicted_svm)
 auc_score_rf = roc_auc_score(Y, Y_predicted_rf)
 auc_score_xgb = roc_auc_score(Y, Y_predicted_xgb)
 auc_score_bag = roc_auc_score(Y, Y_predicted_bag)
 auc_score_xt = roc_auc_score(Y, Y_predicted_xt)
 auc_score_knn = roc_auc_score(Y, Y_predicted_knn)
 plt.figure(figsize =(14,7))
plt.plot([0,1], [0,1])
 plt.plot(log_fpr, log_tpr, label = "Logistic Regression(AUC-Score: " + str(round(auc_score_log, 2)) + ")")
 plt.plot(boost_fpr, boost_tpr, label = "Gradient Boosting(AUC-Score: " + str(round(auc_score_boost, 2)) + ")")
 plt.plot(svm_fpr, svm_tpr, label = "SVM(AUC-Score: " + str(round(auc_score_svm, 2)) + ")")
 plt.plot(rf_fpr, rf_tpr, label = "Random Forest(AUC-Score: " + str(round(auc_score_rf, 2)) + ")")
 plt.plot(xgb_fpr, xgb_tpr, label = "XGBoost(AUC-Score: " + str(round(auc_score_xgb, 2)) + ")")
 plt.plot(bag_fpr, bag_tpr, label = "Bagging Classifier(AUC-Score: " + str(round(auc_score_bag, 2)) + ")")
 plt.plot(xt_fpr, xt_tpr, label = "Extra Trees Clasifier(AUC-Score: " + str(round(auc_score_xt, 2)) + ")")
 plt.plot(knn_fpr, knn_tpr, label = "K-Nearest-Neighbor(AUC-Score: " + str(round(auc_score_knn, 2)) + ")")
 plt.title("ROC-Curve")
 plt.xlabel("FPR")
 plt.ylabel("TPR")
 plt.legend()
plt.show()<choose_model_class>
skills = train_df["content_id"].unique()
n_skill = len(skills)
 print("number skills", len(skills))
stack = StackingClassifier(estimators = [("XGBoost", grid_xgb), 
("GradientBoosting", grid_boost), 
("RandomForest", grid_rf), 
("Logistic Regression", grid_log), 
("SVM", grid_svm), 
("Extra Trees Classifier", grid_xt), 
("KNeighborsClassifier", grid_knn), 
("CatBoost", grid_cat), 
("LightGBM", grid_gbm)], n_jobs = -1)
 cv = StratifiedKFold(n_splits = 5)
 stack = stack.fit(X, Y)
 stack_score = cross_val_score(stack, X, Y, cv = cv, n_jobs = -1)
 print("
print("Cross-Validated-Score: " + str(round(stack_score.mean() , 6)))
 print("
Y_predicted_stack = stack.predict(test.drop(["PassengerId"], axis = 1))<save_to_csv>
group = train_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
del train_df
gc.collect()
submission = pd.DataFrame({"PassengerId" : test.PassengerId,
"Survived" : Y_predicted_stack})
 submission.to_csv("submission_clf.csv", index = False )<save_to_csv>
random.seed(1)
submission_nn = pd.DataFrame({"PassengerId" : test.PassengerId,
"Survived" : Y_predicted_nn})
 submission_nn.to_csv("submission_nn.csv", index = False )<load_from_csv>
class SAKTDataset(Dataset):
def __init__(self, group, n_skill, max_seq=MAX_SEQ):
super(SAKTDataset, self ).__init__()
self.max_seq = max_seq
self.n_skill = n_skill
self.samples = group
self.user_ids = []
for user_id in group.index:
q, qa = group[user_id]
if len(q)< 2:
continue
self.user_ids.append(user_id)
 def __len__(self):
return len(self.user_ids)
 def __getitem__(self, index):
user_id = self.user_ids[index]
q_, qa_ = self.samples[user_id]
seq_len = len(q_)
 q = np.zeros(self.max_seq, dtype=int)
 qa = np.zeros(self.max_seq, dtype=int)
 if seq_len >= self.max_seq:
if random.random() >0.1:
start = random.randint(0,(seq_len-self.max_seq))
end = start + self.max_seq
q[:] = q_[start:end]
qa[:] = qa_[start:end]
else:
q[:] = q_[-self.max_seq:]
qa[:] = qa_[-self.max_seq:]
else:
if random.random() >0.1:
start = 0
end = random.randint(2,seq_len)
 seq_len = end - start
q[-seq_len:] = q_[0:seq_len]
qa[-seq_len:] = qa_[0:seq_len]
else:
q[-seq_len:] = q_
qa[-seq_len:] = qa_
target_id = q[1:]
label = qa[1:]
x = np.zeros(self.max_seq-1, dtype=int)
 x = q[:-1].copy()
x +=(qa[:-1] == 1)* self.n_skill
return x, target_id, label
train = pd.read_csv(".. /input/titanic/train.csv")
 test = pd.read_csv(".. /input/titanic/test.csv" )<train_model>
dataset = SAKTDataset(group, n_skill)
 dataloader = DataLoader(dataset, batch_size=2048, shuffle=True, num_workers=8)
 item = dataset.__getitem__(5)
data_size_train = train.memory_usage().sum() / 1024 / 1024
print("Data memory size: %.2f MB" % data_size_train )<train_model>
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128):
super(SAKTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.embedding = nn.Embedding(2*n_skill+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = x + pos_x
e = self.e_embedding(question_ids)
 x = x.permute(1, 0, 2)
 e = e.permute(1, 0, 2)
 att_mask = future_mask(x.size(0)).to(device)
 att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
 att_output = self.layer_normal(att_output + e)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
data_size_test = test.memory_usage().sum() / 1024 / 1024
print("Data memory size: %.2f MB" % data_size_test )<feature_engineering>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model = SAKTModel(n_skill, embed_dim=128)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3)
 criterion = nn.BCEWithLogitsLoss()
model.to(device)
 criterion.to(device )
train["NameTitle"] = train["Name"].str.split(", ", expand = True)[1].str.split(".", expand = True)[0]
test["NameTitle"] = test["Name"].str.split(", ", expand = True)[1].str.split(".", expand = True)[0]
train = train.drop("Name", axis = 1)
 test = test.drop("Name", axis = 1)
 print(train["NameTitle"].unique())
 print("------------------------" * 3)
 print(test["NameTitle"].unique() )<count_values>
def train_epoch(model, train_iterator, optim, criterion, device="cpu"):
model.train()
train_loss = []
num_corrects = 0
num_total = 0
labels = []
outs = []
tbar = tqdm(train_iterator)
 for item in tbar:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
label = item[2].to(device ).float()
optim.zero_grad()
output, atten_weight = model(x, target_id)
 loss = criterion(output, label)
 loss.backward()
optim.step()
train_loss.append(loss.item())
 output = output[:, -1]
label = label[:, -1]
pred =(torch.sigmoid(output)>= 0.5 ).long()
num_corrects +=(pred == label ).sum().item()
num_total += len(label)
 labels.extend(label.view(-1 ).data.cpu().numpy())
 outs.extend(output.view(-1 ).data.cpu().numpy())
 tbar.set_description('loss - {:.4f}'.format(loss))
acc = num_corrects / num_total
auc = roc_auc_score(labels, outs)
 loss = np.mean(train_loss)
 return loss, acc, auc
min_titles =(train["NameTitle"].value_counts() < 10)
 train["NameTitle"] = train["NameTitle"].apply(lambda x: "Misc" if min_titles.loc[x] == True else x)
 print(train["NameTitle"].unique() )<count_values>
epochs = 35
for epoch in range(epochs):
loss, acc, auc = train_epoch(model, dataloader, optimizer, criterion, device)
 print("epoch - {} train_loss - {:.2f} acc - {:.3f} auc - {:.3f}".format(epoch, loss, acc, auc))
min_titles =(test["NameTitle"].value_counts() < 10)
 test["NameTitle"] = test["NameTitle"].apply(lambda x: "Misc" if min_titles.loc[x] == True else x)
 print(test["NameTitle"].unique() )<feature_engineering>
torch.save(model.state_dict() , "SAKT-HDKIM.pt" )
train["FamilySize"] = train["SibSp"] + train["Parch"]
test["FamilySize"] = test["SibSp"] + test["Parch"]<feature_engineering>
del dataset
gc.collect()
train["AgeGroup"] = ""
train.loc[train["Age"] < 21, "AgeGroup"] = "under 21"
train.loc[train["Age"] >= 21, "AgeGroup"] = "21-65"
train.loc[train["Age"] > 65, "AgeGroup"] = "65+"
test["AgeGroup"] = ""
test.loc[test["Age"] < 21, "AgeGroup"] = "under 21"
test.loc[test["Age"] >= 21, "AgeGroup"] = "21-65"
test.loc[test["Age"] > 65, "AgeGroup"] = "65+"
train = train.drop("Age", axis = 1)
 test = test.drop("Age", axis = 1 )<feature_engineering>
env = riiideducation.make_env()
iter_test = env.iter_test()
<drop_column><EOS> 
model.eval()
prev_test_df = None
for(test_df, sample_prediction_df)in tqdm(iter_test):
if(prev_test_df is not None)&(psutil.virtual_memory().percent<90):
print(psutil.virtual_memory().percent)
 prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prev_test_df = prev_test_df[prev_test_df.content_type_id == False]
prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
prev_group_content = prev_group[prev_user_id][0]
prev_group_ac = prev_group[prev_user_id][1]
if prev_user_id in group.index:
group[prev_user_id] =(np.append(group[prev_user_id][0],prev_group_content),
np.append(group[prev_user_id][1],prev_group_ac))
else:
group[prev_user_id] =(prev_group_content,prev_group_ac)
 if len(group[prev_user_id][0])>MAX_SEQ:
new_group_content = group[prev_user_id][0][-MAX_SEQ:]
new_group_ac = group[prev_user_id][1][-MAX_SEQ:]
group[prev_user_id] =(new_group_content,new_group_ac)
 prev_test_df = test_df.copy()
test_df = test_df[test_df.content_type_id == False]
test_dataset = TestDataset(group, test_df, skills)
 test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)
 outs = []
for item in tqdm(test_dataloader):
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, att_weight = model(x, target_id)
 output = torch.sigmoid(output)
 output = output[:, -1]
outs.extend(output.view(-1 ).data.cpu().numpy())
 test_df['answered_correctly'] = outs
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )