Kaggle competition: Riiid Answer Correctness Prediction
merged_df['Sex'] = le.fit_transform(merged_df['Sex'])
 dummiesSex = pd.get_dummies(merged_df['Sex'], prefix = 'Sex')
 merged_df = pd.concat([merged_df, dummiesSex], axis=1)
 merged_df.head()<categorify>
import gc
import random
from tqdm import tqdm
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split
import seaborn as sns
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.nn.utils.rnn as rnn_utils
from torch.autograd import Variable
from torch.utils.data import Dataset, DataLoader
dummiesCabin = pd.get_dummies(merged_df['Cabin'], prefix = 'Cabin')
 merged_df = pd.concat([merged_df, dummiesCabin], axis=1)
 merged_df.head()<categorify>
MAX_SEQ = 100
merged_df['Embarked'] = le.fit_transform(merged_df['Embarked'])
 dummiesEmbarked = pd.get_dummies(merged_df['Embarked'], prefix = 'Embarked')
 merged_df = pd.concat([merged_df, dummiesEmbarked], axis=1)
 merged_df.head()<feature_engineering>
%%time
dtype = {'timestamp': 'int64', 'user_id': 'int32' ,'content_id': 'int16','content_type_id': 'int8','answered_correctly':'int8'}
train_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', usecols=[1, 2, 3,4,7], dtype=dtype)
 train_df.head()
merged_df['Family'] = merged_df['Family'].apply(lambda x: 'N' if x == 0 else('S' if x < 4 else 'L'))<categorify>
train_df = train_df[train_df.content_type_id == False]
train_df = train_df.sort_values(['timestamp'], ascending=True ).reset_index(drop = True )
merged_df['Family'] = le.fit_transform(merged_df['Family'])
 dummiesFamily = pd.get_dummies(merged_df['Family'], prefix = 'Family')
 merged_df = pd.concat([merged_df, dummiesFamily], axis=1)
 merged_df.head()<drop_column>
skills = train_df["content_id"].unique()
n_skill = len(skills)
 print("number skills", len(skills))
merged_df.drop(['Pclass', 'Sex', 'Fare', 'Cabin', 'Embarked', 'Title', 'Family'], axis = 1, inplace = True )<prepare_x_and_y>
group = train_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
del train_df
gc.collect()
train_df_x = merged_df[:891]
test_df_x = merged_df[891:]<prepare_x_and_y>
class SAKTDataset(Dataset):
def __init__(self, group, n_skill, max_seq=MAX_SEQ):
super(SAKTDataset, self ).__init__()
self.max_seq = max_seq
self.n_skill = n_skill
self.samples = group
self.user_ids = []
for user_id in group.index:
q, qa = group[user_id]
if len(q)< 5:
continue
self.user_ids.append(user_id)
 if len(q)>self.max_seq:
group[user_id] =(q[-self.max_seq:],qa[-self.max_seq:])
 def __len__(self):
return len(self.user_ids)
 def __getitem__(self, index):
user_id = self.user_ids[index]
q_, qa_ = self.samples[user_id]
seq_len = len(q_)
 q = np.zeros(self.max_seq, dtype=int)
 qa = np.zeros(self.max_seq, dtype=int)
 if seq_len >= self.max_seq:
q[:] = q_[-self.max_seq:]
qa[:] = qa_[-self.max_seq:]
else:
q[-seq_len:] = q_
qa[-seq_len:] = qa_
target_id = q[1:]
label = qa[1:]
x = np.zeros(self.max_seq-1, dtype=int)
 x = q[:-1].copy()
x +=(qa[:-1] == 1)* self.n_skill
return x, target_id, label
train_df_y = train_df['Survived']<drop_column>
dataset = SAKTDataset(group, n_skill)
 dataloader = DataLoader(dataset, batch_size=2048, shuffle=True, num_workers=8)
 item = dataset.__getitem__(5)
train_df = train_df_x.copy()
train_df['Survived'] = train_df_y
train_df.drop(['PassengerId'], axis = 1, inplace = True)
 train_df.head()<split>
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128):
super(SAKTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.embedding = nn.Embedding(2*n_skill+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = x + pos_x
e = self.e_embedding(question_ids)
 x = x.permute(1, 0, 2)
 e = e.permute(1, 0, 2)
 att_mask = future_mask(x.size(0)).to(device)
 att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
 att_output = self.layer_normal(att_output + e)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
def train_and_test(model_specific_tasks, df, it = 20):
accsum = 0
minacc = 1.0
maxacc = 0
for i in range(it):
print('Iteration: ',(i + 1), end = '\r')
 train, test = train_test_split(df, test_size=0.2)
 train_x = train.drop(['Survived'], axis=1)
 test_x = test.drop(['Survived'], axis=1)
 train_y = train['Survived']
test_y = test['Survived']
train_x = np.asarray(train_x ).astype('float32')
 train_y = np.asarray(train_y ).astype('float32')
 acc = model_specific_tasks(train_x, train_y, test_x, test_y)
 accsum += acc
minacc = acc if acc < minacc else minacc
maxacc = acc if acc > maxacc else maxacc
print('Avg.accuracy: ',(accsum / it))
print('Min.accuracy: ', minacc)
 print('Max.accuracy: ', maxacc )<train_model>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model = SAKTModel(n_skill, embed_dim=128)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3)
 criterion = nn.BCEWithLogitsLoss()
model.to(device)
 criterion.to(device )
def logistic_reg(train_x, train_y, test_x, test_y):
model = LogisticRegression(solver='sag', max_iter=1000)
 model.fit(train_x, train_y)
 return model.score(test_x, test_y )<train_model>
def train_epoch(model, train_iterator, optim, criterion, device="cpu"):
model.train()
train_loss = []
num_corrects = 0
num_total = 0
labels = []
outs = []
tbar = tqdm(train_iterator)
 for item in tbar:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
label = item[2].to(device ).float()
optim.zero_grad()
output, atten_weight = model(x, target_id)
 loss = criterion(output, label)
 loss.backward()
optim.step()
train_loss.append(loss.item())
 output = output[:, -1]
label = label[:, -1]
pred =(torch.sigmoid(output)>= 0.5 ).long()
num_corrects +=(pred == label ).sum().item()
num_total += len(label)
 labels.extend(label.view(-1 ).data.cpu().numpy())
 outs.extend(output.view(-1 ).data.cpu().numpy())
 tbar.set_description('loss - {:.4f}'.format(loss))
acc = num_corrects / num_total
auc = roc_auc_score(labels, outs)
 loss = np.mean(train_loss)
 return loss, acc, auc
train_and_test(logistic_reg, train_df.copy() , it = 50 )<train_model>
epochs = 20
for epoch in range(epochs):
loss, acc, auc = train_epoch(model, dataloader, optimizer, criterion, device)
 print("epoch - {} train_loss - {:.2f} acc - {:.3f} auc - {:.3f}".format(epoch, loss, acc, auc))
def rfc(train_x, train_y, test_x, test_y):
model = RandomForestClassifier(n_estimators=100)
 model.fit(train_x, train_y)
 return model.score(test_x, test_y )<train_model>
del dataset
gc.collect()
train_and_test(rfc, train_df.copy() )<train_model>
env = riiideducation.make_env()
iter_test = env.iter_test()
def nn(train_x, train_y, test_x, test_y):
model = Sequential()
model.add(layers.Dense(32, activation='relu', input_shape =(22,)))
 model.add(layers.Dropout(0.5))
model.add(layers.Dense(8, activation='relu'))
model.add(layers.Dense(1, activation='sigmoid'))
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])
 model.fit(train_x, train_y, epochs=150, batch_size=16, verbose = 0)
 return model.evaluate(test_x, test_y, verbose = 0)[1]<train_model>
model.eval()
prev_test_df = None
for(test_df, sample_prediction_df)in tqdm(iter_test):
if(prev_test_df is not None)&(psutil.virtual_memory().percent<90):
print(psutil.virtual_memory().percent)
 prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prev_test_df = prev_test_df[prev_test_df.content_type_id == False]
prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
prev_group_content = prev_group[prev_user_id][0]
prev_group_ac = prev_group[prev_user_id][1]
if prev_user_id in group.index:
group[prev_user_id] =(np.append(group[prev_user_id][0],prev_group_content),
np.append(group[prev_user_id][1],prev_group_ac))
else:
group[prev_user_id] =(prev_group_content,prev_group_ac)
 if len(group[prev_user_id][0])>MAX_SEQ:
new_group_content = group[prev_user_id][0][-MAX_SEQ:]
new_group_ac = group[prev_user_id][1][-MAX_SEQ:]
group[prev_user_id] =(new_group_content,new_group_ac)
 prev_test_df = test_df.copy()
test_df = test_df[test_df.content_type_id == False]
test_dataset = TestDataset(group, test_df, skills)
 test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)
 outs = []
for item in tqdm(test_dataloader):
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, att_weight = model(x, target_id)
 output = torch.sigmoid(output)
 output = output[:, -1]
outs.extend(output.view(-1 ).data.cpu().numpy())
 test_df['answered_correctly'] = outs
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )