Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<choose_model_class>
import gc
import psutil
import random
from tqdm.notebook import tqdm
import numpy as np
import pandas as pd
from sklearn.metrics import roc_auc_score
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
input = Input(shape=[28, 28, 1])
 x = Conv2D(32,(5, 5), strides=1, padding='same', name='conv1' )(input)
 x = BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer='uniform',name='batch1' )(x)
 x = Activation('relu', name='relu1' )(x)
 x = Conv2D(32,(5, 5), strides=1, padding='same', name='conv2' )(x)
 x = BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer='uniform',name='batch2' )(x)
 x = Activation('relu', name='relu2' )(x)
 x = Conv2D(32,(5, 5), strides=1, padding='same', name='conv2add' )(x)
 x = BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer='uniform',name='batch2add' )(x)
 x = Activation('relu', name='relu2add' )(x)
 x = Dropout(0.15 )(x)
 x = MaxPool2D(pool_size=2, strides=2, padding='same' )(x)
 x = Conv2D(64,(3, 3), strides=1, padding='same', name='conv3' )(x)
 x = BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer='uniform',name='batch3' )(x)
 x = Activation('relu', name='relu3' )(x)
 x = Conv2D(64,(3, 3), strides=1, padding='same', name='conv4' )(x)
 x = BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer='uniform',name='batch4' )(x)
 x = Activation('relu', name='relu4' )(x)
 x = Conv2D(32,(3, 3), strides=1, padding='same', name='conv5' )(x)
 x = BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer='uniform',name='batch5' )(x)
 x = Activation('relu', name='relu5' )(x)
 x = Dropout(0.15 )(x)
 x = MaxPool2D(pool_size=2, strides=2 )(x)
 x = Flatten()(x)
 x = Dense(100, name='Dense30' )(x)
 x = Activation('relu', name='relu6' )(x)
 x = Dropout(0.05 )(x)
 x = Dense(10, name='Dense10' )(x)
 x = Activation('softmax' )(x)
 model = Model(inputs = input, outputs =x)
 print(model.summary() )<init_hyperparams>
MAX_SEQ = 50
NUM_HEADS = 10
NUM_EMBED = 128
n_skill = 13523
checkpoint = ModelCheckpoint("best_weights.hdf5", monitor='val_acc', verbose=1, save_best_only=True, mode='max')
 datagen = ImageDataGenerator(
 rotation_range= 8,
zoom_range = 0.13,
width_shift_range=0.13,
height_shift_range=0.13)
 epochs = 60
lr_initial = 0.0011
optimizer = Adam(lr=lr_initial, decay= lr_initial /(epochs*1.3))
model.compile(optimizer=optimizer,
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
 datagen.fit(x_train)
 batch_size = 64
history = model.fit_generator(datagen.flow(x_train,y_train, batch_size=batch_size),
epochs = epochs, validation_data =(x_val,y_val),
verbose = 1, steps_per_epoch=x_train.shape[0] // batch_size, callbacks=[checkpoint])
 model.load_weights("best_weights.hdf5")
<save_to_csv>
%%time
dtypes = {'timestamp': 'int64',
'user_id': 'int32' ,
'content_id': 'int16',
'content_type_id': 'int8',
'answered_correctly':'int8'}
train_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', usecols=[1, 2, 3, 4, 7], dtype=dtypes)
results = model.predict(x_test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("MNIST-CNN-ENSEMBLE.csv",index=False )<import_modules>
train_df = train_df[train_df.content_type_id == False]
train_df = train_df.sort_values(['timestamp'], ascending=True ).reset_index(drop=True )
import os
from glob import glob
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import cv2 
<load_from_csv>
group = train_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda x:(
 x['content_id'].values,
x['answered_correctly'].values))
del train_df
gc.collect()
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 sub = pd.read_csv('.. /input/sample_submission.csv')
 train_dir = '.. /input/train_images'
test_dir = '.. /input/test_images'<train_model>
random.seed(1 )
print('Total images for train {0}'.format(len(os.listdir(train_dir))))
print('Total images for test {0}'.format(len(os.listdir(test_dir))))<feature_engineering>
class SAKTDataset(Dataset):
def __init__(self, group, n_skill, max_seq=MAX_SEQ):
super(SAKTDataset, self ).__init__()
self.max_seq = max_seq
self.n_skill = n_skill
self.samples = group
self.user_ids = []
for user_id in group.index:
q, qa = group[user_id]
if len(q)< 2:
continue
self.user_ids.append(user_id)
 def __len__(self):
return len(self.user_ids)
 def __getitem__(self, index):
user_id = self.user_ids[index]
q_, qa_ = self.samples[user_id]
seq_len = len(q_)
 q = np.zeros(self.max_seq, dtype=int)
 qa = np.zeros(self.max_seq, dtype=int)
 if seq_len >= self.max_seq:
if random.random() >0.1:
start = random.randint(0,(seq_len-self.max_seq))
end = start + self.max_seq
q[:] = q_[start:end]
qa[:] = qa_[start:end]
else:
q[:] = q_[-self.max_seq:]
qa[:] = qa_[-self.max_seq:]
else:
if random.random() >0.1:
start = 0
end = random.randint(2,seq_len)
 seq_len = end - start
q[-seq_len:] = q_[0:seq_len]
qa[-seq_len:] = qa_[0:seq_len]
else:
q[-seq_len:] = q_
qa[-seq_len:] = qa_
target_id = q[1:]
label = qa[1:]
x = np.zeros(self.max_seq-1, dtype=int)
 x = q[:-1].copy()
x +=(qa[:-1] == 1)* self.n_skill
return x, target_id, label
classes_wild = {0: 'empty', 1: 'deer', 2: 'moose', 3: 'squirrel', 4: 'rodent', 5: 'small_mammal', \
6: 'elk', 7: 'pronghorn_antelope', 8: 'rabbit', 9: 'bighorn_sheep', 10: 'fox', 11: 'coyote', \
12: 'black_bear', 13: 'raccoon', 14: 'skunk', 15: 'wolf', 16: 'bobcat', 17: 'cat',\
18: 'dog', 19: 'opossum', 20: 'bison', 21: 'mountain_goat', 22: 'mountain_lion'}
train_df['classes_wild'] = train_df['category_id'].apply(lambda cw: classes_wild[cw] )<count_values>
dataset = SAKTDataset(group, n_skill)
 dataloader = DataLoader(dataset, batch_size=2048, shuffle=True, num_workers=8)
 item = dataset.__getitem__(5)
train_df['classes_wild'].value_counts()<import_modules>
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128):
super(SAKTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.embedding = nn.Embedding(2*n_skill+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = x + pos_x
e = self.e_embedding(question_ids)
 x = x.permute(1, 0, 2)
 e = e.permute(1, 0, 2)
 att_mask = future_mask(x.size(0)).to(device)
 att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
 att_output = self.layer_normal(att_output + e)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
import torch
import torch.nn as nn
import torchvision
from torchvision.transforms import transforms
from torch.utils.data import Dataset, DataLoader
from torchvision import models
from sklearn.model_selection import train_test_split<drop_column>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model = SAKTModel(n_skill, embed_dim=128)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3)
 criterion = nn.BCEWithLogitsLoss()
model.to(device)
 criterion.to(device )
train_df = train_df[['file_name','category_id']]
train_df.head() 
<categorify>
def train_epoch(model, train_iterator, optim, criterion, device="cpu"):
model.train()
train_loss = []
num_corrects = 0
num_total = 0
labels = []
outs = []
tbar = tqdm(train_iterator)
 for item in tbar:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
label = item[2].to(device ).float()
optim.zero_grad()
output, atten_weight = model(x, target_id)
 loss = criterion(output, label)
 loss.backward()
optim.step()
train_loss.append(loss.item())
 output = output[:, -1]
label = label[:, -1]
pred =(torch.sigmoid(output)>= 0.5 ).long()
num_corrects +=(pred == label ).sum().item()
num_total += len(label)
 labels.extend(label.view(-1 ).data.cpu().numpy())
 outs.extend(output.view(-1 ).data.cpu().numpy())
 tbar.set_description('loss - {:.4f}'.format(loss))
acc = num_corrects / num_total
auc = roc_auc_score(labels, outs)
 loss = np.mean(train_loss)
 return loss, acc, auc
category = train_df['category_id'].unique()
encoder = dict([(v, k)for v, k in zip(category, range(len(category)))])
 decoder = dict([(v, k)for k, v in encoder.items() ])
 print(pd.DataFrame({
'Before encoding': list(encoder.keys()),
'After encoding': list(encoder.values())} ).to_string(index=False))
def encoding(labels):
return encoder[int(labels)]<count_values>
epochs = 35
for epoch in range(epochs):
loss, acc, auc = train_epoch(model, dataloader, optimizer, criterion, device)
 print("epoch - {} train_loss - {:.2f} acc - {:.3f} auc - {:.3f}".format(epoch, loss, acc, auc))
train_df['category_id'] = train_df['category_id'].apply(encoding)
 train_df['category_id'].value_counts()<categorify>
torch.save(model.state_dict() , "SAKT.pt")
 del dataset
gc.collect()
class WildDataset(Dataset):
def __init__(self, df, img_dir, transforms=None):
self.df = df
self.img_dir = img_dir
self.transforms = transforms
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
img_name = os.path.join(self.img_dir,
self.df.iloc[idx, 0])
 image = cv2.imread(img_name)
 label = self.df.iloc[idx, 1]
if self.transforms is not None:
image = self.transforms(image)
 return image, label<split>
env = riiideducation.make_env()
iter_test = env.iter_test()
train, val = train_test_split(train_df, stratify=train_df.category_id, test_size=0.1)
 len(train), len(val )<categorify>
model.eval()
prev_test_df = None
<create_dataframe><EOS> 
for(test_df, sample_prediction_df)in tqdm(iter_test):
if(prev_test_df is not None)&(psutil.virtual_memory().percent < 90):
prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prev_test_df = prev_test_df[prev_test_df.content_type_id == False]
prev_group = prev_test_df[['user_id',
'content_id',
'answered_correctly']]\ 
.groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
if prev_user_id in group.index:
group[prev_user_id] =(
 np.append(group[prev_user_id][0],
prev_group[prev_user_id][0])[-MAX_SEQ:],
np.append(group[prev_user_id][1],
prev_group[prev_user_id][1])[-MAX_SEQ:] 
)
 else:
group[prev_user_id] =(
 prev_group[prev_user_id][0],
prev_group[prev_user_id][1] 
)
 prev_test_df = test_df.copy()
test_df = test_df[test_df.content_type_id == False]
test_dataset = TestDataset(group, test_df, n_skill)
 test_dataloader = DataLoader(test_dataset, batch_size=len(test_df), shuffle=False)
 outs = []
for item in tqdm(test_dataloader):
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, att_weight = model(x, target_id)
 outs.extend(torch.sigmoid(output)[:, -1].view(-1 ).data.cpu().numpy())
 test_df['answered_correctly'] = outs
env.predict(test_df.loc[test_df['content_type_id'] == 0,
['row_id', 'answered_correctly']] )