Kaggle competition: Riiid Answer Correctness Prediction
opt = RMSprop(lr=0.001)
 model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'] )<import_modules>
import gc
import random
from tqdm import tqdm
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split
import seaborn as sns
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.nn.utils.rnn as rnn_utils
from torch.autograd import Variable
from torch.utils.data import Dataset, DataLoader
from keras.preprocessing.image import ImageDataGenerator,img_to_array,load_img
import itertools<train_model>
MAX_SEQ = 160
datagen = ImageDataGenerator(rotation_range=10, zoom_range = 0.1, width_shift_range=0.1, height_shift_range=0.1)
 datagen.fit(X_train)
 history = model.fit_generator(datagen.flow(X_train,y_train, batch_size=86),
epochs = 30, validation_data =(X_test,y_test),verbose = 2,
steps_per_epoch=X_train.shape[0]/86 )<predict_on_test>
%%time
dtype = {'timestamp':'int64',
'user_id':'int32' ,
'content_id':'int16',
'content_type_id':'int8',
'answered_correctly':'int8'}
train_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', usecols=[1, 2, 3, 4, 7], dtype=dtype)
 train_df.head()
pred = model.predict(test_data)
 pred<prepare_output>
train_df = train_df[train_df.content_type_id == False]
train_df = train_df.sort_values(['timestamp'], ascending=True ).reset_index(drop = True )
prediction = np.argmax(pred, axis = 1)
 prediction<load_from_csv>
skills = train_df["content_id"].unique()
n_skill = len(skills)
 print("number skills", len(skills))
submission = pd.read_csv('.. /input/digit-recognizer/sample_submission.csv')
 submission['Label'] = prediction
submission.head(10 )<save_to_csv>
group = train_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
del train_df
gc.collect()
submission.to_csv("submission.csv", index=False, header=True )<load_from_csv>
random.seed(1)
data_train = pd.read_csv('/kaggle/input/titanic/train.csv', sep = ',', header = 0)
 data_test = pd.read_csv('/kaggle/input/titanic/test.csv', sep = ',', header = 0)
 print(data_train)
 data_train.info()
print(data_test)
 data_test.info()<feature_engineering>
class SAKTDataset(Dataset):
def __init__(self, group, n_skill, max_seq=MAX_SEQ):
super(SAKTDataset, self ).__init__()
self.max_seq = max_seq
self.n_skill = n_skill
self.samples = group
self.user_ids = []
for user_id in group.index:
q, qa = group[user_id]
if len(q)< 2:
continue
self.user_ids.append(user_id)
 def __len__(self):
return len(self.user_ids)
 def __getitem__(self, index):
user_id = self.user_ids[index]
q_, qa_ = self.samples[user_id]
seq_len = len(q_)
 q = np.zeros(self.max_seq, dtype=int)
 qa = np.zeros(self.max_seq, dtype=int)
 if seq_len >= self.max_seq:
if random.random() >0.1:
start = random.randint(0,(seq_len-self.max_seq))
end = start + self.max_seq
q[:] = q_[start:end]
qa[:] = qa_[start:end]
else:
q[:] = q_[-self.max_seq:]
qa[:] = qa_[-self.max_seq:]
else:
if random.random() >0.1:
start = 0
end = random.randint(2,seq_len)
 seq_len = end - start
q[-seq_len:] = q_[0:seq_len]
qa[-seq_len:] = qa_[0:seq_len]
else:
q[-seq_len:] = q_
qa[-seq_len:] = qa_
target_id = q[1:]
label = qa[1:]
x = np.zeros(self.max_seq-1, dtype=int)
 x = q[:-1].copy()
x +=(qa[:-1] == 1)* self.n_skill
return x, target_id, label
combine1 = [data_train]
for data_train in combine1:
data_train['Salutation'] = data_train.Name.str.extract('([A-Za-z]+ ).', expand=False)
 for data_train in combine1:
data_train['Salutation'] = data_train['Salutation'].replace(['Lady', 'Countess', 'Capt', 'Col', 'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 data_train['Salutation'] = data_train['Salutation'].replace('Mlle', 'Miss')
 data_train['Salutation'] = data_train['Salutation'].replace('Ms', 'Miss')
 data_train['Salutation'] = data_train['Salutation'].replace('Mme', 'Mrs')
 combine2 = [data_test]
for data_test in combine2:
data_test['Salutation'] = data_test.Name.str.extract('([A-Za-z]+ ).', expand=False)
 for data_test in combine2:
data_test['Salutation'] = data_test['Salutation'].replace(['Lady', 'Countess', 'Capt', 'Col', 'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 data_test['Salutation'] = data_test['Salutation'].replace('Mlle', 'Miss')
 data_test['Salutation'] = data_test['Salutation'].replace('Ms', 'Miss')
 data_test['Salutation'] = data_test['Salutation'].replace('Mme', 'Mrs')
 Salutation_mapping = {'Mr':1, 'Miss':2, 'Mrs':3, 'Master':4, 'Rare':5}
for data_train in combine1:
data_train['Salutation'] = data_train['Salutation'].map(Salutation_mapping)
 data_train['Salutation'] = data_train['Salutation'].fillna(0)
 for data_test in combine2:
data_test['Salutation'] = data_test['Salutation'].map(Salutation_mapping)
 data_test['Salutation'] = data_test['Salutation'].fillna(0)
 Age_Salutation_dn1 = data_train[['Age', 'Salutation']].dropna()
Age_Salutation_dn2 = data_test[['Age', 'Salutation']].dropna()
Age_Salutation_dn1 = np.array(Age_Salutation_dn1)
 Age_Salutation_dn2 = np.array(Age_Salutation_dn2)
 t_age_Mr, t_age_Miss, t_age_Mrs, t_age_Master, t_age_Rare = 0, 0, 0, 0, 0
Mr_count, Miss_count, Mrs_count, Master_count, Rare_count = 0, 0, 0, 0, 0
for i in range(len(Age_Salutation_dn1)) :
age1 = Age_Salutation_dn1[i][0]
if Age_Salutation_dn1[i][1] == 1:
t_age_Mr += age1
Mr_count += 1
elif Age_Salutation_dn1[i][1] == 2:
t_age_Miss += age1
Miss_count += 1
elif Age_Salutation_dn1[i][1] == 3:
t_age_Mrs += age1
Mrs_count += 1
elif Age_Salutation_dn1[i][1] == 4:
t_age_Master += age1
Master_count += 1
else:
t_age_Rare += age1
Rare_count += 1
for i in range(len(Age_Salutation_dn2)) :
age2 = Age_Salutation_dn2[i][0]
if Age_Salutation_dn2[i][1] == 1:
t_age_Mr += age2
Mr_count += 1
elif Age_Salutation_dn2[i][1] == 2:
t_age_Miss += age2
Miss_count += 1
elif Age_Salutation_dn2[i][1] == 3:
t_age_Mrs += age2
Mrs_count += 1
elif Age_Salutation_dn2[i][1] == 4:
t_age_Master += age2
Master_count += 1
else:
t_age_Rare += age2
Rare_count += 1
m_age_Mr = t_age_Mr / Mr_count
m_age_Miss = t_age_Miss / Miss_count
m_age_Mrs = t_age_Mrs / Mrs_count
m_age_Master = t_age_Master / Master_count
m_age_Rare = t_age_Rare / Rare_count
print('Mr:', m_age_Mr)
 print('Miss:', m_age_Miss)
 print('Mrs:', m_age_Mrs)
 print('Master:', m_age_Master)
 print('Rare:', m_age_Rare )<create_dataframe>
dataset = SAKTDataset(group, n_skill)
 dataloader = DataLoader(dataset, batch_size=2048, shuffle=True, num_workers=8)
 item = dataset.__getitem__(5)
data_set = DataFrame(data_train.drop(['PassengerId', 'Name', 'Ticket'], axis=1))
data_set = data_set.fillna({'Fare':0, 'Cabin':0,'Embarked':0})
 data_set = data_set.replace({'male':0, 'female':1, 'S':1, 'C':2, 'Q':3})
 data_set = data_set.replace({'A':1, 'B':2, 'C':3, 'D':4, 'E':5, 'F':6, 'G':7, 'T':8}, regex=True)
 for i in range(len(data_set)) :
salutation = data_set['Salutation'][i]
if salutation == 1:
if pd.isnull(data_set['Age'][i]):
data_set['Age'].loc[i] = m_age_Mr
elif salutation == 2:
if pd.isnull(data_set['Age'][i]):
data_set['Age'].loc[i] = m_age_Miss
elif salutation == 3:
if pd.isnull(data_set['Age'][i]):
data_set['Age'].loc[i] = m_age_Mrs
elif salutation == 4:
if pd.isnull(data_set['Age'][i]):
data_set['Age'].loc[i] = m_age_Master
else:
if pd.isnull(data_set['Age'][i]):
data_set['Age'].loc[i] = m_age_Rare
data_set['FamilySize'] = data_set['SibSp'] + data_set['Parch'] + 1
isAlone = []
FamilySize_M = []
FamilySize_L = []
isCabinNo = []
roundAgeArray = []
roundFareArray = []
for i in range(len(data_set)) :
if data_set['FamilySize'][i] == 1:
isAlone.append(1)
 else:
isAlone.append(0)
 if data_set['Cabin'][i] == 0:
isCabinNo.append(1)
 else:
isCabinNo.append(0)
 if data_set['FamilySize'][i] < 2:
FamilySize_M.append(0)
 FamilySize_L.append(0)
 elif data_set['FamilySize'][i] < 5:
FamilySize_M.append(1)
 FamilySize_L.append(0)
 else:
FamilySize_M.append(0)
 FamilySize_L.append(1)
 roundAge = int(data_set['Age'][i] / 10)
 roundAgeArray.append(roundAge)
 if data_set['Fare'][i] > 0:
roundFare = int(math.log(data_set['Fare'][i]))
else:
roundFare = 0
roundFareArray.append(roundFare)
 data_set['isAlone'] = isAlone
data_set['FamilySize_M'] = FamilySize_M
data_set['FamilySize_L'] = FamilySize_L
data_set['isCabinNo'] = isCabinNo
data_set['roundAge'] = roundAgeArray
data_set['roundFare'] = roundFareArray
data_set.info() 
<data_type_conversions>
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128):
super(SAKTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.embedding = nn.Embedding(2*n_skill+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = x + pos_x
e = self.e_embedding(question_ids)
 x = x.permute(1, 0, 2)
 e = e.permute(1, 0, 2)
 att_mask = future_mask(x.size(0)).to(device)
 att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
 att_output = self.layer_normal(att_output + e)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
testData = np.array(data_set)
 testData = testData.astype('int32')
 for i in range(len(testData)) :
print(testData[i])
 data_set = data_set.dropna()
x = DataFrame(data_set[['Pclass', 'Sex', 'Age', 'Salutation', 'FamilySize', 'isAlone']])
 t = DataFrame(data_set['Survived'])
 x = np.array(x)
 t = np.array(t)
 t = t.ravel()
x = x.astype('float32')
 t = t.astype('int32')
 print('x shape:', x.shape)
 print(x[:10])
 print('t shape:', t.shape)
 print(t[:10] )<split>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model = SAKTModel(n_skill, embed_dim=128)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3)
 criterion = nn.BCEWithLogitsLoss()
model.to(device)
 criterion.to(device )
dataset = TupleDataset(x, t)
 train_val, innerTest = split_dataset_random(dataset, int(len(dataset)* 0.9), seed=0)
 train, valid = split_dataset_random(train_val, int(len(train_val)* 0.7), seed=0)
 train_iter = SerialIterator(train, batch_size=64, repeat=True, shuffle=True)
 print(dataset[0] )<normalization>
def train_epoch(model, train_iterator, optim, criterion, device="cpu"):
model.train()
train_loss = []
num_corrects = 0
num_total = 0
labels = []
outs = []
tbar = tqdm(train_iterator)
 for item in tbar:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
label = item[2].to(device ).float()
optim.zero_grad()
output, atten_weight = model(x, target_id)
 loss = criterion(output, label)
 loss.backward()
optim.step()
train_loss.append(loss.item())
 output = output[:, -1]
label = label[:, -1]
pred =(torch.sigmoid(output)>= 0.5 ).long()
num_corrects +=(pred == label ).sum().item()
num_total += len(label)
 labels.extend(label.view(-1 ).data.cpu().numpy())
 outs.extend(output.view(-1 ).data.cpu().numpy())
 tbar.set_description('loss - {:.4f}'.format(loss))
acc = num_corrects / num_total
auc = roc_auc_score(labels, outs)
 loss = np.mean(train_loss)
 return loss, acc, auc
class Net(chainer.Chain):
def __init__(self, n_in=6, n_hidden=100, n_out=2):
super().__init__()
with self.init_scope() :
self.l1 = L.Linear(n_in, n_hidden)
 self.l2 = L.Linear(n_hidden, n_hidden)
 self.l3 = L.Linear(n_hidden, n_out)
 def forward(self, x):
h = F.sigmoid(self.l1(x))
h = F.sigmoid(self.l2(h))
h = self.l3(h)
 return h
net = Net()
optimizer = optimizers.Adam(alpha=0.0001, beta1=0.9, beta2=0.999, eps=1e-08, eta=1.0,
weight_decay_rate=0.00001, amsgrad=False, adabound=False, final_lr=0.1, gamma=0.001)
 optimizer.setup(net)
 gpu_id = 0
n_epoch =2000
net.to_gpu(gpu_id)
 results_train, results_valid = {}, {}
results_train['loss'], results_train['accuracy'] = [], []
results_valid['loss'], results_valid['accuracy'] = [], []
train_iter.reset()
count = 1
for epoch in range(n_epoch):
while True:
train_batch = train_iter.next()
x_train, t_train = chainer.dataset.concat_examples(train_batch, gpu_id)
 y_train = net(x_train)
 loss_train = F.softmax_cross_entropy(y_train, t_train)
 acc_train = F.accuracy(y_train, t_train)
 net.cleargrads()
loss_train.backward()
optimizer.update()
count += 1
if train_iter.is_new_epoch:
with chainer.using_config('train', False), chainer.using_config('enable_backprop', False):
x_valid, t_valid = chainer.dataset.concat_examples(valid, gpu_id)
 y_valid = net(x_valid)
 loss_valid = F.softmax_cross_entropy(y_valid, t_valid)
 acc_valid = F.accuracy(y_valid, t_valid)
 loss_train.to_cpu()
loss_valid.to_cpu()
acc_train.to_cpu()
acc_valid.to_cpu()
if epoch % 10 == 0:
print('epoch: {}, iteration: {}, loss(train): {:.4f}, loss(valid): {:.4f}, acc(train): {:.4f}, acc(valid): {:.4f}'.format(epoch, count, loss_train.array.mean() , loss_valid.array.mean() , acc_train.array.mean() , acc_valid.array.mean()))
results_train['loss'].append(loss_train.array)
 results_train['accuracy'].append(acc_train.array)
 results_valid['loss'].append(loss_valid.array)
 results_valid['accuracy'].append(acc_valid.array)
 break 
<compute_train_metric>
epochs = 35
for epoch in range(epochs):
loss, acc, auc = train_epoch(model, dataloader, optimizer, criterion, device)
 print("epoch - {} train_loss - {:.2f} acc - {:.3f} auc - {:.3f}".format(epoch, loss, acc, auc))
x_test, t_test = chainer.dataset.concat_examples(innerTest, device=gpu_id)
 with chainer.using_config('train', False), chainer.using_config('enable_backprop', False):
y_test = net(x_test)
 loss_test = F.softmax_cross_entropy(y_test, t_test)
 acc_test = F.accuracy(y_test, t_test)
 print('test loss: {:.4f}'.format(loss_test.array.get()))
print('test accuracy: {:.4f}'.format(acc_test.array.get()))<set_options>
torch.save(model.state_dict() , "SAKT-HDKIM.pt" )
net.to_cpu()
chainer.serializers.save_npz('net.npz', net)
 !ls<drop_column>
del dataset
gc.collect()
test = DataFrame(data_test.drop(['PassengerId', 'Name', 'Ticket'], axis=1))
for i in range(len(test)) :
salutation = test['Salutation'][i]
if salutation == 1:
if pd.isnull(test['Age'][i]):
test['Age'].loc[i] = m_age_Mr
elif salutation == 2:
if pd.isnull(test['Age'][i]):
test['Age'].loc[i] = m_age_Miss
elif salutation == 3:
if pd.isnull(test['Age'][i]):
test['Age'].loc[i] = m_age_Mrs
elif salutation == 4:
if pd.isnull(test['Age'][i]):
test['Age'].loc[i] = m_age_Master
else:
if pd.isnull(test['Age'][i]):
test['Age'].loc[i] = m_age_Rare
test = test.fillna({'Fare':0, 'Cabin':0,'Embarked':0})
 test = test.replace({'male':0, 'female':1, 'S':1, 'C':2, 'Q':3})
 test = test.replace({'A':1, 'B':2, 'C':3, 'D':4, 'E':5, 'F':6, 'G':7, 'T':8}, regex=True)
 test['FamilySize'] = test['SibSp'] + test['Parch'] + 1
isAlone = []
FamilySize_M = []
FamilySize_L = []
isCabinNo = []
roundAgeArray = []
roundFareArray = []
for i in range(len(test)) :
if test['FamilySize'][i] == 1:
isAlone.append(1)
 else:
isAlone.append(0)
 if test['Cabin'][i] == 0:
isCabinNo.append(1)
 else:
isCabinNo.append(0)
 if test['FamilySize'][i] < 2:
FamilySize_M.append(0)
 FamilySize_L.append(0)
 elif test['FamilySize'][i] < 5:
FamilySize_M.append(1)
 FamilySize_L.append(0)
 else:
FamilySize_M.append(0)
 FamilySize_L.append(1)
 roundAge = int(test['Age'][i] / 10)
 roundAgeArray.append(roundAge)
 if test['Fare'][i] > 0:
roundFare = int(math.log(test['Fare'][i]))
else:
roundFare = 0
roundFareArray.append(roundFare)
 test['isAlone'] = isAlone
test['FamilySize_M'] = FamilySize_M
test['FamilySize_L'] = FamilySize_L
test['isCabinNo'] = isCabinNo
test['roundAge'] = roundAgeArray
test['roundFare'] = roundFareArray
test = DataFrame(test[['Pclass', 'Sex', 'Age', 'Salutation', 'FamilySize', 'isAlone']])
 test.info()<normalization>
env = riiideducation.make_env()
iter_test = env.iter_test()
<set_options><EOS> 
model.eval()
prev_test_df = None
for(test_df, sample_prediction_df)in tqdm(iter_test):
if(prev_test_df is not None)&(psutil.virtual_memory().percent<90):
print(psutil.virtual_memory().percent)
 prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prev_test_df = prev_test_df[prev_test_df.content_type_id == False]
prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
prev_group_content = prev_group[prev_user_id][0]
prev_group_ac = prev_group[prev_user_id][1]
if prev_user_id in group.index:
group[prev_user_id] =(np.append(group[prev_user_id][0],prev_group_content),
np.append(group[prev_user_id][1],prev_group_ac))
else:
group[prev_user_id] =(prev_group_content,prev_group_ac)
 if len(group[prev_user_id][0])>MAX_SEQ:
new_group_content = group[prev_user_id][0][-MAX_SEQ:]
new_group_ac = group[prev_user_id][1][-MAX_SEQ:]
group[prev_user_id] =(new_group_content,new_group_ac)
 prev_test_df = test_df.copy()
test_df = test_df[test_df.content_type_id == False]
test_dataset = TestDataset(group, test_df, skills)
 test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)
 outs = []
for item in tqdm(test_dataloader):
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, att_weight = model(x, target_id)
 output = torch.sigmoid(output)
 output = output[:, -1]
outs.extend(output.view(-1 ).data.cpu().numpy())
 test_df['answered_correctly'] = outs
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )