Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<init_hyperparams>
import pickle
import pandas as pd
import numpy as np
import gc
from sklearn.metrics import roc_auc_score
from collections import defaultdict, deque
from tqdm.notebook import tqdm
import lightgbm as lgb
num_epochs = 2
num_classes = 14
learning_rate = 0.02
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu' )<define_search_model>
train_pickle = '.. /input/riiid-cross-validation-files/cv1_train.pickle'
valid_pickle = '.. /input/riiid-cross-validation-files/cv1_valid.pickle'
question_file = '.. /input/riiid-test-answer-prediction/questions.csv'
def conv3x3(in_channels, out_channels, stride=1):
return nn.Conv2d(in_channels, out_channels, kernel_size=3,
stride=stride, padding=1, bias=False)
 class ResidualBlock(nn.Module):
def __init__(self, in_channels, out_channels, stride=1, downsample=None):
super(ResidualBlock, self ).__init__()
self.conv1 = conv3x3(in_channels, out_channels, stride)
 self.bn1 = nn.BatchNorm2d(out_channels)
 self.relu = nn.ReLU(inplace=True)
 self.conv2 = conv3x3(out_channels, out_channels)
 self.bn2 = nn.BatchNorm2d(out_channels)
 self.downsample = downsample
def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 if self.downsample:
residual = self.downsample(x)
 out += residual
out = self.relu(out)
 return out
class ResNet(nn.Module):
def __init__(self, block, layers, num_classes=14):
super(ResNet, self ).__init__()
self.in_channels = 16
self.conv = conv3x3(3, 16)
 self.bn = nn.BatchNorm2d(16)
 self.relu = nn.LeakyReLU(inplace=True)
 self.layer1 = self.make_layer(block, 16, layers[0])
 self.layer2 = self.make_layer(block, 32, layers[1], 2)
 self.layer3 = self.make_layer(block, 64, layers[2], 2)
 self.layer4 = self.make_layer(block, 128, layers[3], 2)
 self.avg_pool = nn.AdaptiveAvgPool2d(4)
 self.fc = nn.Linear(128, num_classes)
 def make_layer(self, block, out_channels, blocks, stride=1):
downsample = None
if(stride != 1)or(self.in_channels != out_channels):
downsample = nn.Sequential(
 conv3x3(self.in_channels, out_channels, stride=stride),
nn.BatchNorm2d(out_channels))
layers = []
layers.append(block(self.in_channels, out_channels, stride, downsample))
self.in_channels = out_channels
for i in range(1, blocks):
layers.append(block(out_channels, out_channels))
return nn.Sequential(*layers)
 def forward(self, x):
out = self.conv(x)
 out = self.bn(out)
 out = self.relu(out)
 out = self.layer1(out)
 out = self.layer2(out)
 out = self.layer3(out)
 out = self.layer4(out)
 out = self.avg_pool(out)
 out = out.view(out.size(0), -1)
 out = self.fc(out)
 return out
def create_resnet_model(output_dim: int = 1)-> nn.Module:
model = ResNet(ResidualBlock, [2, 2, 2, 2])
 in_features = model.fc.in_features
model.avg_pool = nn.AdaptiveAvgPool2d(1)
 model.fc = nn.Linear(in_features, output_dim)
 model = model.to(device)
 return model
model = create_resnet_model(output_dim=num_classes)
 criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adamax(model.parameters() , lr=learning_rate )<train_model>
last_time_u_dict = defaultdict(int)
 answered_correctly_sum_u_dict = defaultdict(int)
 count_u_dict = defaultdict(int)
 past_question = defaultdict(list)
 past_time_diff = defaultdict(list)
 past_prior_elaps = defaultdict(list)
 past_answer = defaultdict(list )
total_step = len(train_loader)
 for epoch in range(num_epochs):
for i,(images, labels)in enumerate(train_loader):
images = images.to(device)
 labels = labels.to(device)
 outputs = model(images)
 loss = criterion(outputs, labels)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
if(i+1)% 100 == 0:
print("Epoch [{}/{}], Step [{}/{}] Loss: {:.4f}" 
.format(epoch+1, num_epochs, i+1, total_step, loss.item())) 
<train_model>
def add_user_feats(df,last_time_u_dict, answered_correctly_sum_u_dict, count_u_dict):
acsu = np.zeros(len(df), dtype=np.int32)
 cu = np.zeros(len(df), dtype=np.int32)
 td = np.zeros(len(df), dtype=np.int64)
 for cnt,row in enumerate(tqdm(df[['user_id','answered_correctly','timestamp']].values)) :
acsu[cnt] = answered_correctly_sum_u_dict[row[0]]
cu[cnt] = count_u_dict[row[0]]
td[cnt] = row[2] - last_time_u_dict[row[0]]
if row[2] == last_time_u_dict[row[0]]:
td[cnt] = td[cnt-1]
answered_correctly_sum_u_dict[row[0]] += row[1]
count_u_dict[row[0]] += 1
last_time_u_dict[row[0]] = row[2]
user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu, 'time_diff':td})
 user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']
df = pd.concat([df, user_feats_df], axis=1)
 return df
model.eval()
with torch.no_grad() :
correct = 0
total = 0
for images, labels in val_loader:
images = images.to(device)
 labels = labels.to(device)
 outputs = model(images)
 _, predicted = torch.max(outputs.data, 1)
 total += labels.size(0)
 correct +=(predicted == labels ).sum().item()
print('Accuracy of the model on the 19630 test images: {} %'.format(100 * correct / total)) 
<load_from_csv>
def add_user_feats2(df, past_question, past_answer, past_prior_elaps, past_time_diff):
for _, row in enumerate(tqdm(df[['user_id','answered_correctly','content_id','prior_question_elapsed_time','time_diff']].values)) :
if row[0] not in past_question:
new_list = deque([0]*30, maxlen = 30)
 new_list.append(row[2])
 past_question[row[0]] = new_list
else:
past_question[row[0]].append(row[2])
 if row[0] not in past_answer:
new_list = deque([0]*30, maxlen = 30)
 new_list.append(row[1])
 past_answer[row[0]] = new_list
else:
past_answer[row[0]].append(row[1])
 if row[0] not in past_prior_elaps:
new_list = deque([0]*30, maxlen = 30)
 new_list.append(row[3])
 past_prior_elaps[row[0]] = new_list
else:
past_prior_elaps[row[0]].append(row[3])
 if row[0] not in past_time_diff:
new_list = deque([0]*30, maxlen = 30)
 new_list.append(row[4])
 past_time_diff[row[0]] = new_list
else:
past_time_diff[row[0]].append(row[4] )
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['Id'] = sub['Id'] + '.jpg'
sub.head()<create_dataframe>
def add_user_feats_without_update(df ,last_time_u_dict, answered_correctly_sum_u_dict, count_u_dict):
acsu = np.zeros(len(df), dtype=np.int32)
 cu = np.zeros(len(df), dtype=np.int32)
 td = np.zeros(len(df), dtype=np.int64)
 for cnt, row in enumerate(df[['user_id','part','timestamp']].values):
td[cnt] = row[2] - last_time_u_dict[row[0]]
acsu[cnt] = answered_correctly_sum_u_dict[row[0]]
cu[cnt] = count_u_dict[row[0]]
user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu, 'time_diff':td})
 user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']
df = pd.concat([df, user_feats_df], axis=1)
 return df
dataset_valid = WildDataset(df=sub,
img_dir=test_dir,
transforms=aug)
 test_loader = DataLoader(dataset_valid, batch_size=24, shuffle=False )<predict_on_test>
def get_user_feats_for_nn_without_update(df, past_question, past_answer, past_prior_elaps, past_time_diff):
current_question = []
past_question_answer = []
past_answer_correctly = []
past_time = []
past_prior = []
for cnt,row in enumerate(tqdm(df[['user_id','content_id','prior_question_elapsed_time','time_diff']].values)) :
if row[0] not in past_answer:
temp_answer = [0]*30
temp_past_answer = [0]*30
else:
temp_answer = past_answer[row[0]].copy()
temp_past_answer = past_answer[row[0]].copy()
if row[0] not in past_question:
temp_question = [0]*30
else:
temp_question = past_question[row[0]].copy()
temp_past_answer= [x+1 if y > 0 else 0 for x , y in zip(temp_past_answer, temp_question)]
past_answer_correctly.append(temp_past_answer)
 temp_past_question_answer = [x+y*13523 for x,y in zip(temp_question, temp_answer)]
past_question_answer.append(temp_past_question_answer)
 temp_question.append(row[1]+1)
 current_question.append([temp_question[i] for i in range(30)])
 if row[0] not in past_prior_elaps:
temp_elaps = [0]*29 + [row[2]/3e5]
else:
temp_elaps = past_prior_elaps[row[0]].copy()
temp_elaps.append(row[2]/3e5)
 past_prior.append(temp_elaps)
 if row[0] not in past_time_diff:
temp_time_diff = [0]*29 + [row[3]/1e6]
else:
temp_time_diff = past_time_diff[row[0]].copy()
temp_time_diff.append(row[3]/1e6)
 past_time.append(temp_time_diff)
 current_question = np.array(current_question)
 past_question_answer = np.array(past_question_answer)
 past_other_feats = np.dstack(( past_prior,past_time))
past_answer_correctly = np.array(past_answer_correctly)
 return current_question, past_question_answer, past_other_feats, past_answer_correctly
model.eval()
preds = []
for images, labels in test_loader:
images = images.to(device)
 labels = labels.to(device)
 outputs = model(images)
 _, predicted = torch.max(outputs.data, 1)
 predicted
for i in predicted.detach().cpu().numpy() :
preds.append(i)
<prepare_output>
def update_user_feats(df, last_time_u_dict, answered_correctly_sum_u_dict, count_u_dict):
for row in df[['user_id', 'answered_correctly', 'content_type_id', 'timestamp', 'content_id','prior_question_elapsed_time','time_diff']].values:
if row[2] == 0:
if row[0] not in past_question:
new_list = deque([0]*30, maxlen = 30)
 new_list.append(row[4]+1)
 past_question[row[0]] = new_list
else:
past_question[row[0]].append(row[4]+1)
 if row[0] not in past_answer:
new_list = deque([0]*30, maxlen = 30)
 new_list.append(row[1])
 past_answer[row[0]] = new_list
else:
past_answer[row[0]].append(row[1])
 if row[0] not in past_prior_elaps:
new_list = deque([0]*30, maxlen = 30)
 new_list.append(row[5]/3e5)
 past_prior_elaps[row[0]] = new_list
else:
past_prior_elaps[row[0]].append(row[5]/3e5)
 if row[0] not in past_time_diff:
new_list = deque([0]*30, maxlen = 30)
 if row[6] >= 1e6:
new_list.append(0)
 else:
new_list.append(row[6]/1e6)
 past_time_diff[row[0]] = new_list
else:
if row[6] >= 1e6:
past_time_diff[row[0]].append(0)
 else:
past_time_diff[row[0]].append(row[6]/1e6)
 last_time_u_dict[row[0]] = row[3]
answered_correctly_sum_u_dict[row[0]] += row[1]
count_u_dict[row[0]] += 1
sub['Predicted'] = preds
sub['Id'] = sub['Id'].str[:-4]
sub.head() 
<categorify>
feld_needed = ['user_id','content_id','answered_correctly','timestamp','prior_question_elapsed_time','prior_question_had_explanation']
train = pd.read_pickle(train_pickle)[feld_needed]
valid = pd.read_pickle(valid_pickle)[feld_needed]
def decoding(labels):
return decoder[int(labels)]<save_to_csv>
train = train.loc[train.answered_correctly != -1].reset_index(drop=True)
 valid = valid.loc[valid.answered_correctly != -1].reset_index(drop=True)
 _=gc.collect()
sub['Predicted'] = sub['Predicted'].apply(decoding)
 sub.head()
sub.to_csv('submission.csv', index=False )<count_values>
content_df = train[['content_id','answered_correctly']].groupby(['content_id'] ).agg(['mean','std'] ).reset_index()
content_df.columns = ['content_id', 'answered_correctly_avg_c','answered_correctly_std_c']
sub['Predicted'].value_counts()<import_modules>
train = add_user_feats(train, last_time_u_dict, answered_correctly_sum_u_dict, count_u_dict)
 valid = add_user_feats(valid, last_time_u_dict, answered_correctly_sum_u_dict, count_u_dict )
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt 
<load_from_csv>
questions_df = pd.read_csv(question_file)
 questions_df.tags.fillna('-1-1', inplace = True)
 questions_df['tag'] = pd.factorize(questions_df.tags)[0]
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv' )<create_dataframe>
last_time_u_dict = defaultdict(int)
 answered_correctly_sum_u_dict = defaultdict(int)
 count_u_dict = defaultdict(int)
 train1 = add_user_feats(train1, last_time_u_dict, answered_correctly_sum_u_dict, count_u_dict)
 valid1 = add_user_feats(valid1, last_time_u_dict, answered_correctly_sum_u_dict, count_u_dict)
 train1 = pd.merge(train1, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')
 valid1 = pd.merge(valid1, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')
 train1.time_diff[train1.time_diff >= 1e6] = 1e6
valid1.time_diff[valid1.time_diff >= 1e6] = 1e6
train1.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean, inplace = True)
 valid1.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean, inplace = True)
 add_user_feats2(train1, past_question, past_answer, past_prior_elaps, past_time_diff)
 add_user_feats2(valid1, past_question, past_answer, past_prior_elaps, past_time_diff)
 del(train1)
 del(valid1)
 _=gc.collect()
pd.pivot_table(train, index = 'Pclass', values = ['Survived'] )<count_values>
train = pd.merge(train, content_df, on=['content_id'], how="left")
 valid = pd.merge(valid, content_df, on=['content_id'], how="left" )
train['Title'] = train.Name.apply(lambda x: x.split(',')[1].split('.')[0])
 train['Family'] = train.Name.apply(lambda x: x.split(',')[0])
 print('Title values')
 print(train.Title.value_counts())
 print('Family values')
 print(train.Family.value_counts() )<feature_engineering>
train = pd.merge(train, questions_df[['question_id', 'part','tag']], left_on = 'content_id', right_on = 'question_id', how = 'left')
 valid = pd.merge(valid, questions_df[['question_id', 'part','tag']], left_on = 'content_id', right_on = 'question_id', how = 'left' )
test['Title'] = test.Name.apply(lambda x: x.split(',')[1].split('.')[0])
 test['Title'] = test.Title.apply(lambda x: x.replace('Mlle','Miss' ).replace('Mme','Mrs' ).replace('Col','Others' ).replace('Major','Others' ).replace('Don','Others' ).replace('Lady','Others' ).replace('Sir','Others' ).replace('Capt','Others' ).replace('the Countess','Others' ).replace('Ms','Others' ).replace('Jonkheer','Others'))
test.head()<drop_column>
train['prior_question_elapsed_time'] = train.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 valid['prior_question_elapsed_time'] = valid.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 train['prior_question_had_explanation'] = train.prior_question_had_explanation.fillna(False ).astype('int8')
 valid['prior_question_had_explanation'] = valid.prior_question_had_explanation.fillna(False ).astype('int8' )
train = train.drop('Name',axis = 1)
 test = test.drop('Name',axis = 1 )<groupby>
TARGET = 'answered_correctly'
FEATS = ['answered_correctly_avg_u', 'answered_correctly_avg_c','answered_correctly_std_c','answered_correctly_sum_u','count_u', 'part','tag', 'prior_question_elapsed_time','time_diff', 'prior_question_had_explanation']
dro_cols = list(set(train.columns)- set(FEATS))
y_tr = train[TARGET]
y_va = valid[TARGET]
_=gc.collect()
ageMean = train.groupby('Title')['Age'].median()
meanDr = ageMean[0]
meanMaster = ageMean[1]
meanMiss = ageMean[2]
meanMr = ageMean[3]
meanMrs = ageMean[4]
meanOthers = ageMean[5]
meanRev = ageMean[6]
ageMean 
<categorify>
lgb_train = lgb.Dataset(train[FEATS], y_tr, categorical_feature = ['part', 'prior_question_had_explanation', 'tag'],free_raw_data=False)
 lgb_valid = lgb.Dataset(valid[FEATS], y_va, categorical_feature = ['part', 'prior_question_had_explanation', 'tag'], reference=lgb_train,free_raw_data=False)
 _=gc.collect()
for x in range(len(train["Age"])) :
if pd.isna(train["Age"][x]):
title = train["Title"][x]
title = title.strip()
if(title == 'Dr'):
train["Age"][x] = meanDr
elif(title == "Master"):
train["Age"][x] = meanMaster
elif(title == "Miss"):
train["Age"][x] = meanMiss
elif(title =="Mr"):
train["Age"][x] = meanMr
elif(title == "Mrs"):
train["Age"][x] = meanMrs
elif(title == "Others"):
train["Age"][x] = meanOthers
elif(title == "Rev"):
train["Age"][x] = meanRev
for x in range(len(test["Age"])) :
if pd.isna(test["Age"][x]):
title = test["Title"][x]
title = title.strip()
if(title == 'Dr'):
test["Age"][x] = meanDr
elif(title == "Master"):
test["Age"][x] = meanMaster
elif(title == "Miss"):
test["Age"][x] = meanMiss
elif(title =="Mr"):
test["Age"][x] = meanMr
elif(title == "Mrs"):
test["Age"][x] = meanMrs
elif(title == "Others"):
test["Age"][x] = meanOthers
elif(title == "Rev"):
test["Age"][x] = meanRev 
<feature_engineering>
import optuna
test['AgeGroup'] = test.Age.apply(lambda x: 'Baby' if x <= 5 else('Child' if x <= 12 else('Teenager' if x <= 18 else('Young')if x <= 24 else('Young Adult' if x <= 35 else('Adult' if x <= 60 else 'Senior')))))
<define_variables>
def objective(trial):
params = {
'num_leaves': trial.suggest_int('num_leaves', 32, 512),
'max_bin': trial.suggest_int('max_bin', 700, 900),
'boosting' : 'gbdt',
'objective': 'binary',
'metric': 'auc',
'max_depth': trial.suggest_int('max_depth', 4, 16),
'min_child_weight': trial.suggest_int('min_child_weight', 1, 16),
'feature_fraction': trial.suggest_uniform('feature_fraction', 0.4, 1.0),
'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.4, 1.0),
'bagging_freq': trial.suggest_int('bagging_freq', 1, 8),
'min_child_samples': trial.suggest_int('min_child_samples', 4, 80),
'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 1.0),
'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 1.0),
'early_stopping_rounds': 5
}
model = lgb.train(params, lgb_train, valid_sets=[lgb_train, lgb_valid], verbose_eval=1000, num_boost_round=10000)
 val_pred = model.predict(valid[FEATS])
 score = roc_auc_score(y_va, val_pred)
 print(f"AUC = {score}")
 return score
train.Ticket<create_dataframe>
study = optuna.create_study(direction='maximize')
 study.optimize(objective, n_trials=1 )
train['NumericTicket'] = train.Ticket.apply(lambda x: 1 if x.isnumeric() else 0)
 pd.pivot_table(train, index = 'NumericTicket', values = ['Survived'] )<drop_column>
print('Number of finished trials: {}'.format(len(study.trials)))
 print('Best trial:')
 trial = study.best_trial
print(' Value: {}'.format(trial.value))
print(' Params: ')
 for key, value in trial.params.items() :
print(' {}: {}'.format(key, value))
train = train.drop('NumericTicket',axis=1)
 train = train.drop('Ticket',axis=1)
 test = test.drop('Ticket',axis=1 )<feature_engineering>
parameters = trial.params
parameters['objective'] = 'binary'
parameters['metric'] = 'auc'
parameters['early_stopping_rounds'] = 5
parameters['boosting'] = 'gbdt'
train['has_cabin'] = train.Cabin.apply(lambda x: 0 if pd.isna(x)else 1)
 train['CabinLetter'] = train.Cabin.apply(lambda x: str(x)[0])
 train.head()<drop_column>
import tensorflow as tf
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.layers import Dense, Dropout, LSTM, Masking, Embedding, Concatenate, Input
from tensorflow.keras.layers import concatenate
from tensorflow.keras.metrics import AUC
from tensorflow.keras import backend as K
from tensorflow.keras.layers import Lambda
from tensorflow.keras.layers import multiply, Reshape
from tensorflow.keras.utils import Sequence
test['has_cabin'] = test.Cabin.apply(lambda x: 0 if pd.isna(x)else 1)
 test['CabinLetter'] = test.Cabin.apply(lambda x: str(x)[0])
 train = train.drop('Cabin',axis=1)
 test = test.drop('Cabin',axis=1 )<categorify>
train.time_diff[train.time_diff >= 1e6] = 1e6
valid.time_diff[valid.time_diff >= 1e6] = 1e6
train.prior_question_elapsed_time = train.prior_question_elapsed_time/3e5
train.time_diff = train.time_diff/1e6
valid.prior_question_elapsed_time = valid.prior_question_elapsed_time/3e5
valid.time_diff = valid.time_diff/1e6
train = train.fillna({"Embarked": "S"} )<categorify>
test = test.fillna({"Fare": train.Fare.mean() })
<drop_column>
max_len = 31
train = train.drop("Age",axis=1)
 test = test.drop("Age",axis=1 )<data_type_conversions>
train_group = train.groupby('user_id')
 train_y = [frame['answered_correctly'].to_numpy() [:, None].tolist() [i:i+max_len] for _, frame in train_group
for i in range(0, len(frame['answered_correctly'].to_numpy() [:, None]), max_len)]
train_y = np.reshape(pad_sequences(train_y, padding="pre"),(-1,max_len))
train_past_answer = [(frame['answered_correctly']+1 ).to_numpy() [:, None].tolist() [i:i+max_len] for _, frame in train_group
for i in range(0, len(frame['answered_correctly'].to_numpy() [:, None]), max_len)]
train_past_answer = np.reshape(pad_sequences(train_past_answer, padding="pre"),(-1,max_len))
train_current_question = [(frame['content_id']+1 ).to_numpy() [:, None].tolist() [i:i+max_len] for _, frame in train_group
for i in range(0, len(frame['content_id'].to_numpy() [:,None]), max_len)]
train_current_question = np.reshape(pad_sequences(train_current_question, padding="pre"),(-1,max_len))
train_prior_elaps_time = [frame['prior_question_elapsed_time'].to_numpy() [:, None].tolist() [i:i+max_len] for _, frame in train_group
for i in range(0, len(frame['prior_question_elapsed_time'].to_numpy() [:,None]), max_len)]
train_prior_elaps_time = np.reshape(pad_sequences(train_prior_elaps_time, padding="pre", dtype = 'float32'),(-1,max_len))
train_time_diff = [frame['time_diff'].to_numpy() [:, None].tolist() [i:i+max_len] for _, frame in train_group
for i in range(0, len(frame['time_diff'].to_numpy() [:,None]), max_len)]
train_time_diff = np.reshape(pad_sequences(train_time_diff, padding="pre", dtype = 'float32'),(-1,max_len))
to_cat = ['Pclass','SibSp','Parch','has_cabin','Sex','Embarked','Title','AgeGroup','CabinLetter']
for x in to_cat:
train[x] = train[x].astype('category')
 test[x] = test[x].astype('category')
<categorify>
train_answered_correcly = train_y[:,1:]
train_past_answered_correctly = train_past_answer[:,:-1]
train_past_question_answer = train_current_question[:,:-1] + train_y[:,:-1]*(train.content_id.max() +1)
 train_current_question = train_current_question[:,1:]
train_prior_elaps_time = train_prior_elaps_time[:,1:]
train_time_diff = train_time_diff[:,1:]
sh_train = train.shape
c1 = pd.concat(( train, test), sort=False ).reset_index(drop=True)
 c = pd.get_dummies(c1)
 train = c[:sh_train[0]]
test = c[sh_train[0]:] 
<normalization>
valid_group = valid.groupby('user_id')
 valid_y = [frame['answered_correctly'].to_numpy() [:, None].tolist() [i:i+max_len] for _, frame in valid_group
for i in range(0, len(frame['answered_correctly'].to_numpy() [:, None]), max_len)]
valid_y = np.reshape(pad_sequences(valid_y, padding="pre"),(-1,max_len))
valid_past_answer = [(frame['answered_correctly']+1 ).to_numpy() [:, None].tolist() [i:i+max_len] for _, frame in valid_group
for i in range(0, len(frame['answered_correctly'].to_numpy() [:, None]), max_len)]
valid_past_answer = np.reshape(pad_sequences(valid_past_answer, padding="pre"),(-1,max_len))
valid_current_question = [(frame['content_id']+1 ).to_numpy() [:, None].tolist() [i:i+max_len] for _, frame in valid_group
for i in range(0, len(frame['content_id'].to_numpy() [:,None]), max_len)]
valid_current_question = np.reshape(pad_sequences(valid_current_question, padding="pre"),(-1,max_len))
valid_prior_elaps_time = [frame['prior_question_elapsed_time'].to_numpy() [:, None].tolist() [i:i+max_len] for _, frame in valid_group
for i in range(0, len(frame['prior_question_elapsed_time'].to_numpy() [:,None]), max_len)]
valid_prior_elaps_time = np.reshape(pad_sequences(valid_prior_elaps_time, padding="pre", dtype = 'float32'),(-1,max_len,1))
valid_time_diff = [frame['time_diff'].to_numpy() [:, None].tolist() [i:i+max_len] for _, frame in valid_group
for i in range(0, len(frame['time_diff'].to_numpy() [:,None]), max_len)]
valid_time_diff = np.reshape(pad_sequences(valid_time_diff, padding="pre", dtype = 'float32'),(-1,max_len,1))
scale = StandardScaler()
train[['Fare']]= scale.fit_transform(train[['Fare']])
 test[['Fare']]= scale.fit_transform(test[['Fare']] )<split>
valid_answered_correcly = valid_y[:,1:]
valid_past_answered_correctly = valid_past_answer[:,:-1]
valid_past_question_answer = valid_current_question[:,:-1] + valid_y[:,:-1]*(valid.content_id.max() +1)
 valid_current_question = valid_current_question[:,1:]
valid_prior_elaps_time = valid_prior_elaps_time[:,1:]
valid_time_diff = valid_time_diff[:,1:]
pred = train.drop(['Survived', 'PassengerId'], axis=1)
 target = train["Survived"]
X_train, X_valid, y_train, y_valid = train_test_split(pred, target, test_size = 0.20, random_state = 0)
<choose_model_class>
def get_angles(pos, i, d_model):
angle_rates = 1 / np.power(10000,(2 *(i//2)) / np.float32(d_model))
return pos * angle_rates
def positional_encoding(position, d_model):
angle_rads = get_angles(np.arange(position)[:, np.newaxis],
np.arange(d_model)[np.newaxis, :],
d_model)
 angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])
 angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])
 pos_encoding = angle_rads[np.newaxis,...]
return tf.cast(pos_encoding, dtype=tf.float32)
 def create_padding_output_mask(seq):
seq = 1 - tf.cast(tf.math.equal(seq, 0), tf.float32)
 return seq[:, :, tf.newaxis]
def create_padding_mask(seq):
seq = tf.cast(tf.math.equal(seq, 0), tf.float32)
 return seq[:, tf.newaxis, tf.newaxis, :]
def create_look_ahead_mask(size):
mask = 1 - tf.linalg.band_part(tf.ones(( size, size)) , -1, 0)
 return mask
def scaled_dot_product_attention(q, k, v, mask):

 matmul_qk = tf.matmul(q, k, transpose_b=True)
 dk = tf.cast(tf.shape(k)[-1], tf.float32)
 scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)
 if mask is not None:
scaled_attention_logits +=(mask * -1e9)
 attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)
 output = tf.matmul(attention_weights, v)
 return output, attention_weights
class MultiHeadAttention(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads):
super(MultiHeadAttention, self ).__init__()
self.num_heads = num_heads
self.d_model = d_model
assert d_model % self.num_heads == 0
self.depth = d_model // self.num_heads
self.wq = tf.keras.layers.Dense(d_model)
 self.wk = tf.keras.layers.Dense(d_model)
 self.wv = tf.keras.layers.Dense(d_model)
 self.dense = tf.keras.layers.Dense(d_model)
 def split_heads(self, x, batch_size):

 x = tf.reshape(x,(batch_size, -1, self.num_heads, self.depth))
return tf.transpose(x, perm=[0, 2, 1, 3])
 def call(self, v, k, q, mask):
batch_size = tf.shape(q)[0]
q = self.wq(q)
 k = self.wk(k)
 v = self.wv(v)
 q = self.split_heads(q, batch_size)
 k = self.split_heads(k, batch_size)
 v = self.split_heads(v, batch_size)
 scaled_attention, attention_weights = scaled_dot_product_attention(
 q, k, v, mask)
 scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])
 concat_attention = tf.reshape(scaled_attention, 
(batch_size, -1, self.d_model))
output = self.dense(concat_attention)
 return output, attention_weights
def point_wise_feed_forward_network(d_model, dff):
return tf.keras.Sequential([
tf.keras.layers.Dense(dff, activation='relu'),
tf.keras.layers.Dense(d_model)
 ])
 class EncoderLayer(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads, dff, rate=0.2):
super(EncoderLayer, self ).__init__()
self.mha = MultiHeadAttention(d_model, num_heads)
 self.ffn = point_wise_feed_forward_network(d_model, dff)
 self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
 self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
 self.dropout1 = tf.keras.layers.Dropout(rate)
 self.dropout2 = tf.keras.layers.Dropout(rate)
 def call(self, x, e, training, mask):
attn_output, _ = self.mha(x, x, e, mask)
 attn_output = self.dropout1(attn_output, training=training)
 out1 = self.layernorm1(e + attn_output)
 ffn_output = self.ffn(out1)
 ffn_output = self.dropout2(ffn_output, training=training)
 out2 = self.layernorm2(out1 + ffn_output)
 return out2
class Encoder(tf.keras.layers.Layer):
def __init__(self, num_layers, d_model, num_heads, dff,
maximum_position_encoding, rate=0.2):
super(Encoder, self ).__init__()
self.d_model = d_model
self.num_layers = num_layers
self.pos_encoding = positional_encoding(maximum_position_encoding,
self.d_model)
 self.enc_layers = [EncoderLayer(d_model, num_heads, dff, rate)
 for _ in range(num_layers)]
self.dropout = tf.keras.layers.Dropout(rate)
 def call(self, x, e, training, mask):
seq_len = tf.shape(x)[1]
x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))
x += self.pos_encoding[:, :seq_len, :]
x = self.dropout(x, training=training)
 for i in range(self.num_layers):
x = self.enc_layers[i](x, e, training, mask)
 return x
class Transformer(tf.keras.Model):
def __init__(self, num_layers, en_d_model, en_num_heads, dff, pe_input, rate=0.2):
super(Transformer, self ).__init__()
self.encoder = Encoder(num_layers, en_d_model, en_num_heads, dff,
pe_input, rate)
 self.second_final_layer = tf.keras.layers.Dense(dff)
 self.final_layer = Dense(1,activation = 'sigmoid')
 def call(self, inp1, inp2, training, mask):
enc_output = self.encoder(inp1, inp2, training, mask)
 second_final_output = self.second_final_layer(enc_output)
 final_output = self.final_layer(second_final_output)
 return final_output
logreg = LogisticRegression()
logreg.fit(X_train, y_train)
 y_pred = logreg.predict(X_valid)
 acc_logreg = round(accuracy_score(y_pred, y_valid)* 100, 2)
 print(acc_logreg )<train_model>
num_layers = 2
d_model = 64
num_heads = 2
dff = 256
question_answer_pair_size = train_past_question_answer.max() +1
n_question = train_current_question.max() +1
pe_input = 30
def build(num_layers, d_model, num_heads, dff, question_answer_pair_size, n_question, pe_input):
masking_func = lambda inputs, previous_mask: previous_mask
en_input1 = Input(batch_shape =(None, None), name = 'question_answer_pair')
 en_input1_embed = Embedding(question_answer_pair_size, d_model )(en_input1)
 en_input2 = Input(batch_shape =(None, None, 1), name = 'other_feature1')
 en_input2_embed = Dense(d_model, input_shape =(None, None, 1))(en_input2)
 en_input3 = Input(batch_shape =(None, None, 1), name = 'other_feature2')
 en_input3_embed = Dense(d_model, input_shape =(None, None, 1))(en_input3)
 en_input_embed_sum = tf.math.add_n([en_input1_embed, en_input2_embed, en_input3_embed])
 en_input4 = Input(batch_shape =(None, None), name = 'current_question')
 en_input4_embed = Embedding(n_question, d_model )(en_input4)
 look_ahead_mask = create_look_ahead_mask(tf.shape(en_input_embed_sum)[1])
 padding_mask = create_padding_mask(en_input1)
 combined_mask = tf.maximum(look_ahead_mask, padding_mask)
 transformer = Transformer(num_layers, d_model, num_heads, dff, pe_input)
 final_output = transformer(en_input_embed_sum, en_input4_embed, True, combined_mask)
 output_mask = create_padding_output_mask(en_input1)
 output = multiply([final_output, output_mask])
 model = Model(inputs=[en_input1, en_input2, en_input3, en_input4], outputs=output)
 model.compile(optimizer = 'adam',
loss = 'binary_crossentropy',
metrics=['accuracy',AUC() ])
 return model
SAKT_model = build(num_layers, d_model, num_heads, dff, question_answer_pair_size, n_question, pe_input)
randomforest = RandomForestClassifier()
randomforest.fit(X_train, y_train)
 y_pred = randomforest.predict(X_valid)
 acc_randomforest = round(accuracy_score(y_pred, y_valid)* 100, 2)
 print(acc_randomforest )<choose_model_class>
SAKT_model.fit([train_past_question_answer, train_prior_elaps_time, train_time_diff, train_current_question],train_answered_correcly,
validation_data=([valid_past_question_answer, valid_prior_elaps_time, valid_time_diff, valid_current_question], valid_answered_correcly), batch_size = 200,
epochs = 2, verbose = 1 )
input_shape = [X_valid.shape[1]]
model = keras.Sequential([
layers.BatchNormalization(input_shape=input_shape),
layers.Dense(25,kernel_initializer='he_uniform', activation='relu'),
layers.BatchNormalization() ,
layers.Dropout(0.5),
layers.Dense(12,kernel_initializer='he_uniform', activation='relu'),
layers.BatchNormalization() ,
layers.Dropout(0.5),
layers.Dense(1, kernel_initializer='glorot_uniform',activation='sigmoid'),
])
 model.compile(
 optimizer='adam',
loss='binary_crossentropy',
metrics=['binary_accuracy'], 
)
 early_stopping = keras.callbacks.EarlyStopping(
 patience=5,
min_delta=0.001,
restore_best_weights=True, 
)
 history = model.fit(
 X_train, y_train,
validation_data=(X_valid, y_valid),
batch_size=512,
epochs=200,
callbacks=[early_stopping], 
)
 history_df = pd.DataFrame(history.history)
 history_df.loc[:, ['loss', 'val_loss']].plot(title="Cross-entropy")
 history_df.loc[:, ['binary_accuracy', 'val_binary_accuracy']].plot(title="Accuracy" )<predict_on_test>
class EncoderLayer2(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads, dff, rate=0.2):
super(EncoderLayer2, self ).__init__()
self.mha = MultiHeadAttention(d_model, num_heads)
 self.ffn = point_wise_feed_forward_network(d_model, dff)
 self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
 self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
 self.dropout1 = tf.keras.layers.Dropout(rate)
 self.dropout2 = tf.keras.layers.Dropout(rate)
 def call(self, x, training, mask):
attn_output, _ = self.mha(x, x, x, mask)
 attn_output = self.dropout1(attn_output, training=training)
 out1 = self.layernorm1(x + attn_output)
 ffn_output = self.ffn(out1)
 ffn_output = self.dropout2(ffn_output, training=training)
 out2 = self.layernorm2(out1 + ffn_output)
 return out2
class Encoder2(tf.keras.layers.Layer):
def __init__(self, num_layers, d_model, num_heads, dff,
maximum_position_encoding, rate=0.2):
super(Encoder2, self ).__init__()
self.d_model = d_model
self.num_layers = num_layers
self.pos_encoding = positional_encoding(maximum_position_encoding,
self.d_model)
 self.enc_layers = [EncoderLayer2(d_model, num_heads, dff, rate)
 for _ in range(num_layers)]
self.dropout = tf.keras.layers.Dropout(rate)
 def call(self, x, training, mask):
seq_len = tf.shape(x)[1]
x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))
x += self.pos_encoding[:, :seq_len, :]
x = self.dropout(x, training=training)
 for i in range(self.num_layers):
x = self.enc_layers[i](x, training, mask)
 return x
class DecoderLayer2(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads, dff, rate=0.2):
super(DecoderLayer2, self ).__init__()
self.mha1 = MultiHeadAttention(d_model, num_heads)
 self.mha2 = MultiHeadAttention(d_model, num_heads)
 self.ffn = point_wise_feed_forward_network(d_model, dff)
 self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
 self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
 self.layernorm3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
 self.dropout1 = tf.keras.layers.Dropout(rate)
 self.dropout2 = tf.keras.layers.Dropout(rate)
 self.dropout3 = tf.keras.layers.Dropout(rate)
 def call(self, x, enc_output, training,
look_ahead_mask, padding_mask):
attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask)
 attn1 = self.dropout1(attn1, training=training)
 out1 = self.layernorm1(attn1 + x)
 attn2, attn_weights_block2 = self.mha2(
 enc_output, enc_output, out1, padding_mask)
 attn2 = self.dropout2(attn2, training=training)
 out2 = self.layernorm2(attn2 + out1)
 ffn_output = self.ffn(out2)
 ffn_output = self.dropout3(ffn_output, training=training)
 out3 = self.layernorm3(ffn_output + out2)
 return out3, attn_weights_block1, attn_weights_block2
class Decoder2(tf.keras.layers.Layer):
def __init__(self, num_layers, d_model, num_heads, dff,
maximum_position_encoding, rate=0.2):
super(Decoder2, self ).__init__()
self.d_model = d_model
self.num_layers = num_layers
self.pos_encoding = positional_encoding(maximum_position_encoding, d_model)
 self.dec_layers = [DecoderLayer2(d_model, num_heads, dff, rate)
 for _ in range(num_layers)]
self.dropout = tf.keras.layers.Dropout(rate)
 def call(self, x, enc_output, training,
look_ahead_mask, padding_mask):
seq_len = tf.shape(x)[1]
attention_weights = {}
x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))
x += self.pos_encoding[:, :seq_len, :]
x = self.dropout(x, training=training)
 for i in range(self.num_layers):
x, block1, block2 = self.dec_layers[i](x, enc_output, training, look_ahead_mask, padding_mask)
 attention_weights['decoder_layer{}_block1'.format(i+1)] = block1
attention_weights['decoder_layer{}_block2'.format(i+1)] = block2
return x, attention_weights
class Transformer2(tf.keras.Model):
def __init__(self, num_layers, d_model, num_heads, dff, padding_length, rate=0.2):
super(Transformer2, self ).__init__()
self.encoder = Encoder2(num_layers, d_model, num_heads, dff, padding_length)
 self.decoder = Decoder2(num_layers, d_model, num_heads, dff, padding_length)
 self.second_final_layer = tf.keras.layers.Dense(dff)
 self.final_layer = Dense(1,activation = 'sigmoid')
 def call(self, inp1, inp2, training, en_combined_mask, de_look_ahead_mask, de_padding_mask):
enc_output = self.encoder(inp1, training, en_combined_mask)
 dec_output, attention_weights = self.decoder(
 inp2, enc_output, training, de_look_ahead_mask, de_padding_mask)
 second_final_output = self.second_final_layer(dec_output)
 final_output = self.final_layer(second_final_output)
 return final_output
ynew = model.predict_classes(test.drop(['PassengerId','Survived'], axis=1))<save_to_csv>
num_layers = 1
d_model = 64
num_heads = 2
dff = 256
n_question = train_current_question.max() +1
n_answer = 3
pe_input = 30
def build(num_layers, d_model, num_heads, dff, n_question, n_answer, pe_input):
en_input1 = Input(batch_shape =(None, None), name = 'current_question')
 en_input1_embed = Embedding(n_question, d_model )(en_input1)
 en_look_ahead_mask = create_look_ahead_mask(tf.shape(en_input1)[1])
 en_padding_mask = create_padding_mask(en_input1)
 en_combined_mask = tf.maximum(en_look_ahead_mask, en_padding_mask)
 de_input2 = Input(batch_shape =(None, None), name = 'past_answer')
 de_input2_embed = Embedding(n_answer, d_model )(de_input2)
 de_input3 = Input(batch_shape =(None, None, 1), name = 'other_feature1')
 de_input3_embed = Dense(d_model, input_shape =(None, None, 1))(de_input3)
 de_input4 = Input(batch_shape =(None, None, 1), name = 'other_feature2')
 de_input4_embed = Dense(d_model, input_shape =(None, None, 1))(de_input4)
 de_input = tf.math.add_n([de_input2_embed, de_input3_embed, de_input4_embed])
 transformer = Transformer2(num_layers, d_model, num_heads, dff, pe_input)
 final_output = transformer(en_input1_embed, de_input, True, en_combined_mask, en_combined_mask, en_combined_mask)
 output_mask = create_padding_output_mask(en_input1)
 model = Model(inputs=[en_input1, de_input2, de_input3, de_input4], outputs=final_output)
 model.compile(optimizer = 'adam',
loss = 'binary_crossentropy',
metrics=['accuracy',AUC() ])
 return model
SAINT_model = build(num_layers, d_model, num_heads, dff, n_question, n_answer, pe_input )
ids = test['PassengerId']
predictions = ynew.flatten()
output = pd.DataFrame({ 'PassengerId' : ids, 'Survived': predictions })
 output.to_csv('submission.csv', index=False )<set_options>
SAINT_model.fit([train_current_question, train_past_answered_correctly, train_prior_elaps_time, train_time_diff],train_answered_correcly,
validation_data=([valid_current_question, valid_past_answered_correctly, valid_prior_elaps_time, valid_time_diff], valid_answered_correcly),
batch_size = 200,
epochs = 2, verbose = 1 )
%matplotlib inline
warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', None)
<load_from_csv>
train_answered_correcly = np.reshape(train_y[:,1:],(-1,30,1))
train_past_question_answer -= 1
train_current_question -= 1
train_other_feats = np.dstack(( train_prior_elaps_time,train_time_diff))
valid_answered_correcly = np.reshape(valid_y[:,1:],(-1,30,1))
valid_past_question_answer -= 1
valid_current_question -= 1
valid_other_feats = np.dstack(( valid_prior_elaps_time,valid_time_diff))
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 test_data = pd.read_csv("/kaggle/input/titanic/test.csv" )<create_dataframe>
other_input_dim = 2
hidden_layer_size = 50
input_dim_order = train_current_question.max() + 1
prev_q_perform_dim = train_past_question_answer.max() + 1
train = train_data.copy()
test = test_data.copy()<count_missing_values>
def dkt_build(hidden_layer_size, input_dim_order, prev_q_perform_dim, other_input_dim):
masking_func = lambda inputs, previous_mask: previous_mask
prev_question_ans = Input(batch_shape =(None, None), dtype = 'int32', name = 'prev_qn_ans')
 one_hot_prev_question_ans = tf.one_hot(prev_question_ans, prev_q_perform_dim, axis = -1)
 other_input = Input(batch_shape =(None, None, other_input_dim), name= 'other_input')
 one_hot = Concatenate()([one_hot_prev_question_ans, other_input])
 masked_one_hot =(Masking(mask_value= 0, input_shape =(None, None, prev_q_perform_dim + other_input_dim)) )(one_hot)
 lstm_out = LSTM(hidden_layer_size, input_shape =(None, None, prev_q_perform_dim + other_input_dim),
dropout=0.2, recurrent_dropout =0.2, return_sequences = True )(masked_one_hot)
 dense_out = Dense(input_dim_order, input_shape =(None, None, hidden_layer_size), activation='sigmoid' )(lstm_out)
 order = Input(batch_shape =(None, None), dtype = 'int32', name = 'order')
 one_hot_order = tf.one_hot(order, input_dim_order, axis = -1)
 merged = multiply([dense_out, one_hot_order])
 def reduce_dim(x):
x = K.max(x, axis = 2, keepdims = True)
 return x
def reduce_dim_shape(input_shape):
shape = list(input_shape)
 shape[-1] = 1
print("reduced_shape", shape)
 return tuple(shape)
 reduced = Lambda(reduce_dim, output_shape = reduce_dim_shape, mask = masking_func )(merged)
 model = Model(inputs=[prev_question_ans, other_input, order], outputs=reduced)
 model.compile(optimizer = 'adam',
loss = 'binary_crossentropy',
metrics=['accuracy',AUC() ])
 return model
100*train.isnull().sum() /len(train )<count_values>
dkt_model = dkt_build(hidden_layer_size, input_dim_order, prev_q_perform_dim, other_input_dim )
train['Pclass'].value_counts()<count_values>
dkt_model.fit([train_past_question_answer, train_other_feats, train_current_question], train_answered_correcly,
validation_data=([valid_past_question_answer, valid_other_feats, valid_current_question], valid_answered_correcly),
epochs = 2, verbose = 1, batch_size = 200 )
train['Sex'].value_counts()<count_values>
env = riiideducation.make_env()
iter_test = env.iter_test()
set_predict = env.predict
<count_values><EOS> 
previous_test_df = None
for(test_df, sample_prediction_df)in iter_test:
if previous_test_df is not None:
previous_test_df[TARGET] = np.array(eval(test_df["prior_group_answers_correct"].iloc[0])) [mask]
update_user_feats(previous_test_df, last_time_u_dict, answered_correctly_sum_u_dict, count_u_dict)
 test_df = pd.merge(test_df, questions_df[['question_id', 'part','tag']], left_on='content_id', right_on = 'question_id', how='left')
 mask =(test_df['content_type_id'] == 0 ).values.tolist()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df = add_user_feats_without_update(test_df , last_time_u_dict, answered_correctly_sum_u_dict, count_u_dict)
 test_df = pd.merge(test_df, content_df, on='content_id', how="left")
 test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False ).astype('int8')
 test_df['prior_question_elapsed_time'] = test_df.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 previous_test_df = test_df.copy()
lgbm_predict = lgbm_model.predict(test_df[FEATS])
 current_question, past_question_answer, past_other_feats, past_answer_correctly = get_user_feats_for_nn_without_update(test_df, past_question, past_answer, past_prior_elaps, past_time_diff)
 prior_elaps_time = np.reshape(past_other_feats[:,:,0],(-1,30,1))
time_diff = np.reshape(past_other_feats[:,:,1],(-1,30,1))
sakt_predict = SAKT_model.predict([past_question_answer, prior_elaps_time, time_diff, current_question], batch_size = 500)
 saint_predict = SAINT_model.predict([current_question, past_answer_correctly, prior_elaps_time, time_diff], batch_size = 500)
 past_question_answer = past_question_answer - 1
current_question = current_question - 1
dkt_predict = dkt_model.predict([past_question_answer, past_other_feats, current_question], batch_size = 200)
 test_df[TARGET] = dkt_predict[:,-1,0]*(0.1)+ sakt_predict[:,-1,0]*(0.15)+ lgbm_predict*0.6 + saint_predict[:,-1,0]*(0.15)
 set_predict(test_df[['row_id', TARGET]])