Kaggle competition: Riiid Answer Correctness Prediction
def imputer_specific_categories(dataset, listfeatures):
data = dataset.copy()
for names in listfeatures:
data[names] = data[names].fillna("None")
 return data<feature_engineering>
debug = True
validaten_flg = False
EXP_ID = 'exp004'
CV_NUM = 'cv1'
def unpickle(filename):
with open(filename, mode='rb')as fo:
p = pickle.load(fo)
 return p
class CFG:
INPUT_DIR = '.. /input/riiid-test-answer-prediction'
OUT_DIR = '.. /input/riiid-trained-models-exp004'
DATA_DIR = '.. /input/riiid-cv-datasets'
TRAIN_PATH = os.path.join(INPUT_DIR, "train.csv")
 TEST_PATH = os.path.join(INPUT_DIR, "example_test.csv")
 LECTURE_PATH = os.path.join(INPUT_DIR, "lectures.csv")
 QUESTION_PATH = os.path.join(INPUT_DIR, "questions.csv")
 SUBMISSION_PATH = os.path.join(INPUT_DIR, "example_sample_submission.csv")
 SEED = 6718
train_pickle = '.. /input/riiid-cross-validation-files/cv1_train.pickle'
valid_pickle = '.. /input/riiid-cross-validation-files/cv1_valid.pickle'
question_file = '.. /input/riiid-test-answer-prediction/questions.csv'
debug = True
validaten_flg = False
features_meaning_none_value= ["Alley", "PoolQC", "MiscFeature","Fence","MiscVal","FireplaceQu",
'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2','MSSubClass']
train_drop = imputer_specific_categories(train, features_meaning_none_value)
 test = imputer_specific_categories(test,features_meaning_none_value )<compute_test_metric>
with trace(''):
questions_df = pd.read_csv(question_file)[['question_id', 'part', 'tags', 'bundle_id']]
missing_values_calculate(train_drop )<compute_test_metric>
def to_pickle(filename, obj):
with open(filename, mode='wb')as f:
pickle.dump(obj, f)
 def unpickle(filename):
with open(filename, mode='rb')as fo:
p = pickle.load(fo)
 return p
with trace(''):
group = unpickle('.. /input/riiid-transformer-all-v7-add-elapsed-time-pre/group.pkl')
 val_group = unpickle('.. /input/riiid-transformer-all-v7-add-elapsed-time-pre/val_group.pkl')
 skills = unpickle('.. /input/riiid-transformer-all-v7-add-elapsed-time-pre/skills.pkl')
 elapsed_time_n_unq = unpickle('.. /input/riiid-transformer-all-v7-add-elapsed-time-pre/elapsed_time_n_unq.pkl' )
missing_values_calculate(test )<feature_engineering>
MAX_SEQ = 160
with trace(''):
n_skill = 13523
print("number skills", len(skills))
train_drop['total_year_build_and_remod']= train_drop['YearBuilt']+train_drop['YearRemodAdd']
train_drop['Total_sqr_are'] =(train_drop['BsmtFinSF1'] + train_drop['BsmtFinSF2'] +
train_drop['1stFlrSF'] + train_drop['2ndFlrSF'])
 train_drop['Total_Bathrooms'] =(train_drop['FullBath'] +(0.5 * train_drop['HalfBath'])+
train_drop['BsmtFullBath'] +(0.5 * train_drop['BsmtHalfBath']))
train_drop['Total_porch_sf'] =(train_drop['OpenPorchSF'] + train_drop['3SsnPorch'] +
train_drop['EnclosedPorch'] + train_drop['ScreenPorch'] +
train_drop['WoodDeckSF'])
 test['total_year_build_and_remod']= test['YearBuilt']+test['YearRemodAdd']
test['Total_sqr_are'] =(test['BsmtFinSF1'] + test['BsmtFinSF2'] +
test['1stFlrSF'] + test['2ndFlrSF'])
 test['Total_Bathrooms'] =(test['FullBath'] +(0.5 * test['HalfBath'])+
test['BsmtFullBath'] +(0.5 * test['BsmtHalfBath']))
test['Total_porch_sf'] =(test['OpenPorchSF'] + test['3SsnPorch'] +
test['EnclosedPorch'] + test['ScreenPorch'] +
test['WoodDeckSF'] )<feature_engineering>
class SAKTDataset(Dataset):
def __init__(self, group, n_skill, max_seq=MAX_SEQ):
super(SAKTDataset, self ).__init__()
self.max_seq = max_seq
self.n_skill = n_skill
self.samples = {}
self.user_ids = []
for user_id in group.index:
q, qa, pt, elt = group[user_id]
if len(q)< 5:
continue
if len(q)> self.max_seq:
total_questions = len(q)
 initial = total_questions % self.max_seq
if initial >= 5:
self.user_ids.append(f"{user_id}_0")
 self.samples[f"{user_id}_0"] =(q[:initial], qa[:initial], pt[:initial], elt[:initial])
 for seq in range(total_questions // self.max_seq):
self.user_ids.append(f"{user_id}_{seq+1}")
 start = initial + seq * self.max_seq
end = start + self.max_seq
self.samples[f"{user_id}_{seq+1}"] =(q[start:end], qa[start:end], pt[start:end], elt[start:end])
 else:
user_id = str(user_id)
 self.user_ids.append(user_id)
 self.samples[user_id] =(q, qa, pt, elt)
 def __len__(self):
return len(self.user_ids)
 def __getitem__(self, index):
user_id = self.user_ids[index]
q_, qa_, pt_, elt_ = self.samples[user_id]
seq_len = len(q_)
 q = np.zeros(self.max_seq, dtype=int)
 qa = np.zeros(self.max_seq, dtype=int)
 pt = np.zeros(self.max_seq, dtype=int)
 elt = np.zeros(self.max_seq, dtype=int)
 if seq_len == self.max_seq:
q[:] = q_
qa[:] = qa_
pt[:] = pt_
elt[:] = elt_
else:
q[-seq_len:] = q_
qa[-seq_len:] = qa_
pt[-seq_len:] = pt_
elt[-seq_len:] = elt_
target_id = q[1:]
label = qa[1:]
part_id = pt[1:]
elapsed_time = elt[:-1]
x = np.zeros(self.max_seq-1, dtype=int)
 x = q[:-1].copy()
x +=(qa[:-1] == 1)* self.n_skill
return x, target_id, part_id, elapsed_time, label
def has_feature_function(data, column_name_features={}):
for column,name in column_name_features.items() :
data[name] = data[column].apply(lambda x: 1 if x > 0 else 0 )<define_variables>
def get_clones(module, N):
return nn.ModuleList([copy.deepcopy(module)for i in range(N)])
 def get_mask(seq_len):
return torch.from_numpy(np.triu(np.ones(( seq_len ,seq_len)) , k=1 ).astype('bool')).to(device)
 def get_pos(seq_len):
return torch.arange(seq_len ).unsqueeze(0 ).to(device)
 class Feed_Forward_block(nn.Module):

 def __init__(self, dim_ff):
super().__init__()
self.layer1 = nn.Linear(in_features=dim_ff , out_features=dim_ff)
 self.layer2 = nn.Linear(in_features=dim_ff , out_features=dim_ff)
 def forward(self,ffn_in):
return self.layer2(F.relu(self.layer1(ffn_in)))
 class Encoder_block(nn.Module):

 def __init__(self , dim_model, heads_en, total_ex ,total_pt, seq_len):
super().__init__()
self.seq_len = seq_len
self.embd_ex = nn.Embedding(total_ex , embedding_dim = dim_model)
 self.embd_pt = nn.Embedding(total_pt, embedding_dim = dim_model)
 self.embd_pos = nn.Embedding(seq_len , embedding_dim = dim_model)
 self.multi_en = nn.MultiheadAttention(embed_dim= dim_model, num_heads= heads_en,)
 self.ffn_en = Feed_Forward_block(dim_model)
 self.layer_norm1 = nn.LayerNorm(dim_model)
 self.layer_norm2 = nn.LayerNorm(dim_model)
 def forward(self, in_ex, in_pt, first_block=True):
if first_block:
in_ex = self.embd_ex(in_ex)
 in_pt = self.embd_pt(in_pt)
 out = in_ex + in_pt
else:
out = in_ex
in_pos = get_pos(self.seq_len)
 in_pos = self.embd_pos(in_pos)
 out = out + in_pos
out = out.permute(1,0,2)
 n,_,_ = out.shape
out = self.layer_norm1(out)
 skip_out = out
out, attn_wt = self.multi_en(out , out , out ,
attn_mask=get_mask(seq_len=n))
out = out + skip_out
out = out.permute(1,0,2)
 out = self.layer_norm2(out)
 skip_out = out
out = self.ffn_en(out)
 out = out + skip_out
return out
class Decoder_block(nn.Module):

 def __init__(self, dim_model, total_in, total_et, heads_de, seq_len):
super().__init__()
self.seq_len = seq_len
self.embd_in = nn.Embedding(total_in, embedding_dim = dim_model)
 self.embd_et = nn.Embedding(total_et, embedding_dim = dim_model)
 self.embd_pos = nn.Embedding(seq_len, embedding_dim = dim_model)
 self.multi_de1 = nn.MultiheadAttention(embed_dim=dim_model, num_heads=heads_de)
 self.multi_de2 = nn.MultiheadAttention(embed_dim=dim_model, num_heads=heads_de)
 self.ffn_en = Feed_Forward_block(dim_model)
 self.layer_norm1 = nn.LayerNorm(dim_model)
 self.layer_norm2 = nn.LayerNorm(dim_model)
 self.layer_norm3 = nn.LayerNorm(dim_model)
 def forward(self, in_in, in_et, en_out, first_block=True):
if first_block:
in_in = self.embd_in(in_in)
 in_et = self.embd_et(in_et)
 out = in_in + in_et
else:
out = in_in
in_pos = get_pos(self.seq_len)
 in_pos = self.embd_pos(in_pos)
 out = out + in_pos
out = out.permute(1,0,2)
 n,_,_ = out.shape
out = self.layer_norm1(out)
 skip_out = out
out, attn_wt = self.multi_de1(out , out , out,
attn_mask=get_mask(seq_len=n))
out = skip_out + out
en_out = en_out.permute(1,0,2)
 en_out = self.layer_norm2(en_out)
 skip_out = out
out, attn_wt = self.multi_de2(out , en_out , en_out,
attn_mask=get_mask(seq_len=n))
out = out + skip_out
out = out.permute(1,0,2)
 out = self.layer_norm3(out)
 skip_out = out
out = self.ffn_en(out)
 out = out + skip_out
return out
class SAINT(nn.Module):
def __init__(self, dim_model, num_en, num_de, heads_en, total_ex, total_pt, total_in, total_et, heads_de, seq_len):
super().__init__()
self.num_en = num_en
self.num_de = num_de
self.encoder = get_clones(Encoder_block(dim_model, heads_en, total_ex, total_pt, seq_len), num_en)
 self.decoder = get_clones(Decoder_block(dim_model ,total_in, total_et, heads_de, seq_len), num_de)
 self.out = nn.Linear(in_features= dim_model, out_features=1)
 def forward(self, in_in, in_ex, in_pt, in_et):
first_block = True
for x in range(self.num_en):
if x>=1:
first_block = False
in_ex = self.encoder[x](in_ex, in_pt, first_block=first_block)
 in_pt = in_ex
first_block = True
for x in range(self.num_de):
if x>=1:
first_block = False
in_in = self.decoder[x](in_in, in_et, en_out=in_ex, first_block=first_block)
 in_in = self.out(in_in)
 return in_in.squeeze(-1 )
columns = {'PoolArea': 'haspool',
'2ndFlrSF': 'has2ndfloor',
'GarageArea': 'hasgarage',
'TotalBsmtSF': 'hasbasement',
'Fireplaces': 'hasfireplace',
}<drop_column>
with trace(''):
n_part = 7
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model = SAINT(dim_model=128,
num_en=3,
num_de=3,
heads_en=8,
total_ex=n_skill+1,
total_pt=n_part+1,
total_in=2*n_skill+1,
total_et=elapsed_time_n_unq+1,
heads_de=8,
seq_len=MAX_SEQ-1)
 model.load_state_dict(torch.load('.. /input/riiid-transformer-add-part-v11/SAKT-HDKIM.pt'))
model.to(device )
train_drop.drop(train_drop[(train_drop["SalePrice"] < 300000)&(train_drop["GrLivArea"] > 4000)].index, inplace=True )<define_variables>
ADD = [
'answered_correctly_true_weight_sum_u',
'count_u_c',
'count_c',
'task_container_id',
'task_container_count',
'answered_correctly_u_avg', 'elapsed_time_u_avg', 'explanation_u_avg',
'answered_correctly_q_avg', 'elapsed_time_q_avg', 'explanation_q_avg',
'answered_correctly_uq_count', 'timestamp_u_recency_1', 'timestamp_u_recency_2', 'timestamp_u_recency_3',
'timestamp_u_incorrect_recency',
'part_bundle_id',
'tags1', 'tags2', 'tags3',
'tags4', 'tags5', 'tags6',
'part_bundle_count',
'count_u_tag1',
'count_u_tag2',
'count_u_tag3',
'count_u_part_bundle_id',
'task_container_id_u',
]
TARGET = 'answered_correctly'
FEATS = ['answered_correctly_avg_u', 'answered_correctly_sum_u', 'count_u', 'answered_correctly_avg_c', 'part', 'prior_question_had_explanation', 'prior_question_elapsed_time']
FEATS = FEATS + ADD
train_labels = train_drop["SalePrice"].copy()<drop_column>
train_drop.drop(["SalePrice",'Utilities', 'Street', 'PoolQC','Id'], axis=1, inplace=True)
 test.drop(['Utilities', 'Street', 'PoolQC','Id'], axis=1, inplace=True )<define_variables>
num_attribs = train_drop.select_dtypes(exclude=['object'])
 cat_attribs = train_drop.select_dtypes(exclude=['int64','float64'] )<choose_model_class>
<prepare_x_and_y><EOS> 
model.eval()
env = riiideducation.make_env()
iter_test = env.iter_test()
set_predict = env.predict
previous_test_df = None
for(test_df, sample_prediction_df)in iter_test:
if(previous_test_df is not None)&(psutil.virtual_memory().percent<90):
print(psutil.virtual_memory().percent)
 previous_test_df[TARGET] = eval(test_df["prior_group_answers_correct"].iloc[0])
 previous_test_df = previous_test_df[previous_test_df.content_type_id == False]
previous_test_df = previous_test_df.merge(questions_df[questions_df.question_id.isin(previous_test_df.content_id)][['part']],
how='left', left_on='content_id', right_index=True)
 prev_group = previous_test_df[['user_id', 'content_id', 'answered_correctly', 'part', 'timestamp', 'prior_question_elapsed_time']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values,
r['part'].values, 
(r['prior_question_elapsed_time'].fillna(0 ).values/1000 ).astype(np.int64), 
))
for prev_user_id in prev_group.index:
if prev_user_id in group.index:
group[prev_user_id] =(
 np.append(group[prev_user_id][0], prev_group[prev_user_id][0])[-MAX_SEQ:],
np.append(group[prev_user_id][1], prev_group[prev_user_id][1])[-MAX_SEQ:],
np.append(group[prev_user_id][2], prev_group[prev_user_id][2])[-MAX_SEQ:],
np.append(group[prev_user_id][3], prev_group[prev_user_id][3])[-MAX_SEQ:] 
)
 else:
group[prev_user_id] =(
 prev_group[prev_user_id][0],
prev_group[prev_user_id][1],
prev_group[prev_user_id][2],
prev_group[prev_user_id][3] 
)
 previous_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df = test_df.merge(questions_df[questions_df.question_id.isin(test_df.content_id)][['part']],
how='left', left_on='content_id', right_index=True)
 test_dataset = TestDataset(group, test_df, skills)
 test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)
 outs = []
for item in tqdm(test_dataloader):
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
part_id = item[2].to(device ).long()
elapsed_time = item[3].to(device ).long()
with torch.no_grad() :
output = model(x, target_id, part_id, elapsed_time)
 output = torch.sigmoid(output)
 output = output[:, -1]
outs.extend(output.view(-1 ).data.cpu().numpy())

test_df[TARGET] = outs
set_predict(test_df[['row_id', TARGET]] )