Kaggle competition: Tabular Playground Series - Feb 2021
with trace(''):
questions_df = pd.read_csv(question_file)[['question_id', 'part', 'tags', 'bundle_id']] 
<load_pretrained>
PATH = '.. /input/tabular-playground-series-feb-2021/'
train = pd.read_csv(PATH + 'train.csv')
 test = pd.read_csv(PATH + 'test.csv')
 sample = pd.read_csv(PATH + 'sample_submission.csv')
 print(train.shape, test.shape )
def to_pickle(filename, obj):
with open(filename, mode='wb')as f:
pickle.dump(obj, f)
 def unpickle(filename):
with open(filename, mode='rb')as fo:
p = pickle.load(fo)
 return p
with trace(''):
group = unpickle('.. /input/riiid-transformer-all-v7-add-elapsed-time-pre/group.pkl')
 val_group = unpickle('.. /input/riiid-transformer-all-v7-add-elapsed-time-pre/val_group.pkl')
 skills = unpickle('.. /input/riiid-transformer-all-v7-add-elapsed-time-pre/skills.pkl')
 elapsed_time_n_unq = unpickle('.. /input/riiid-transformer-all-v7-add-elapsed-time-pre/elapsed_time_n_unq.pkl' )<define_variables>
FEATURES = train.drop(['id', 'target'], 1 ).columns
FEATURES
MAX_SEQ = 160
with trace(''):
n_skill = 13523
print("number skills", len(skills))<create_dataframe>
for i in cat_features:
le = LabelEncoder()
le.fit(train[i])
 train[i] = le.transform(train[i])
 test[i] = le.transform(test[i])
 train.head()
class SAKTDataset(Dataset):
def __init__(self, group, n_skill, max_seq=MAX_SEQ):
super(SAKTDataset, self ).__init__()
self.max_seq = max_seq
self.n_skill = n_skill
self.samples = {}
self.user_ids = []
for user_id in group.index:
q, qa, pt, elt = group[user_id]
if len(q)< 5:
continue
if len(q)> self.max_seq:
total_questions = len(q)
 initial = total_questions % self.max_seq
if initial >= 5:
self.user_ids.append(f"{user_id}_0")
 self.samples[f"{user_id}_0"] =(q[:initial], qa[:initial], pt[:initial], elt[:initial])
 for seq in range(total_questions // self.max_seq):
self.user_ids.append(f"{user_id}_{seq+1}")
 start = initial + seq * self.max_seq
end = start + self.max_seq
self.samples[f"{user_id}_{seq+1}"] =(q[start:end], qa[start:end], pt[start:end], elt[start:end])
 else:
user_id = str(user_id)
 self.user_ids.append(user_id)
 self.samples[user_id] =(q, qa, pt, elt)
 def __len__(self):
return len(self.user_ids)
 def __getitem__(self, index):
user_id = self.user_ids[index]
q_, qa_, pt_, elt_ = self.samples[user_id]
seq_len = len(q_)
 q = np.zeros(self.max_seq, dtype=int)
 qa = np.zeros(self.max_seq, dtype=int)
 pt = np.zeros(self.max_seq, dtype=int)
 elt = np.zeros(self.max_seq, dtype=int)
 if seq_len == self.max_seq:
q[:] = q_
qa[:] = qa_
pt[:] = pt_
elt[:] = elt_
else:
q[-seq_len:] = q_
qa[-seq_len:] = qa_
pt[-seq_len:] = pt_
elt[-seq_len:] = elt_
target_id = q[1:]
label = qa[1:]
part_id = pt[1:]
elapsed_time = elt[:-1]
x = np.zeros(self.max_seq-1, dtype=int)
 x = q[:-1].copy()
x +=(qa[:-1] == 1)* self.n_skill
return x, target_id, part_id, elapsed_time, label<categorify>
cv = KFold(n_splits=5, shuffle=True)
 cv
def get_clones(module, N):
return nn.ModuleList([copy.deepcopy(module)for i in range(N)])
 def get_mask(seq_len):
return torch.from_numpy(np.triu(np.ones(( seq_len ,seq_len)) , k=1 ).astype('bool')).to(device)
 def get_pos(seq_len):
return torch.arange(seq_len ).unsqueeze(0 ).to(device)
 class Feed_Forward_block(nn.Module):

 def __init__(self, dim_ff):
super().__init__()
self.layer1 = nn.Linear(in_features=dim_ff , out_features=dim_ff)
 self.layer2 = nn.Linear(in_features=dim_ff , out_features=dim_ff)
 def forward(self,ffn_in):
return self.layer2(F.relu(self.layer1(ffn_in)))
 class Encoder_block(nn.Module):

 def __init__(self , dim_model, heads_en, total_ex ,total_pt, seq_len):
super().__init__()
self.seq_len = seq_len
self.embd_ex = nn.Embedding(total_ex , embedding_dim = dim_model)
 self.embd_pt = nn.Embedding(total_pt, embedding_dim = dim_model)
 self.embd_pos = nn.Embedding(seq_len , embedding_dim = dim_model)
 self.multi_en = nn.MultiheadAttention(embed_dim= dim_model, num_heads= heads_en,)
 self.ffn_en = Feed_Forward_block(dim_model)
 self.layer_norm1 = nn.LayerNorm(dim_model)
 self.layer_norm2 = nn.LayerNorm(dim_model)
 def forward(self, in_ex, in_pt, first_block=True):
if first_block:
in_ex = self.embd_ex(in_ex)
 in_pt = self.embd_pt(in_pt)
 out = in_ex + in_pt
else:
out = in_ex
in_pos = get_pos(self.seq_len)
 in_pos = self.embd_pos(in_pos)
 out = out + in_pos
out = out.permute(1,0,2)
 n,_,_ = out.shape
out = self.layer_norm1(out)
 skip_out = out
out, attn_wt = self.multi_en(out , out , out ,
attn_mask=get_mask(seq_len=n))
out = out + skip_out
out = out.permute(1,0,2)
 out = self.layer_norm2(out)
 skip_out = out
out = self.ffn_en(out)
 out = out + skip_out
return out
class Decoder_block(nn.Module):

 def __init__(self, dim_model, total_in, total_et, heads_de, seq_len):
super().__init__()
self.seq_len = seq_len
self.embd_in = nn.Embedding(total_in, embedding_dim = dim_model)
 self.embd_et = nn.Embedding(total_et, embedding_dim = dim_model)
 self.embd_pos = nn.Embedding(seq_len, embedding_dim = dim_model)
 self.multi_de1 = nn.MultiheadAttention(embed_dim=dim_model, num_heads=heads_de)
 self.multi_de2 = nn.MultiheadAttention(embed_dim=dim_model, num_heads=heads_de)
 self.ffn_en = Feed_Forward_block(dim_model)
 self.layer_norm1 = nn.LayerNorm(dim_model)
 self.layer_norm2 = nn.LayerNorm(dim_model)
 self.layer_norm3 = nn.LayerNorm(dim_model)
 def forward(self, in_in, in_et, en_out, first_block=True):
if first_block:
in_in = self.embd_in(in_in)
 in_et = self.embd_et(in_et)
 out = in_in + in_et
else:
out = in_in
in_pos = get_pos(self.seq_len)
 in_pos = self.embd_pos(in_pos)
 out = out + in_pos
out = out.permute(1,0,2)
 n,_,_ = out.shape
out = self.layer_norm1(out)
 skip_out = out
out, attn_wt = self.multi_de1(out , out , out,
attn_mask=get_mask(seq_len=n))
out = skip_out + out
en_out = en_out.permute(1,0,2)
 en_out = self.layer_norm2(en_out)
 skip_out = out
out, attn_wt = self.multi_de2(out , en_out , en_out,
attn_mask=get_mask(seq_len=n))
out = out + skip_out
out = out.permute(1,0,2)
 out = self.layer_norm3(out)
 skip_out = out
out = self.ffn_en(out)
 out = out + skip_out
return out
class SAINT(nn.Module):
def __init__(self, dim_model, num_en, num_de, heads_en, total_ex, total_pt, total_in, total_et, heads_de, seq_len):
super().__init__()
self.num_en = num_en
self.num_de = num_de
self.encoder = get_clones(Encoder_block(dim_model, heads_en, total_ex, total_pt, seq_len), num_en)
 self.decoder = get_clones(Decoder_block(dim_model ,total_in, total_et, heads_de, seq_len), num_de)
 self.out = nn.Linear(in_features= dim_model, out_features=1)
 def forward(self, in_in, in_ex, in_pt, in_et):
first_block = True
for x in range(self.num_en):
if x>=1:
first_block = False
in_ex = self.encoder[x](in_ex, in_pt, first_block=first_block)
 in_pt = in_ex
first_block = True
for x in range(self.num_de):
if x>=1:
first_block = False
in_in = self.decoder[x](in_in, in_et, en_out=in_ex, first_block=first_block)
 in_in = self.out(in_in)
 return in_in.squeeze(-1 )<set_options>
model = lgb.LGBMRegressor()
model
NUM_BOOST_ROUNDS = 20000
EARLY_STOPPING_ROUNDS = 500
VERBOSE_EVAL = 0
oof_df = train[['id', 'target']].copy()
fold_ = 1
X_train, X_val, y_train, y_val = train_test_split(
 X, y, test_size=0.3, shuffle=True )
with trace(''):
n_part = 7
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model = SAINT(dim_model=128,
num_en=3,
num_de=3,
heads_en=8,
total_ex=n_skill+1,
total_pt=n_part+1,
total_in=2*n_skill+1,
total_et=elapsed_time_n_unq+1,
heads_de=8,
seq_len=MAX_SEQ-1)
 model.load_state_dict(torch.load('.. /input/riiid-transformer-add-part-v11/SAKT-HDKIM.pt'))
model.to(device )<define_variables>
def objective(trial):
train_set = lgb.Dataset(X_train, y_train)
 val_set = lgb.Dataset(X_val, y_val)
 param = {
"objective": "regression",
"metric": "rmse",
"verbosity": 1,
"boosting_type": "gbdt",
"num_leaves": trial.suggest_int("num_leaves", 0, 256),
"max_depth": trial.suggest_int("max_depth", 3, 31),
"lambda_l1": trial.suggest_float("lambda_l1", 0.0, 10),
"lambda_l2": trial.suggest_float("lambda_l2", 0.4, 10),
"feature_fraction": trial.suggest_float("feature_fraction", 0.4, 0.9),
"bagging_fraction": trial.suggest_float("bagging_fraction", 0.4, 0.9),
"bagging_freq": trial.suggest_int("bagging_freq", 5, 15),
"min_child_samples": trial.suggest_int("min_child_samples", 5, 100),
}
model = lgb.train(param,
train_set,
num_boost_round=NUM_BOOST_ROUNDS,
early_stopping_rounds=EARLY_STOPPING_ROUNDS,
verbose_eval=VERBOSE_EVAL,
valid_sets=[train_set, val_set])
 val_preds = model.predict(X_val, num_iteration=model.best_iteration)
 scc = math.sqrt(mean_squared_error(val_preds, y_val))
return -1*scc
ADD = [
'answered_correctly_true_weight_sum_u',
'count_u_c',
'count_c',
'task_container_id',
'task_container_count',
'answered_correctly_u_avg', 'elapsed_time_u_avg', 'explanation_u_avg',
'answered_correctly_q_avg', 'elapsed_time_q_avg', 'explanation_q_avg',
'answered_correctly_uq_count', 'timestamp_u_recency_1', 'timestamp_u_recency_2', 'timestamp_u_recency_3',
'timestamp_u_incorrect_recency',
'part_bundle_id',
'tags1', 'tags2', 'tags3',
'tags4', 'tags5', 'tags6',
'part_bundle_count',
'count_u_tag1',
'count_u_tag2',
'count_u_tag3',
'count_u_part_bundle_id',
'task_container_id_u',
]
TARGET = 'answered_correctly'
FEATS = ['answered_correctly_avg_u', 'answered_correctly_sum_u', 'count_u', 'answered_correctly_avg_c', 'part', 'prior_question_had_explanation', 'prior_question_elapsed_time']
FEATS = FEATS + ADD<prepare_x_and_y>
study = optuna.create_study(direction="maximize")
 study.optimize(objective, n_trials=100)
 trial = study.best_trial
trial.params['metric'] = 'rmse'
<create_dataframe>
print(trial.params )
<train_model>
for train_idx, val_idx in cv.split(X, y):
X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]
y_train, y_val = y[train_idx], y[val_idx]
train_set = lgb.Dataset(X_train, y_train)
 val_set = lgb.Dataset(X_val, y_val)
 model = lgb.train(trial.params,
train_set,
num_boost_round=NUM_BOOST_ROUNDS,
early_stopping_rounds=EARLY_STOPPING_ROUNDS,
verbose_eval=-10,
valid_sets=[train_set, val_set] 
)
 val_preds = model.predict(X_val, num_iteration=model.best_iteration)
 test_preds = model.predict(
 test[FEATURES], num_iteration=model.best_iteration)
 oof_df.loc[oof_df.iloc[val_idx].index, 'oof'] = val_preds
sample[f'fold{fold_}'] = test_preds
score = mean_squared_error(
 oof_df.loc[oof_df.iloc[val_idx].index]['target'], oof_df.loc[oof_df.iloc[val_idx].index]['oof'])
 print(math.sqrt(score))
fold_ += 1
<merge>
print(math.sqrt(mean_squared_error(oof_df.target, oof_df.oof)))
 sample['target'] = sample.drop(['id', 'target'], 1 ).mean(axis=1)
 sample[['id', 'target']].to_csv('submission.csv', index=False )