Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<split>
warnings.filterwarnings('ignore')
 %matplotlib inline
iter_test = env.iter_test()<create_dataframe>
path_train = '.. /input/train.csv'
path_test = '.. /input/test.csv'
t_cols = ["ans_rate_class","study_rate_class"] + [f"part{p}_rate_class" for p in range(1,8)]
n_cols = ["num_que", "num_que"] + num_part_cols
r_cols = ['row_id','timestamp','user_id','question_id','content_type_id','task_container_id','prior_question_elapsed_time','prior_question_had_explanation']
df_test = pd.DataFrame()<data_type_conversions>
train_df_raw = pd.read_csv(path_train)
 train_df_raw.head()
%%time
for idx,(test_df, sample_prediction_df)in enumerate(iter_test):
test_df.prior_question_had_explanation =(test_df.prior_question_had_explanation.fillna(False)* 1 ).astype(int)
 test_df.prior_question_elapsed_time = test_df.prior_question_elapsed_time.fillna(0 ).astype(int)
 prior_ans = eval(test_df.prior_group_answers_correct.values[0])
 test_ary = np.array(test_df)[:,:8]
lec_ary = test_ary[test_ary[:,4]==1, 2:4]
test_ary = test_ary[test_ary[:,4]==0, :]
qusers, qucnt = np.unique(np.sort(test_ary[:,2]), return_counts=True)
 qusers = qusers.astype("int32")
 existing_qusers_bol = np.isin(ar_hist[:,0], qusers)
 not_existing_qusers = qusers[~np.isin(qusers, ar_hist[existing_qusers_bol, 0])]
questions = np.unique(test_ary[:,3].astype("int32"))
group_num = test_df.index[:test_ary.shape[0]]
if len(prior_ans)!= 0 and prior_ans is not None and prior_ans is not np.nan:
prior_ans = np.array([a for a in prior_ans if a != -1])
 if 0 < len(prior_ans)and len(prior_ans)== len(prior_test_user_ids)== len(prior_test_devp):
df_prev['answered_correctly'] = prior_ans
upd_history_ans(df_prev)
 if ENSEMBLE:
prev_group = df_prev.groupby('user_id' ).apply(lambda x:(x['question_id'].values,
x['answered_correctly'].values,
x["lagtime_mean"].values))
for prev_user_id in prev_group.index:
if prev_user_id in group.index:
group[prev_user_id] =(np.append(group[prev_user_id][0], prev_group[prev_user_id][0])[-180:],
np.append(group[prev_user_id][1], prev_group[prev_user_id][1])[-180:])
 lags [prev_user_id] =(np.append(lags [prev_user_id], prev_group[prev_user_id][2])[-180:])
 else:
group[prev_user_id] =(prev_group[prev_user_id][0], prev_group[prev_user_id][1])
 lags [prev_user_id] = prev_group[prev_user_id][2]
ans_ary = np.array([prior_test_user_ids,
prior_ans,
prior_test_devp.fillna(9),
prior_test_devm,
prior_test_etime] ).T
ans_ary = ans_ary[ans_ary[:,2]!=9]
if 0 < ans_ary.shape[0]:
ans_ary[:,2] *= ans_ary[:,1]
ans_ary[:,3] = ans_ary[:,2] + np.where(ans_ary[:,1]==0, -1, 0)* ans_ary[:,3]
part_ans_ary = prior_test_part * ans_ary[:,1].reshape(-1,1)
 abi_ans_ary = prior_test_qc * ans_ary[:,1].reshape(-1,1)
 ans_ary = np.hstack([ans_ary,
np.ones(ans_ary.shape[0] ).reshape(-1,1),
part_ans_ary,
prior_test_part,
abi_ans_ary,
prior_test_qc])
 ans_ary = ans_ary[np.argsort(ans_ary[:,0])]
if len(prior_test_user_ids)!= len(np.unique(prior_test_user_ids)) :
_, qacnt = np.unique(ans_ary[:,0], return_counts=True)
 ans_ary = groupby_with_numpy(ans_ary, np.arange(1, ans_ary.shape[1]), qacnt)
 else:
ans_ary = ans_ary[:,1:]
ar_hist[prior_hist_equb, 2:] += ans_ary
if 0 < not_existing_qusers.shape[0]:
ins_exp = np.hstack([not_existing_qusers.reshape(-1,1),
np.zeros(( not_existing_qusers.shape[0], ar_hist.shape[1]-1)) ])
 ar_hist = insert_new_data_to_hist(ar_hist, ins_exp)
 existing_qusers_bol = np.isin(ar_hist[:,0], qusers)
 if 0 < test_ary.shape[0]:
ary = test_ary[:,[2,7]].copy()
ary = ary.astype(int)
 ary = ary[np.argsort(ary[:,0])]
ar_hist[existing_qusers_bol, 1] += groupby_with_numpy(ary, np.array([1]), qucnt ).flatten()
df_hist = pd.DataFrame(ar_hist[existing_qusers_bol], columns=sort_cols)
 df_test = pd.DataFrame(test_ary, columns=r_cols)
 df_test = df_test.merge(df_hist, on="user_id", sort=False)
 df_test = df_test.merge(df_qs[np.isin(ar_qs, questions)], on=["question_id","prior_question_had_explanation"], sort=False)
 df_test["ans_rate"] =(df_test.num_ans / df_test.num_que ).fillna(0)
 df_test["devp_rate"] = df_test.num_devp / df_test.num_que
df_test["devm_rate"] = df_test.num_devm / df_test.num_que
df_test["etime_rate"] = df_test.num_etime / df_test.num_que
ar_study_rate = np.array(df_test.num_exp / df_test.num_que)
 ar_study_rate[np.isnan(ar_study_rate)] = 0
ar_study_rate[np.isinf(ar_study_rate)] = 0
ar_study_rate[1 < ar_study_rate] = 1
df_test["study_rate"] = ar_study_rate
df_class = df_ans_class[np.isin(ar_ans_class, questions)].copy()
df_test = cut_and_merge_with_class(df_test, df_class, "ans_rate", "ans_rate_class")
 df_test = cut_and_merge_with_class(df_test, df_class, "study_rate", "study_rate_class")
 df_test = add_mistake_feat(df_test, "prob_mistake_que", "num_que")
 for p in range(0,10):
if p in [0,3,4,5,6,7,8,9]:
df_test[f"abi_rate{p}"] =(df_test[f"ans_abi{p}"] / df_test[f"num_abi{p}"] ).fillna(0)
 if p in [1,2,3,4,5,6,7]:
df_test[f"part{p}_rate"] =(df_test[f"ans_part{p}"] / df_test[f"num_part{p}"] ).fillna(0)
 df_test = add_mistake_feat(df_test, f"prob_mistake_part{p}", f"num_part{p}")
 df_test = cut_and_merge_with_class(df_test, df_class, f"part{p}_rate", f"part{p}_rate_class")
 for t, n in zip(t_cols, n_cols):
if df_test[n].min() < 5:
df_test = change_rate_with_num(df_test, t, n, method="respectively")
 df_test = change_rate_with_num(df_test, t, n, method="bundle")
 df_test["lagtime1"] =(df_test[["user_id","timestamp"]].apply(lambda x: get_lagtime(x[0], x[1], -1), axis=1)/(1000*3600)).astype("float32")
 df_test["lagtime2"] =(df_test[["user_id","timestamp"]].apply(lambda x: get_lagtime(x[0], x[1], -2), axis=1)/(1000*3600)).astype("float32")
 df_test["lagtime3"] =(df_test[["user_id","timestamp"]].apply(lambda x: get_lagtime(x[0], x[1], -3), axis=1)/(1000*3600)).astype("float32")
 df_test["lagtime_mean"] =(df_test.lagtime1 + df_test.lagtime2 + df_test.lagtime3)/ 3
df_test["timestamp"] =(df_test.timestamp /(1000*3600)).astype("float32")
 df_test["lagtime1"] =(df_test.timestamp - df_test.lagtime1 ).astype("float32")
 df_test["lagtime2"] =(df_test.timestamp - df_test.lagtime2 ).astype("float32")
 df_test["lagtime3"] =(df_test.timestamp - df_test.lagtime3 ).astype("float32")
 df_test["lagtime_mean"] = np.log(( df_test.timestamp - df_test.lagtime_mean)+ 1 ).astype("float32")
 df_test["noplay_days"] =(df_test[["user_id","timestamp"]].apply(lambda x: get_noplay_days(x[0], x[1]), axis=1)).astype("float32")
 df_test["elapsed_days"] =(df_test.timestamp / 24 ).astype("int32")
 df_test["noplay_rate"] =(df_test.noplay_days / df_test.num_que ).astype("float32")
 df_test["timestamp_rate"] =(df_test.timestamp / df_test.num_que ).astype("float32")
 df_test["elapsed_rate"] =(df_test.elapsed_days / df_test.num_que ).astype("float32")
 df_test["attempt_que"] = df_test[["user_id","question_id"]].apply(lambda x: get_attempt_que(x[0], 101230332+idx, x[1]), axis=1)
 df_test["attempt_tags"] = df_test[["user_id","tag1"]].apply(lambda x: get_attempt_tags(x[0], 101230332+idx, x[1]), axis=1)
 df_test["attempt_curtar"] = df_test[["user_id"]].apply(lambda x: get_attempt_curt_ans_rate(x[0], 101230332+idx), axis=1)
 all_prob_mistake = np.array(df_test[part_mistake_cols])
 all_part_rate = np.array(df_test[["part1_rate","part2_rate","part3_rate","part4_rate","part5_rate","part6_rate","part7_rate"]])
 all_part_ratec = np.array(df_test[["part1_rate_class","part2_rate_class","part3_rate_class","part4_rate_class","part5_rate_class","part6_rate_class","part7_rate_class"]])
 all_abi_rate = np.array(df_test[["abi_rate0","abi_rate3","abi_rate4","abi_rate5","abi_rate6","abi_rate7","abi_rate8","abi_rate9"]])
 one_hot_part = np.identity(8)[df_test.part][:,1:]
one_hot_ability = np.identity(10)[df_test.question_rate_class_exp][:,[0,3,4,5,6,7,8,9]]
df_test["prob_mistake_part_curt"] = np.nanmax(one_hot_part*all_prob_mistake, axis=1)
 df_test["part_rate_curt"] = np.nanmax(one_hot_part*all_part_rate, axis=1)
 df_test["part_ratec_curt"] = np.nanmax(one_hot_part*all_part_ratec, axis=1)
 df_test["abi_rate_curt"] = np.nanmax(one_hot_ability*all_abi_rate, axis=1)
 df_test["part_ratec_mean"] = np.nan_to_num(all_part_ratec ).sum(1)/ 7
df_test["harmonic_ar_prc"] = 2 *(df_test.ans_rate * df_test.part_rate_curt)/(df_test.ans_rate + df_test.part_rate_curt)
 df_test["harmonic_arc_qr"] = 2 *(df_test.ans_rate_class * df_test.question_rate_exp)/(df_test.ans_rate_class + df_test.question_rate_exp)
 df_test["harmonic_arc_prcc"] = 2 *(df_test.ans_rate_class * df_test.part_ratec_curt)/(df_test.ans_rate_class + df_test.part_ratec_curt)
 df_test["harmonic_prc_pr"] = 2 *(df_test.part_rate_curt * df_test.part_rate)/(df_test.part_rate_curt + df_test.part_rate)
 df_test["harmonic_prcc_qr"] = 2 *(df_test.part_ratec_curt * df_test.question_rate_exp)/(df_test.part_ratec_curt + df_test.question_rate_exp)
 df_test["harmonic_qr_pr"] = 2 *(df_test.part_rate * df_test.question_rate_exp)/(df_test.part_rate + df_test.question_rate_exp)
 df_test["harmonic_abrc_qr"] = 2 *(df_test.abi_rate_curt * df_test.question_rate_exp)/(df_test.abi_rate_curt + df_test.question_rate_exp)
 df_test["harmonic_arc_src"] = 2 *(df_test.ans_rate_class * df_test.study_rate_class)/(df_test.ans_rate_class + df_test.study_rate_class)
 df_test["harmonic_prcc_src"] = 2 *(df_test.part_ratec_curt * df_test.study_rate_class)/(df_test.part_ratec_curt + df_test.study_rate_class)
 df_test["harmonic_src_qr"] = 2 *(df_test.study_rate_class * df_test.question_rate_exp)/(df_test.study_rate_class + df_test.question_rate_exp)
 df_test["rate_arc_prcc"] = df_test.ans_rate_class / df_test.part_ratec_curt
df_test["rate_arc_src"] = df_test.ans_rate_class / df_test.study_rate_class
df_test["rate_prcc_src"] = df_test.part_ratec_curt / df_test.study_rate_class
df_test["rate_arc_qr"] = df_test.ans_rate_class / df_test.question_rate_exp
df_test["rate_qr_pr"] = df_test.part_rate / df_test.question_rate_exp
df_test["rate_prcc_prccm"] = df_test.part_ratec_curt / df_test.part_ratec_mean
prior_hist_equb = existing_qusers_bol
prior_test_user_ids = np.array(df_test.user_id ).astype("int32")
 prior_test_part = one_hot_part
prior_test_qc = one_hot_ability
prior_test_devp = df_test.deviation_p
prior_test_devm = df_test.deviation_m
prior_test_etime = get_num_etimes(df_test)
 df_test.fillna(0, inplace=True)
 df_test.sort_values("row_id", inplace=True)
 df_test.reset_index(drop=True, inplace=True)
 if idx < 1:
display(test_df)
 display(df_test[features+categoricals])
 reults = np.zeros(df_test.shape[0])
 for model in models:
reults += model.predict(df_test[features+categoricals], num_iteration=model.best_iteration)/ len(models)
 if ENSEMBLE:
outs = []
test_dataset = TestDataset(group, lags, df_test[["user_id","question_id","lagtime_mean"]])
 test_dataloader = DataLoader(test_dataset, batch_size=df_test.shape[0], shuffle=False)
 for item in test_dataloader:
x1 = item[0].to(device ).long()
x2 = item[1].to(device ).float()
target_id = item[2].to(device ).long()
with torch.no_grad() :
output, att_weight = nn_model(x1, x2, target_id)
 outs.extend(torch.sigmoid(output)[:, -1].view(-1 ).data.cpu().numpy())
 if ENSEMBLE:
df_test['answered_correctly'] = S * np.array(outs)+ L * reults
else:
df_test['answered_correctly'] = reults
add_to_timedict(test_df)
 add_to_history(df_test, 101230332+idx)
 df_prev = df_test[['user_id','question_id',"lagtime_mean"]].copy()
df_test = df_test[['row_id', 'answered_correctly']].copy()
df_test.index = group_num
env.predict(df_test )<set_options>
train_df_raw.isnull().values.any()
train_df_raw.isna().values.any()
_ = np.seterr(divide='ignore', invalid='ignore' )<load_pretrained>
train_df = train_df_raw.copy()
X_train = train_df.drop(['label'], 1)
 Y_train = train_df['label']
X_train = X_train.values.reshape(X_train.shape[0], 28, 28, 1 ).astype('float32')/ 255
Y_train = np_utils.to_categorical(Y_train, 10 )
with open('.. /input/file-collected-for-inference/baggingfeat/attempt_no_sum_dict.pickle', 'rb')as handle:
attempt_no_sum_dict = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/user_prior_question_elapsed_time_dict.pickle', 'rb')as handle:
user_prior_question_elapsed_time_dict = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/max_timestamp_u_dict3.pickle', 'rb')as handle:
max_timestamp_u_dict3 = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/max_timestamp_u_dict2.pickle', 'rb')as handle:
max_timestamp_u_dict2 = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/max_timestamp_u_dict.pickle', 'rb')as handle:
max_timestamp_u_dict = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/user_lecture_count_dict.pickle', 'rb')as handle:
user_lecture_count_dict = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/user_lecture_sum_dict.pickle', 'rb')as handle:
user_lecture_sum_dict = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/explanation_count_dict.pickle', 'rb')as handle:
explanation_count_dict = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/explanation_sum_dict.pickle', 'rb')as handle:
explanation_sum_dict = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/task_container_std_dict.pickle', 'rb')as handle:
task_container_std_dict = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/task_container_count_dict.pickle', 'rb')as handle:
task_container_count_dict = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/task_container_sum_dict.pickle', 'rb')as handle:
task_container_sum_dict = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/user_sum_dict.pickle', 'rb')as handle:
user_sum_dict = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/user_count_dict.pickle', 'rb')as handle:
user_count_dict = pickle.load(handle)
 lgb_model = joblib.load('.. /input/file-collected-for-inference/lgbmmodel/lgb.pkl')
 prior_question_elapsed_time_mean = 25423.810042960275
lagtime_mean2,lagtime_mean3,lagtime_mean =(39610585.65514838, 58879590.7244385, 20061832.085456338)
 delta_prior_question_elapsed_time_mean = -4.699396307374765
questions_df = pd.read_csv('.. /input/file-collected-for-inference/baggingfeat/questions_df.csv')
 N=[0.4,0.6]
features_dict = {
'timestamp':'float16',
'user_interaction_count':'int16',
'user_interaction_timestamp_mean':'float32',
'lagtime':'float32',
'lagtime2':'float32',
'lagtime3':'float32',
'content_id':'int16',
'task_container_id':'int16',
'user_lecture_sum':'int16',
'user_lecture_lv':'float16',
'prior_question_elapsed_time':'float32',
'delta_prior_question_elapsed_time':'int32',
'user_correctness':'float16',
'user_uncorrect_count':'int16',
'user_correct_count':'int16',
'content_correctness_std':'float16',
'content_correct_count':'int32',
'content_uncorrect_count':'int32',
'content_elapsed_time_mean':'float16',
'content_had_explanation_mean':'float16',
'content_explation_false_mean':'float16',
'content_explation_true_mean':'float16',
'task_container_correctness':'float16',
'task_container_std':'float16',
'task_container_cor_count':'int32',
'task_container_uncor_count':'int32',
'attempt_no':'int8',
'part':'int8',
'part_correctness_mean':'float16',
'part_correctness_std':'float16',
'part_uncor_count':'int32',
'part_cor_count':'int32',
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
'part_bundle_id':'int32',
'content_sub_bundle':'int8',
'prior_question_had_explanation':'int8',
'explanation_mean':'float16',
'explanation_false_count':'int16',
'explanation_true_count':'int16',
}
features=list(features_dict.keys())
 target = 'answered_correctly'
def get_max_attempt(user_id,content_id):
k =(user_id,content_id)
 if k in attempt_no_sum_dict.keys() :
attempt_no_sum_dict[k]+=1
return attempt_no_sum_dict[k]
attempt_no_sum_dict[k] = 1
return attempt_no_sum_dict[k]<choose_model_class>
def build_cnn() :
model = Sequential()
model.add(Conv2D(32,kernel_size=3,activation='relu',input_shape=(28,28,1)))
 model.add(BatchNormalization())
 model.add(Conv2D(32,kernel_size=3,activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(32,kernel_size=5,strides=2,padding='same',activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(64,kernel_size=3,activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64,kernel_size=3,activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64,kernel_size=5,strides=2,padding='same',activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Flatten())
 model.add(Dense(128, activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Dense(10, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model
MAX_SEQ = 240
DROPOUT=0.1
class FFN(nn.Module):
def __init__(self, state_size = 200, forward_expansion = 1, bn_size = MAX_SEQ - 1, dropout=0.2):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, forward_expansion * state_size)
 self.relu = nn.ReLU()
self.bn = nn.BatchNorm1d(bn_size)
 self.lr2 = nn.Linear(forward_expansion * state_size, state_size)
 self.dropout = nn.Dropout(dropout)
 def forward(self, x):
x = self.relu(self.lr1(x))
x = self.bn(x)
 x = self.lr2(x)
 return self.dropout(x)
 class FFN0(nn.Module):
def __init__(self, state_size = 200, forward_expansion = 1, bn_size = MAX_SEQ - 1, dropout=0.2):
super(FFN0, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, forward_expansion * state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(forward_expansion * state_size, state_size)
 self.layer_normal = nn.LayerNorm(state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 x=self.layer_normal(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class TransformerBlock(nn.Module):
def __init__(self, embed_dim, heads = 8, dropout = DROPOUT, forward_expansion = 1):
super(TransformerBlock, self ).__init__()
self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=heads, dropout=dropout)
 self.dropout = nn.Dropout(dropout)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim, forward_expansion = forward_expansion, dropout=dropout)
 self.ffn0 = FFN0(embed_dim, forward_expansion = forward_expansion, dropout=dropout)
 self.layer_normal_2 = nn.LayerNorm(embed_dim)
 def forward(self, value, key, query, att_mask):
att_output, att_weight = self.multi_att(value, key, query, attn_mask=att_mask)
 att_output = self.dropout(self.layer_normal(att_output + value))
att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x1 = self.ffn0(att_output)
 x = self.dropout(self.layer_normal_2(x + x1 + att_output))
return x.squeeze(-1), att_weight
class Encoder(nn.Module):
def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, num_layers=1, heads = 8):
super(Encoder, self ).__init__()
self.n_skill, self.embed_dim = n_skill, embed_dim
self.embedding = nn.Embedding(2 * n_skill + 1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq - 1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.layers = nn.ModuleList([TransformerBlock(embed_dim, forward_expansion = forward_expansion)for _ in range(num_layers)])
 self.dropout = nn.Dropout(dropout)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = self.dropout(x + pos_x)
 x = x.permute(1, 0, 2)
 e = self.e_embedding(question_ids)
 e = e.permute(1, 0, 2)
 for layer in self.layers:
att_mask = future_mask(e.size(0)).to(device)
 x, att_weight = layer(e, x, x, att_mask=att_mask)
 x = x.permute(1, 0, 2)
 x = x.permute(1, 0, 2)
 return x, att_weight
class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, enc_layers=1, heads = 8):
super(SAKTModel, self ).__init__()
self.encoder = Encoder(n_skill, max_seq, embed_dim, dropout, forward_expansion, num_layers=enc_layers)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
x, att_weight = self.encoder(x, question_ids)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
skills = joblib.load(".. /input/file-collected-for-inference/sakt100/skills.pkl.zip")
 n_skill = len(skills)
 group = joblib.load(".. /input/file-collected-for-inference/sakt100/group.pkl.zip")
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 sakt_model = SAKTModel(n_skill, max_seq=240, embed_dim=256, forward_expansion=1, enc_layers=1, heads=4, dropout=0.1)
 sakt_model.load_state_dict(torch.load(".. /input/v4-fork-of-riiid-sakt-model-full/sakt_model.pt", map_location='cpu'))
sakt_model.to(device)
 sakt_model.eval()
class TestDataset(Dataset):
def __init__(self, samples, test_df, skills, max_seq=MAX_SEQ):
super(TestDataset, self ).__init__()
self.samples = samples
self.user_ids = [x for x in test_df["user_id"].unique() ]
self.test_df = test_df
self.skills = skills
self.n_skill = len(skills)
 self.max_seq = max_seq
def __len__(self):
return self.test_df.shape[0]
def __getitem__(self, index):
test_info = self.test_df.iloc[index]
user_id = test_info["user_id"]
target_id = test_info["content_id"]
q = np.zeros(self.max_seq, dtype=int)
 qa = np.zeros(self.max_seq, dtype=int)
 if user_id in self.samples.index:
q_, qa_ = self.samples[user_id]
seq_len = len(q_)
 if seq_len >= self.max_seq:
q = q_[-self.max_seq:]
qa = qa_[-self.max_seq:]
else:
q[-seq_len:] = q_
qa[-seq_len:] = qa_
x = np.zeros(self.max_seq-1, dtype=int)
 x = q[1:].copy()
x +=(qa[1:] == 1)* self.n_skill
questions = np.append(q[2:], [target_id])
 return x, questions<load_pretrained>
X_test = pd.read_csv(path_test)
 X_test = X_test.values.reshape(X_test.shape[0], 28, 28, 1 ).astype('float32')/ 255
cat_model = CatBoostClassifier()
cat_model.load_model('.. /input/file-collected-for-inference/catmodel/catboost_model' )<choose_model_class>
model = build_cnn()
model.fit(X_train, Y_train, batch_size=64, epochs=16 )
<split><EOS> 
prediction = model.predict_classes(X_test, verbose=0)
 submission = pd.DataFrame({"ImageId": list(range(1,len(prediction)+1)) ,
"Label": prediction})
 submission.to_csv("submission.csv", index=False, header=True )