Kaggle competition: Digit Recognizer
class PositionalEncoding(nn.Module):
def __init__(self, d_model, dropout=0.1, max_len=5000):
super(PositionalEncoding, self ).__init__()
self.dropout = nn.Dropout(p=dropout)
 self.scale = nn.Parameter(torch.ones(1))
pe = torch.zeros(max_len, d_model)
 position = torch.arange(0, max_len, dtype=torch.float ).unsqueeze(1)
 div_term = torch.exp(torch.arange(
 0, d_model, 2 ).float() *(-math.log(10000.0)/ d_model))
pe[:, 0::2] = torch.sin(position * div_term)
 pe[:, 1::2] = torch.cos(position * div_term)
 pe = pe.unsqueeze(0 ).transpose(0, 1)
 self.register_buffer('pe', pe)
 def forward(self, x):
x = x + self.scale * self.pe[:x.size(0), :]
return self.dropout(x )<categorify>
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
import tensorflow as tf
from keras.utils import to_categorical
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import EarlyStopping
class EmbedTag(nn.Module):
def __init__(self, d_model, que_arr, tags_n):
super(EmbedTag, self ).__init__()
self.que_arr = torch.LongTensor(que_arr ).to(device)
 self.embedding = nn.Embedding(tags_n, d_model)
 def forward(self, x):
x = self.que_arr[x, :]
x = self.embedding(x)
 return torch.sum(x, dim=-2 )<categorify>
train_data = pd.read_csv('.. /input/digit-recognizer/train.csv')
 test_data = pd.read_csv('.. /input/digit-recognizer/test.csv' )
class TransformerModel(nn.Module):
def __init__(self, intoken, outtoken, hidden, que_arr, part_arr, difficulty, enc_layers=4, dec_layers=4, dropout=0.1, ts_unique=70, prior_unique=50):
super(TransformerModel, self ).__init__()
nhead = hidden//64
self.encoder = nn.Embedding(intoken, hidden)
 self.pos_encoder = PositionalEncoding(hidden, dropout)
 self.decoder = nn.Embedding(outtoken, hidden)
 self.pos_decoder = PositionalEncoding(hidden, dropout)
 self.tagsEmbedder = EmbedTag(hidden, que_arr, tags_n)
 self.transformer = nn.Transformer(d_model=hidden, nhead=nhead, num_encoder_layers=enc_layers, num_decoder_layers=dec_layers, dim_feedforward=hidden*4, dropout=dropout, activation='relu')
 self.fc_out = nn.Linear(hidden, 1)
 self.src_mask = None
self.trg_mask = None
self.memory_mask = None
self.part_embedding = nn.Embedding(7,hidden)
 self.part_arr = part_arr
self.ts_embedding = nn.Embedding(ts_unique, hidden)
 self.prior_embedding = nn.Embedding(prior_unique, hidden)
 self.task_container_embedding = nn.Embedding(10000, hidden)
 self.user_answer_embedding = nn.Embedding(5, hidden)
 self.difficulty = difficulty
self.difficulty_embedding = nn.Embedding(11, hidden)
 self.dropout_7 = nn.Dropout(dropout)
 self.dropout_1 = nn.Dropout(dropout)
 self.dropout_2 = nn.Dropout(dropout)
 self.dropout_3 = nn.Dropout(dropout)
 self.dropout_4 = nn.Dropout(dropout)
 self.dropout_5 = nn.Dropout(dropout)
 self.dropout_6 = nn.Dropout(dropout)
 self.explan_embedding = nn.Embedding(3, hidden)
 self.dropout_9 = nn.Dropout(dropout)
 def generate_square_subsequent_mask(self, sz, sz1=None):
if sz1 == None:
mask = torch.triu(torch.ones(sz, sz), 1)
 else:
mask = torch.triu(torch.ones(sz, sz1), 1)
 return mask.masked_fill(mask==1, float('-inf'))
def make_len_mask_a(self, inp):
return(inp == a_pad ).transpose(0, 1)
 def make_len_mask_q(self, inp):
return(inp == q_pad ).transpose(0, 1)
 def forward(self, src, trg, ts, prior, task_container, user_answer, explan):
if self.trg_mask is None or self.trg_mask.size(0)!= len(trg):
self.trg_mask = self.generate_square_subsequent_mask(len(trg)).to(trg.device)
 if self.src_mask is None or self.src_mask.size(0)!= len(src):
self.src_mask = self.generate_square_subsequent_mask(len(src)).to(trg.device)
 if self.memory_mask is None or self.memory_mask.size(0)!= len(trg)or self.memory_mask.size(1)!= len(src):
self.memory_mask = self.generate_square_subsequent_mask(len(trg),len(src)).to(trg.device)
 src_pad_mask = self.make_len_mask_q(src)
 trg_pad_mask = self.make_len_mask_a(trg)
 part_emb = self.dropout_1(self.part_embedding(self.part_arr[src]-1))
ts_emb = self.dropout_3(self.ts_embedding(ts))
user_answer_emb = self.dropout_5(self.user_answer_embedding(user_answer))
src = self.encoder(src)
 src = torch.add(src, part_emb)
 src = torch.add(src, ts_emb)
 src = self.pos_encoder(src)
 trg = self.decoder(trg)
 trg = torch.add(trg, user_answer_emb)
 trg = self.pos_decoder(trg)
 output = self.transformer(src, trg, src_mask=self.src_mask, tgt_mask=self.trg_mask, memory_mask=self.memory_mask,
src_key_padding_mask=src_pad_mask, tgt_key_padding_mask=trg_pad_mask, memory_key_padding_mask=src_pad_mask)
 output = self.fc_out(output)
 return output<load_pretrained>
X_train = train_data.drop(labels = ["label"],axis = 1)
 Y_train = train_data["label"]
Y_train = to_categorical(Y_train, num_classes = 10 )
d_model = 128
INPUT_DIM = q_pad+1
OUTPUT_DIM = 4
model_saint = TransformerModel(INPUT_DIM, OUTPUT_DIM, hidden=d_model, que_arr=que_arr,part_arr=part_valus, difficulty=difficulty ).to(device)
 weights = torch.load(".. /input/last-saint/last.torch", map_location=torch.device(device))
model_saint.load_state_dict(weights)
 model_saint.to(device)
 model_saint.eval()<define_variables>
X_train, X_test, Y_train, Y_test = train_test_split(X_train, Y_train, test_size = 0.2, random_state = 42 )
def pred_users(vals):
eval_batch = vals.shape[0]
tensor_question = np.zeros(( eval_batch, 100), dtype=np.long)
 tensor_answers = np.zeros(( eval_batch, 100), dtype=np.long)
 tensor_ts = np.zeros(( eval_batch, 100), dtype=np.long)
 tensor_user_answer = np.zeros(( eval_batch, 100), dtype=np.long)
 val_len = []
preds = []
group_index = group.index
for i, line in enumerate(vals):
if line[2] == True:
val_len.append(0)
 continue
user_id = line[0]
question_id = line[1]
timestamp = get_timestamp(line[4], user_id)
 prior = get_prior(line[5])
 task_container_id = line[3]
que_history = np.array([], dtype=np.int32)
 answers_history = np.array([], dtype=np.int32)
 ts_history = np.array([], dtype=np.int32)
 user_answer_history = np.array([], dtype=np.int32)
 if user_id in group_index:
cap = 99
que_history, answers_history, ts_history, user_answer_history = group[user_id]
que_history = que_history[-cap:]
answers_history = answers_history[-cap:]
ts_history = ts_history[-cap:]
user_answer_history = user_answer_history[-cap:]
a_token = 2
user_a_token = 4
answers_history = np.concatenate(( [a_token],answers_history))
user_answer_history = np.concatenate(( [user_a_token],user_answer_history))
que_history = np.concatenate(( que_history, [question_id]))
ts_history = np.concatenate(( ts_history, [timestamp]))
tensor_question[i][:len(que_history)] = que_history
tensor_answers[i][:len(que_history)] = answers_history
tensor_ts[i][:len(que_history)] = ts_history
tensor_user_answer[i][:len(que_history)] = user_answer_history
val_len.append(len(que_history))
tensor_question = torch.from_numpy(tensor_question ).long().T.to(device)
 tensor_answers = torch.from_numpy(tensor_answers ).long().T.to(device)
 tensor_ts = torch.from_numpy(tensor_ts ).long().T.to(device)
 tensor_user_answer = torch.from_numpy(tensor_user_answer ).long().T.to(device)
 with torch.no_grad() :
out = F.sigmoid(model_saint(tensor_question, tensor_answers, tensor_ts, None, None, tensor_user_answer,2)).squeeze(dim=-1 ).T
for j in range(len(val_len)) :
preds.append(out[j][val_len[j]-1].item())
 return preds<string_transform>
train_datagen = ImageDataGenerator(rescale = 1./255, shear_range = 0.2, zoom_range = 0.2 )
def split_preds(preds):
if preds.shape[0] > 1000:
ret = []
for i in np.array_split(preds, preds.shape[0]//1000):
ret.extend(pred_users(i))
return ret
else:
return pred_users(preds )<prepare_x_and_y>
test_datagen = ImageDataGenerator(rescale = 1./255 )
def update_group_var(vals):
global group
for i, line in enumerate(vals):
user_id = line[0]
question_id = line[1]
content_type_id = line[2]
ts = get_timestamp(line[4], user_id)
 correct = line[6]
user_answer = line[7]
if content_type_id == True:
continue
if st_user_info.get(user_id, -1)== -1:
st_user_info[user_id] = {"timestamp_ms":0}
else:
st_user_info[user_id]["timestamp_ms"] = line[4]
if user_id in group.index:
questions= np.append(group[user_id][0],[question_id])
 answers= np.append(group[user_id][1],[correct])
 ts= np.append(group[user_id][2],[ts])
 user_answer= np.append(group[user_id][3],[user_answer])
 group[user_id] =(questions, answers, ts, user_answer)
 else:
group[user_id] =(np.array([question_id], dtype=np.int32), np.array([correct], dtype=np.int32), np.array([ts], dtype=np.int32)
 ,np.array([user_answer], dtype=np.int32))<define_variables>
train = train_datagen.flow(X_train, Y_train, batch_size = 128 )
ordinal_enc = {0: 0, 1: 1, 2: 2, 3: 3, 4: 4, 5: 5, 6: 6, 7: 7, 8: 8, 9: 9, 10: 10, 11: 11, 12: 12, 13: 13, 14: 14, 15: 15, 16: 16, 17: 17, 18: 18, 19: 19, 20: 20, 30: 21, 40: 22, 50: 23, 60: 24, 70: 25, 80: 26, 90: 27, 100: 28, 110: 29, 120: 30, 130: 31, 140: 32, 150: 33, 160: 34, 170: 35, 180: 36, 190: 37, 200: 38, 210: 39, 220: 40, 230: 41, 240: 42, 250: 43, 260: 44, 270: 45, 280: 46, 290: 47, 300: 48}
boundaries = [120,600,1800,3600,10800,43200,86400,259200,604800]
def get_prior(prior):
if prior != prior:
return 0
prior /= 1000
if prior > 20:
prior = np.round(prior, decimals=-1)
 return ordinal_enc.get(round(prior),0)
 def get_timestamp(ts, user_id):
if st_user_info.get(user_id, -1)== -1:
return 0
diff =(ts - st_user_info[user_id]["timestamp_ms"])/1000
if diff < 0:
return 0
if diff <= 60:
return int(diff)
 for i, boundary in enumerate(boundaries):
if boundary > diff:
break
if i == len(boundaries)- 1:
return 60+i+1
return 60+i<define_variables>
test = test_datagen.flow(X_test, Y_test, batch_size = 128 )
prior_part_mean_dict = {1: 22166.159642501425,
2: 18714.69673913695,
3: 23620.317746179924,
4: 23762.753651169547,
5: 25094.620302855932,
6: 32417.37918735745,
7: 47444.16407400242}<load_pretrained>
callback = EarlyStopping(monitor='loss', patience=8, restore_best_weights=True )
with open('.. /input/lgbm-test/repeated_que_count', 'rb')as handle:
repeated_que_count = pickle.load(handle)
 with open('.. /input/lgbm-test/user_info', 'rb')as handle:
user_info = pickle.load(handle)
 with open('.. /input/lgbm-test/watched_tags', 'rb')as handle:
watched_tags = pickle.load(handle)
 with open('.. /input/lgbm-test/containers_mean', 'rb')as handle:
containers_mean = pickle.load(handle)
 with open('.. /input/lgbm-test/hardest', 'rb')as handle:
hard_questions = pickle.load(handle)
 with open('.. /input/lgbm-test/easiest', 'rb')as handle:
easy_questions = pickle.load(handle)
 with open('.. /input/lgbm-test/que_2_k', 'rb')as handle:
que_2_k = pickle.load(handle)
 gc.collect() 
<feature_engineering>
cnn = tf.keras.models.Sequential()
for u in user_info:
user_info[u]["count_2"] = user_info[u]["count"]
user_info[u]["part_count_2"] = user_info[u]["part_count"].copy()
user_info[u]["last_part"] = 1<init_hyperparams>
cnn.add(tf.keras.layers.Conv2D(filters = 32, kernel_size = 5, padding = 'same', activation = 'relu', input_shape = [28, 28, 1]))
groups = pd.read_pickle(".. /input/lgbm-test/groups")
 def numpy_ewma_vectorized_v2(data, window):
alpha = 2 /(window + 1.0)
 alpha_rev = 1-alpha
n = data.shape[0]
pows = alpha_rev**(np.arange(n+1))
scale_arr = 1/pows[:-1]
offset = data[0]*pows[1:]
pw0 = alpha*alpha_rev**(n-1)
 mult = data*pw0*scale_arr
cumsums = mult.cumsum()
out = offset + cumsums*scale_arr[::-1]
return out<categorify>
cnn.add(tf.keras.layers.MaxPool2D(pool_size = 2, strides = 2, padding = 'valid'))
features_1_path = '.. /input/get-features-1/'
que_data = pd.read_pickle(features_1_path + "que_data.pickle")
 questions = que_data.drop(columns=["options_number","correctness_number", "correct_answer","tags6","tags5", "tags4"] ).to_dict("index")
 questions1 = que_data[["tags1", "tags2", "tags3","tags4","tags5", "tags6"]].to_dict("index")
 parts = que_data.part.to_dict()<load_from_csv>
cnn.add(tf.keras.layers.Conv2D(filters = 64, kernel_size = 3, padding = 'same'))
cnn.add(tf.keras.layers.MaxPool2D(pool_size = 2, strides = 2, padding='valid'))
lec_data = pd.read_csv(".. /input/riiid-test-answer-prediction/lectures.csv")
 lec_dict = lec_data[["lecture_id", "tag"]].set_index("lecture_id" ).tag.to_dict()
features = [
'task_container_id', "ts_diff_shifted", "watched","ts_diff_shifted_2",
'content_id', "k", "k_acc", "el_avg", "wut",
'prior_question_elapsed_time', "time_diff2", "rolling_mean_5", "rolling_mean_10", "rolling_mean_15", "prior_question_had_explanation_u_part_avg",
'prior_question_had_explanation', "hard_ratio_opp", "easy_ratio_opp", "correct_recency", "prior_question_elapsed_time_u_part_avg", "ewm_mean_10", "rolling_mean_5_prior_question",
'last_lecture', "part_mean", "opp_mean", "mean_pause", "timestamp", "prior_part_mean",
"container_mean", "lecs_per", "hard_ratio", "easy_ratio",
'que_count_user', 'question_repeated', "rolling_mean","time_diff3", "time_diff4",
'user_mean', "time_diff1", "time_diff", "sessions", "session_count", "prior_question_had_explanation_ratio"
] + que_data.columns.tolist() [:-1]
features.remove("options_number")
 features.remove("correct_answer")
 features.remove("tags6")
 features.remove("tags5")
 features.remove("tags4")
 test_cols = ['row_id','timestamp','user_id','content_id','content_type_id','task_container_id','prior_question_elapsed_time',
'prior_question_had_explanation','prior_group_answers_correct','prior_group_responses'] 
<define_variables>
cnn.add(tf.keras.layers.MaxPool2D(pool_size = 2, strides = 2, padding='valid'))
cnn.add(Droupout(0.5))
stack_features = [
'task_container_id', "ts_diff_shifted", "watched","ts_diff_shifted_2",
'content_id', "k", "k_acc", "el_avg", "wut", "lgb_preds", "st_preds",
'prior_question_elapsed_time', "time_diff2", "rolling_mean_5", "rolling_mean_10", "rolling_mean_15", "prior_question_had_explanation_u_part_avg",
'prior_question_had_explanation', "hard_ratio_opp", "easy_ratio_opp", "correct_recency", "prior_question_elapsed_time_u_part_avg", "ewm_mean_10", "rolling_mean_5_prior_question",
'last_lecture', "part_mean", "opp_mean", "mean_pause", "timestamp", "prior_part_mean",
"container_mean", "lecs_per", "hard_ratio", "easy_ratio",
'que_count_user', 'question_repeated', "rolling_mean","time_diff3", "time_diff4",
'user_mean', "time_diff1", "time_diff", "sessions", "session_count", "prior_question_had_explanation_ratio"
] + que_data.columns.tolist() [:-1]
stack_features.remove("options_number")
 stack_features.remove("correct_answer")
 stack_features.remove("tags6")
 stack_features.remove("tags5")
 stack_features.remove("tags4")
 lgb_preds_idx = stack_features.index("lgb_preds")
 st_preds_idx = stack_features.index("st_preds" )<define_variables>
cnn.add(tf.keras.layers.Flatten() )
k_size = 20
cols = {test_cols[k]:k for k in range(len(test_cols)) }
features_dict = {features[k]:k for k in range(len(features)) }<init_hyperparams>
cnn.add(tf.keras.layers.Dense(units=256, activation='relu'))
new_user = {'count': 0, 'mean_acc':0.5, 'correct_count': 0, 'last_lec':0, 'tmp':0,"first_timestamp":0, "second_timestamp":0,
"third_timestamp":0, "fourth_timestamp":0, "fifth_timestamp":0, "lecs_n":0,"interaction_n":0, "ts_diff_shifted":0.,
"part_corr":np.zeros(( 7), dtype=np.uint16), "part_count":np.zeros(( 7), dtype=np.uint16), "hard_ct":0, "hard_cr":0, "easy_ct":0, "easy_cr":0,
"sessions":0, "session_count":0, "sum_pauses":0., "had_exp":0, "el_sum":0, "part_et": np.zeros(( 7), dtype=np.float),
"part_explan": np.zeros(( 7), dtype=np.uint16), "k_count": np.zeros(( k_size), dtype=np.uint16), "k_corr": np.zeros(( k_size), dtype=np.uint16),
"recent_corr":0, "priors_5": [], "ts_diff_shifted_2":0., "count_2":0, "part_count_2":np.zeros(( 7), dtype=np.uint16), "last_part":1}<define_variables>
cnn.add(tf.keras.layers.Dense(units=10, activation='softmax'))
def get_meta_data(data_1):
user_id = data_1[cols['user_id']]
content_type_id = data_1[cols['content_type_id']]
content_id = data_1[cols['content_id']]
prior_group_answers_correct = data_1[cols['prior_group_answers_correct']]
timestamp = data_1[cols['timestamp']]
task_container_id = data_1[cols['task_container_id']]
prior_question_had_explanation = data_1[cols['prior_question_had_explanation']]
elapsed = data_1[cols['prior_question_elapsed_time']]
return user_id, content_type_id, content_id, prior_group_answers_correct, timestamp,task_container_id,prior_question_had_explanation, elapsed<feature_engineering>
cnn.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['accuracy'] )
def add_user(user_id):
user_info[user_id] = copy.deepcopy(new_user)
 repeated_que_count[user_id] = {}
groups[user_id] = []<feature_engineering>
model = cnn.fit_generator(train, epochs = 100, validation_data = test, callbacks = [callback] )
def update_user_part_acc(user_id, question, answered_correctly, elapsed, explan):
part = parts.get(question, -1)
 user_info[user_id]["part_count"][part-1] += 1
user_info[user_id]["part_corr"][part-1] += answered_correctly
if not isinstance(explan, pd._libs.missing.NAType)and explan == explan:
user_info[user_id]["had_exp"] += explan<feature_engineering>
cnn.evaluate(X_test,Y_test,verbose=2 )
def update_user(user_id, had_exp, elapsed, content_id ,answered_correctly, timestamp):
user_info[user_id]['count'] += 1
if repeated_que_count[user_id].get(content_id, -1)== -1:
repeated_que_count[user_id][content_id] = 1
else:
repeated_que_count[user_id][content_id] += 1
if answered_correctly:
user_info[user_id]['correct_count'] += 1
user_info[user_id]['mean_acc'] = user_info[user_id]['correct_count']/user_info[user_id]['count']
update_user_part_acc(user_id, content_id, answered_correctly, elapsed, had_exp)
 if hard_questions.get(content_id, False):
user_info[user_id]["hard_ct"] += 1
user_info[user_id]["hard_cr"] += answered_correctly
if easy_questions.get(content_id, False):
user_info[user_id]["easy_ct"] += 1
user_info[user_id]["easy_cr"] += answered_correctly
k = que_2_k[content_id]
user_info[user_id]["k_count"][k] += 1
user_info[user_id]["k_corr"][k] += answered_correctly
if answered_correctly:
user_info[user_id]["recent_corr"] = timestamp<feature_engineering>
test_data /= 256
test_data = test_data.values.reshape(-1,28,28,1)
 results = cnn.predict(test_data )
def update_lec_data(user_id, content_id):
if watched_tags.get(str(user_id), -1)== -1:
watched_tags[str(user_id)] = {}
if user_info.get(user_id, -1)== -1:
add_user(user_id)
 user_info[user_id]["lecs_n"] += 1
lec_tag = lec_dict[content_id]
watched_tags[str(user_id)][str(lec_tag)] = 1
user_info[user_id]['last_lec'] = content_id<feature_engineering>
submission = pd.concat([pd.Series(range(1,28001),name = 'ImageId'),results],axis = 1 )
def non_lag_update(user_id, timestamp, elapsed, explan, lec):
timestamp = timestamp/8.64e+7
diff_timestamp_1 = timestamp - user_info[user_id]["first_timestamp"]
diff_timestamp_2 = timestamp - user_info[user_id]["second_timestamp"]
diff_timestamp_3 = timestamp - user_info[user_id]["third_timestamp"]
diff_timestamp_4 = timestamp - user_info[user_id]["fourth_timestamp"]
diff_timestamp_5 = timestamp - user_info[user_id]["fifth_timestamp"]
user_info[user_id]["fifth_timestamp"] = user_info[user_id]["fourth_timestamp"]
user_info[user_id]["fourth_timestamp"] = user_info[user_id]["third_timestamp"]
user_info[user_id]["third_timestamp"] = user_info[user_id]["second_timestamp"]
user_info[user_id]["second_timestamp"] = user_info[user_id]["first_timestamp"]
user_info[user_id]["first_timestamp"] = timestamp
if(user_info[user_id]["second_timestamp"] - user_info[user_id]["third_timestamp"])!= 0:
user_info[user_id]["ts_diff_shifted"] = user_info[user_id]["second_timestamp"]*8.64e+7 - user_info[user_id]["third_timestamp"]*8.64e+7
if(user_info[user_id]["second_timestamp"] - user_info[user_id]["fourth_timestamp"])!= 0:
user_info[user_id]["ts_diff_shifted_2"] = user_info[user_id]["second_timestamp"]*8.64e+7 - user_info[user_id]["fourth_timestamp"]*8.64e+7
if not isinstance(explan, pd._libs.missing.NAType)and explan == explan and not lec:
user_info[user_id]["priors_5"].append(explan)
 if diff_timestamp_1 > 0.083:
user_info[user_id]["sessions"] += 1
user_info[user_id]["session_count"] = 0
user_info[user_id]["sum_pauses"] += diff_timestamp_1
user_info[user_id]["session_count"] += 1
return diff_timestamp_1, diff_timestamp_2, diff_timestamp_3, diff_timestamp_4, diff_timestamp_5<prepare_x_and_y>
submission.to_csv('./submission.csv',index = False )