Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<feature_engineering>
%matplotlib inline
plt.rcParams['figure.figsize'] =(5.0, 4.0)
 plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'
print(os.listdir(".. /input"))
np.random.seed(2)
%%time
if global_test == 2:
old_df = None
for(new_df, sample)in iter_test:
if old_df is not None:
old_df['user_answer'] = np.array(
 [int(x)for x in new_df.iloc[0,9][1:-1].split(', ')], dtype = np.int8)
 old_df['answered_correctly'] = np.array(
 [int(x)for x in new_df.iloc[0,8][1:-1].split(', ')], dtype = np.int8)
 update_arrays(old_df)
 old_df = new_df.iloc[:,:8].copy()
update_user_map(new_df.user_id.unique())
 X = get_features(new_df, True)
 sample['answered_correctly'] =(
 model1.predict_proba(X)[:,1]/2 + model2.predict_proba(X)[:,1]/2 
)
 env.predict(sample )<import_modules>
def convert_to_one_hot(Y, C):
Y = np.eye(C)[Y.reshape(-1)]
return Y
import pandas as pd
import numpy as np
import gc
import pickle
import psutil
import joblib
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
import sqlite3<define_search_space>
def read_csv(filename):
X,Y=[],[]
test=pd.read_csv(filename)
 if filename.find("train.csv")>0:
Y=test.iloc[:,0].values
Y=convert_to_one_hot(Y,10)
 X=test.iloc[:,1:785].values
else:
X=test.iloc[:,0:784].values
return X, Y
c1, c1_2, c2, c3 , c4 = 0.175, 0.075, 0.25, 0.25, 0.25<init_hyperparams>
def write_csv(filename,predictions):
my_submission = pd.DataFrame({'ImageId': range(1,predictions.shape[0]+1), 'Label': predictions})
 my_submission.to_csv('submission.csv', index=False)
MAX_SEQ = 100
n_part = 7
D_MODEL = 256
N_LAYER = 2
DROPOUT = 0.1<feature_engineering>
X_train,Y_train=read_csv(".. /input/train.csv")
 X_test,_=read_csv(".. /input/test.csv")
 m,pixels=X_train.shape
classes=10
height,width,channels=28,28,1
X_train, X_test=X_train/255, X_test/255
X_train=X_train.reshape(-1,height,width,channels)
 X_test=X_test.reshape(-1,height,width,channels)
 print(Y_train.shape,X_train.shape,X_test.shape)
 random_seed = 2
X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, random_state=random_seed)
 print(Y_train.shape,X_train.shape,X_test.shape,X_val.shape,Y_val.shape )
def feature_time_lag(df, time_dict):
tt = np.zeros(len(df), dtype=np.int64)
 for ind, row in enumerate(df[['user_id','timestamp','task_container_id']].values):
if row[0] in time_dict.keys() :
if row[2]-time_dict[row[0]][1] == 0:
tt[ind] = time_dict[row[0]][2]
else:
t_last = time_dict[row[0]][0]
task_ind_last = time_dict[row[0]][1]
tt[ind] = row[1]-t_last
time_dict[row[0]] =(row[1], row[2], tt[ind])
 else:
time_dict[row[0]] =(row[1], row[2], -1)
 tt[ind] = 0
df["time_lag"] = tt
return df<define_search_model>
def DigitalRecognizerModel(input_shape):

 X_input = Input(input_shape)
 X=Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same', activation ='relu', input_shape =(28,28,1))(X_input)
 X=Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same', activation ='relu' )(X)
 X=MaxPooling2D(pool_size=(2,2))(X)
 X=Dropout(0.25 )(X)
 X=Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu' )(X)
 X=Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu' )(X)
 X=MaxPooling2D(pool_size=(2,2), strides=(2,2))(X)
 X=Dropout(0.25 )(X)
 X=Flatten()(X)
 X=Dense(256, activation = "relu" )(X)
 X=Dropout(0.5 )(X)
 X=Dense(10, activation = "softmax" )(X)
 model = Model(inputs = X_input, outputs = X, name='DigitalRecognizer')
 return model
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(DROPOUT)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAINTModel(nn.Module):
def __init__(self, n_skill, n_part, max_seq=MAX_SEQ, embed_dim= 128, elapsed_time_cat_flag = True):
super(SAINTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.n_cat = n_part
self.elapsed_time_cat_flag = elapsed_time_cat_flag
self.e_embedding = nn.Embedding(self.n_skill+1, embed_dim)
 self.c_embedding = nn.Embedding(self.n_cat+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.res_embedding = nn.Embedding(2+1, embed_dim)
 if self.elapsed_time_cat_flag == True:
self.elapsed_time_embedding = nn.Embedding(300+1, embed_dim)
 self.lag_embedding1 = nn.Embedding(300+1, embed_dim)
 self.lag_embedding2 = nn.Embedding(1440+1, embed_dim)
 self.lag_embedding3 = nn.Embedding(365+1, embed_dim)
 else:
self.elapsed_time_embedding = nn.Linear(1, embed_dim, bias=False)
 self.lag_embedding = nn.Linear(1, embed_dim, bias=False)
 self.exp_embedding = nn.Embedding(2+1, embed_dim)
 self.transformer = nn.Transformer(nhead=8, d_model = embed_dim, num_encoder_layers= N_LAYER, num_decoder_layers= N_LAYER, dropout = DROPOUT)
 self.dropout = nn.Dropout(DROPOUT)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, question, part, response, elapsed_time, lag_time, exp):
device = question.device
question = self.e_embedding(question)
 part = self.c_embedding(part)
 pos_id = torch.arange(question.size(1)).unsqueeze(0 ).to(device)
 pos_id = self.pos_embedding(pos_id)
 res = self.res_embedding(response)
 exp = self.exp_embedding(exp)
 if self.elapsed_time_cat_flag == True:
elapsed_time = torch.true_divide(elapsed_time, 1000)
 elapsed_time = torch.round(elapsed_time)
 elapsed_time = torch.where(elapsed_time.float() <= 300, elapsed_time, torch.tensor(300.0 ).to(device)).long()
elapsed_time = self.elapsed_time_embedding(elapsed_time)
 lag_time = torch.true_divide(lag_time, 1000)
 lag_time = torch.round(lag_time)
 lag_time1 = torch.where(lag_time.float() <= 300, lag_time, torch.tensor(300.0 ).to(device)).long()
lag_time = torch.true_divide(lag_time, 60)
 lag_time = torch.round(lag_time)
 lag_time2 = torch.where(lag_time.float() <= 1440, lag_time, torch.tensor(1440.0 ).to(device)).long()
lag_time = torch.true_divide(lag_time, 1440)
 lag_time = torch.round(lag_time)
 lag_time3 = torch.where(lag_time.float() <= 365, lag_time, torch.tensor(365.0 ).to(device)).long()
lag_time1 = self.lag_embedding1(lag_time1)
 lag_time2 = self.lag_embedding2(lag_time2)
 lag_time3 = self.lag_embedding3(lag_time3)
 else:
elapsed_time = elapsed_time.view(-1,1)
 elapsed_time = self.elapsed_time_embedding(elapsed_time)
 elapsed_time = elapsed_time.view(-1, MAX_SEQ-1, self.embed_dim)
 lag_time = lag_time.view(-1,1)
 lag_time = self.lag_embedding(lag_time)
 lag_time = lag_time.view(-1, MAX_SEQ-1, self.embed_dim)
 enc = question + part + pos_id + exp
dec = pos_id + res + elapsed_time + lag_time1 + lag_time2 + lag_time3
enc = enc.permute(1, 0, 2)
 dec = dec.permute(1, 0, 2)
 mask = future_mask(enc.size(0)).to(device)
 att_output = self.transformer(enc, dec, src_mask=mask, tgt_mask=mask, memory_mask = mask)
 att_output = self.layer_normal(att_output)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 x = self.pred(x)
 return x.squeeze(-1 )<load_from_csv>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )
n_skill = 13523
group = joblib.load(".. /input/saint-plus-data-new/group_20210102.pkl.zip")
 questions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')
 time_dict = joblib.load(".. /input/saint-plus-data-new/time_dict.pkl.zip" )<load_pretrained>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model1 = SAINTModel(n_skill, n_part, embed_dim= D_MODEL)
 try:
model1.load_state_dict(torch.load(".. /input/saint-plus-model/saint_plus_model_20210102_padding_v2.pt"))
except:
model1.load_state_dict(torch.load(".. /input/saint-plus-model/saint_plus_model_20210102_padding_v2.pt", map_location='cpu'))
model1.to(device)
 model1.eval()<load_pretrained>
digitalRecognizerModel = DigitalRecognizerModel(X_train[0].shape)
 digitalRecognizerModel.compile(optimizer = "Adam", loss = "binary_crossentropy", metrics = ["accuracy"])
 history = digitalRecognizerModel.fit_generator(datagen.flow(X_train,Y_train, batch_size=62),
epochs = 30, validation_data =(X_val,Y_val),
verbose = 2, steps_per_epoch=610,callbacks=[learning_rate_reduction] )
<define_variables><EOS> 
print("Time Start:" ,time.time())
 val_predictions=digitalRecognizerModel.predict(X_val)
 correct_val_predictions=np.mean(np.equal(np.argmax(val_predictions,axis=1), np.argmax(Y_val,axis=1)))
 print("Validation Accuracy",correct_val_predictions)
 test_predictions=digitalRecognizerModel.predict(X_test)
 correct_test_predictions=np.argmax(test_predictions,axis=1)
 write_csv('submission.csv',correct_test_predictions)
 print("Time End:" ,int(round(time.time())) )