Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<define_variables>
import numpy as np
import pandas as pd
from keras.utils.np_utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense,MaxPooling2D,Flatten,Dropout,MaxPool2D
from keras.layers.convolutional import Conv2D
from keras import backend
from keras.preprocessing.image import ImageDataGenerator
from keras.optimizers import SGD
from keras.optimizers import RMSprop
from keras.losses import categorical_crossentropy
from keras.callbacks import ReduceLROnPlateau
from seaborn import countplot
MAX_SEQ = 256<load_from_csv>
input_neurons=784
output_neurons=10
%%time
dtype = {'timestamp' :'int64',
'user_id' :'int32' ,
'content_id' :'int16',
'content_type_id' :'int8',
'answered_correctly':'int8'}
train_df = pd.read_csv('.. /input/riiid-test-answer-prediction/train.csv', usecols=[1, 2, 3, 4, 7], dtype=dtype)
 train_df.head()<load_from_csv>
np.random.seed(2 )
dtype = {
'question_id' :'int16',
'bundle_id' :'int16',
'tags' :'str',
}
questions_df = pd.read_csv('.. /input/riiid-test-answer-prediction/questions.csv', usecols=[0, 1, 4], dtype=dtype)
 questions_df['tags'] = questions_df['tags'].map(lambda s: [int(v)for v in s.split() ], 'ignore')
 questions_df['tags'] = questions_df['tags'].fillna(0)
 questions_df['tags'] = questions_df['tags'].map(lambda s: [0]*5 if s==0 else s*(5//len(s)) +s[:5%len(s)] if len(s)<5 else s[:5])
 questions_df.head()<sort_values>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )
train_df = train_df[train_df.content_type_id == False]
train_df = train_df.sort_values(['timestamp'], ascending=True ).reset_index(drop = True )<count_unique_values>
xtrain_df=train_df.drop(['label'], axis=1)
 ytrain_df=train_df.label
xtest_df = test_df
known_bundle_ids = set(questions_df["bundle_id"].unique())
 print("number bundles =", len(known_bundle_ids))
n_bundles = max(known_bundle_ids)+ 1
n_bundles<count_unique_values>
del train_df
del test_df
known_qtn_ids = set(questions_df["question_id"].unique())
 print("number questions =", len(known_qtn_ids))
n_skill = max(known_qtn_ids)+ 1
n_skill<data_type_conversions>
ytrain_df.value_counts()
known_tags = questions_df["tags"].apply(pd.Series ).unstack().reset_index().dropna()
known_tags = set(known_tags[0].astype('int16' ).unique())
 n_tags = max(known_tags)+ 1
n_tags, len(known_tags )<merge>
ytrain_df = to_categorical(ytrain_df, output_neurons )
gc.collect()
train_df.head()<groupby>
def normalize(df):
df.astype(np.float32)
 df=df/255.0
return df
xtrain_df=normalize(xtrain_df)
 xtest_df=normalize(xtest_df )
train_df_1 = train_df[:train_df.shape[0]//2]
train_df_2 = train_df[train_df.shape[0]//2:]
del train_df; gc.collect()
group = train_df_1[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values, 
))
del train_df_1
gc.collect()
group<groupby>
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(output_neurons, activation = "softmax"))
group_2 = train_df_2[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values, 
))
del train_df_2
gc.collect()
group_2<concatenate>
learning_rate_reduction = ReduceLROnPlateau(monitor='acc',patience=3,verbose=1,factor=0.5, min_lr=0.00001 )
group_2_set = set(group_2.index)
 for k in group.index:
if k in group_2_set:
group[k] =(np.append(group[k][0], group_2[k][0]), np.append(group[k][1], group_2[k][1]))
group_2.pop(k)
 gc.collect()
group = group.append(group_2)
 del group_2; gc.collect()
print('len(group)=', len(group))
group<define_variables>
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
random.seed(1 )<define_variables>
model.compile(
 loss=categorical_crossentropy,
optimizer=optimizer,
metrics=['accuracy']
)
class SAKTDataset(Dataset):
def __init__(self, group, n_skill, max_seq=MAX_SEQ):
super(SAKTDataset, self ).__init__()
self.max_seq = max_seq
self.n_skill = n_skill
self.n_bundles = n_bundles
self.n_tags = n_tags
self.samples = group
self.user_ids = []
for user_id in group.index:
q, qa = group[user_id]
if len(q)< 2:
continue
self.user_ids.append(user_id)
 return
def __len__(self):
return len(self.user_ids)
 def __getitem__(self, index):
user_id = self.user_ids[index]
q_, qa_ = self.samples[user_id]
q_b_t_df = pd.DataFrame.from_dict({'question_id': q_})
 q_b_t_df = q_b_t_df.merge(questions_df, on='question_id', how='left')
 b_ = q_b_t_df.bundle_id.values
t_ = np.vstack(q_b_t_df.tags.values)
 del q_b_t_df
seq_len = len(q_)
 q = np.zeros(self.max_seq, dtype=int)
 qa = np.zeros(self.max_seq, dtype=int)
 b = np.zeros(self.max_seq, dtype=int)
 t = np.array([[0]*5]*self.max_seq, dtype=int)
 if seq_len >= self.max_seq:
if random.random() >0.1:
start = random.randint(0,(seq_len-self.max_seq))
end = start + self.max_seq
q[:] = q_[start:end]
qa[:] = qa_[start:end]
b[:] = b_[start:end]
t[:] = t_[start:end]
else:
q[:] = q_[-self.max_seq:]
qa[:] = qa_[-self.max_seq:]
b[:] = b_[-self.max_seq:]
t[:] = t_[-self.max_seq:]
else:
if random.random() >0.1:
start = 0
end = random.randint(2,seq_len)
 seq_len = end - start
q[-seq_len:] = q_[0:seq_len]
qa[-seq_len:] = qa_[0:seq_len]
b[-seq_len:] = b_[0:seq_len]
t[-seq_len:] = t_[0:seq_len]
else:
q[-seq_len:] = q_
qa[-seq_len:] = qa_
b[-seq_len:] = b_
t[-seq_len:] = t_
target_id = q[1:]
label = qa[1:]
x = np.zeros(self.max_seq-1, dtype=int)
 x = q[:-1].copy()
x +=(qa[:-1] == 1)* self.n_skill
bundle_x = b[:-1].copy()
tags_x = t[:-1].copy()
b_target = b[1:]
t_target = t[1:]
return x, target_id, label, bundle_x, tags_x, b_target, t_target<define_search_model>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=13,
zoom_range = 0.13,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(xtrain_df )
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128):
super(SAKTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.embedding = nn.Embedding(2*n_skill+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.bundle_embedding = nn.Embedding(n_bundles, embed_dim)
 self.tag_embedding = nn.Embedding(n_tags, embed_dim)
 self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.outDense = nn.Linear(embed_dim, 2)
 self.outActivtn = nn.LogSoftmax(dim=1)
 self.NLLLoss = nn.NLLLoss()
return
def getTagsEmb(self, tags):

 t_emb = self.tag_embedding(tags)
 t_emb = torch.mean(t_emb, dim=-2)
 return t_emb
def forward(self, x, question_ids, bundle_x, tags_x, b_target, t_target, labels=None):

 device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 bundle_x = self.bundle_embedding(bundle_x)
 b_target = self.bundle_embedding(b_target)
 tags_x = self.getTagsEmb(tags_x)
 t_target = self.getTagsEmb(t_target)
 x = x + pos_x + bundle_x + tags_x
e = self.e_embedding(question_ids)+ b_target + t_target
x = x.permute(1, 0, 2);
e = e.permute(1, 0, 2)
 att_mask = future_mask(x.size(0)).to(device)
 att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
 att_output = self.layer_normal(att_output + e)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 out_logits = self.outDense(x)
 if labels is None:
out_logits = out_logits[:, -1]
Logits = namedtuple('Logits',['logits'])
 out_logits = Logits(out_logits)
 return out_logits
out_logits = out_logits.view(out_logits.shape[0] * out_logits.shape[1], -1)
 log_ps = self.outActivtn(out_logits)
 labels = torch.squeeze(labels.view(labels.shape[0] * labels.shape[1], -1), -1)
 batchLoss = self.NLLLoss(log_ps, labels)
 Loss_Logits = namedtuple('Loss_Logits',['loss','logits'])
 loss_logits = Loss_Logits(batchLoss, out_logits)
 return loss_logits<split>
model.fit_generator(
 datagen.flow(xtrain_df, ytrain_df, batch_size=86),
steps_per_epoch=len(xtrain_df)// 86,
epochs=50,
callbacks=[learning_rate_reduction]
)
frac = 0.05
temp = list(group.index)
 temp = shuffle(temp)
 frac = int(len(temp)* frac)
 temp2 = temp[:frac]
temp = temp[frac:]
train_group = group[temp]
eval_group = group[temp2]
del group
gc.collect()
print('len(train_group)=', len(train_group))
print('len(eval_group)=', len(eval_group))
train_group<compute_train_metric>
scores = model.evaluate(xtrain_df, ytrain_df)
 print("
%s: %.2f%%" %(model.metrics_names[1], scores[1]*100))
<train_model><EOS> 
predictions = model.predict(xtest_df,verbose=0)
 results=np.argmax(predictions,axis = 1)
 results = pd.Series(results,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("digit_recognizer_datagen.csv",index=False)
 backend.clear_session()