Kaggle competition: Digit Recognizer
class SAKTDataset(Dataset):
def __init__(self, group, n_skill, max_seq=100):
super(SAKTDataset, self ).__init__()
self.samples, self.n_skill, self.max_seq = {}, n_skill, max_seq
self.user_ids = []
for i, user_id in enumerate(group.index):
if(i % 10000 == 0):
print(f'Processed {i} users')
 content_id, answered_correctly = group[user_id]
if len(content_id)>= ACCEPTED_USER_CONTENT_SIZE:
if len(content_id)> self.max_seq:
total_questions = len(content_id)
 last_pos = total_questions // self.max_seq
for seq in range(last_pos):
index = f"{user_id}_{seq}"
self.user_ids.append(index)
 start = seq * self.max_seq
end =(seq + 1)* self.max_seq
self.samples[index] =(content_id[start:end], answered_correctly[start:end])
 if len(content_id[end:])>= ACCEPTED_USER_CONTENT_SIZE:
index = f"{user_id}_{last_pos + 1}"
self.user_ids.append(index)
 self.samples[index] =(content_id[end:], answered_correctly[end:])
 else:
index = f'{user_id}'
self.user_ids.append(index)
 self.samples[index] =(content_id, answered_correctly)
 def __len__(self):
return len(self.user_ids)
 def __getitem__(self, index):
user_id = self.user_ids[index]
content_id, answered_correctly = self.samples[user_id]
seq_len = len(content_id)
 content_id_seq = np.zeros(self.max_seq, dtype=int)
 answered_correctly_seq = np.zeros(self.max_seq, dtype=int)
 if seq_len >= self.max_seq:
content_id_seq[:] = content_id[-self.max_seq:]
answered_correctly_seq[:] = answered_correctly[-self.max_seq:]
else:
content_id_seq[-seq_len:] = content_id
answered_correctly_seq[-seq_len:] = answered_correctly
target_id = content_id_seq[1:]
label = answered_correctly_seq[1:]
x = content_id_seq[:-1].copy()
x +=(answered_correctly_seq[:-1] == 1)* self.n_skill
return x, target_id, label<split>
import matplotlib.pyplot as plt
from keras.layers import Conv2D, MaxPooling2D, UpSampling2D, Softmax, Activation, Lambda, Concatenate, Dense, Flatten
from keras.utils import to_categorical
from keras.regularizers import l2
from keras import Model, Input
from keras.callbacks import EarlyStopping, ModelCheckpoint
from keras.models import load_model
import keras.backend as K
from keras.applications import ResNet50
from keras.preprocessing.image import ImageDataGenerator
TEST_SIZE = 0.05
train, val = train_test_split(group, test_size = TEST_SIZE, random_state=42 )<create_dataframe>
train = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv")
 train
train_dataset = SAKTDataset(train, n_skill, max_seq=MAX_SEQ)
 train_dataloader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=8)
 del train<create_dataframe>
X = [train.iloc[i,1:].values for i in range(len(train)) ]
X = [x.reshape(28,28)for x in X]
X = [np.pad(x, 2)for x in X]
X = np.array(X)
 X = X.reshape(X.shape[0],X.shape[1], X.shape[2],1)
 X = np.repeat(X, 3, axis=-1)
 X.shape
val_dataset = SAKTDataset(val, n_skill, max_seq=MAX_SEQ)
 val_dataloader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=8)
 del val<feature_engineering>
X_test = [test.iloc[i,:].values for i in range(len(test)) ]
X_test = [x.reshape(28,28)for x in X_test]
X_test = [np.pad(x, 2)for x in X_test]
X_test = np.array(X_test)
 X_test = X_test.reshape(X_test.shape[0],X_test.shape[1], X_test.shape[2],1)
 X_test = np.repeat(X_test, 3, axis=-1)
 X_test.shape
<choose_model_class>
n_classes = 10
y = [train.iloc[i,0] for i in range(len(train)) ]
y = np.array(y)
 print(np.unique(y, return_counts=True))
y = to_categorical(y, num_classes=n_classes)
 y.shape
class FFN(nn.Module):
def __init__(self, state_size = 200, forward_expansion = 1, bn_size = MAX_SEQ - 1, dropout=0.2):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, forward_expansion * state_size)
 self.relu = nn.ReLU()
self.bn = nn.BatchNorm1d(bn_size)
 self.lr2 = nn.Linear(forward_expansion * state_size, state_size)
 self.dropout = nn.Dropout(dropout)
 def forward(self, x):
x = self.relu(self.lr1(x))
x = self.bn(x)
 x = self.lr2(x)
 return self.dropout(x)
 class FFN0(nn.Module):
def __init__(self, state_size = 200, forward_expansion = 1, bn_size = MAX_SEQ - 1, dropout=0.2):
super(FFN0, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, forward_expansion * state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(forward_expansion * state_size, state_size)
 self.layer_normal = nn.LayerNorm(state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 x=self.layer_normal(x)
 return self.dropout(x )<categorify>
val_rate=0.1
def future_mask(seq_length):
future_mask =(np.triu(np.ones([seq_length, seq_length]), k = 1)).astype('bool')
 return torch.from_numpy(future_mask)
 future_mask(5 )<choose_model_class>
gen = ImageDataGenerator(rotation_range=30, width_shift_range=5, height_shift_range=5,
shear_range=30, validation_split=val_rate)
 train_gen = gen.flow(X, y, subset="training", batch_size=32)
 val_gen = gen.flow(X, y, subset="validation", batch_size=128 )
class TransformerBlock(nn.Module):
def __init__(self, embed_dim, heads = 8, dropout = DROPOUT, forward_expansion = 1):
super(TransformerBlock, self ).__init__()
self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=heads, dropout=dropout)
 self.dropout = nn.Dropout(dropout)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim, forward_expansion = forward_expansion, dropout=dropout)
 self.ffn0 = FFN0(embed_dim, forward_expansion = forward_expansion, dropout=dropout)
 self.layer_normal_2 = nn.LayerNorm(embed_dim)
 def forward(self, value, key, query, att_mask):
att_output, att_weight = self.multi_att(value, key, query, attn_mask=att_mask)
 att_output = self.dropout(self.layer_normal(att_output + value))
att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x1 = self.ffn0(att_output)
 x = self.dropout(self.layer_normal_2(x + x1 + att_output))
return x.squeeze(-1), att_weight
class Encoder(nn.Module):
def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, num_layers=1, heads = 8):
super(Encoder, self ).__init__()
self.n_skill, self.embed_dim = n_skill, embed_dim
self.embedding = nn.Embedding(2 * n_skill + 1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq - 1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.layers = nn.ModuleList([TransformerBlock(embed_dim, forward_expansion = forward_expansion)for _ in range(num_layers)])
 self.dropout = nn.Dropout(dropout)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = self.dropout(x + pos_x)
 x = x.permute(1, 0, 2)
 e = self.e_embedding(question_ids)
 e = e.permute(1, 0, 2)
 for layer in self.layers:
att_mask = future_mask(e.size(0)).to(device)
 x, att_weight = layer(e, x, x, att_mask=att_mask)
 x = x.permute(1, 0, 2)
 x = x.permute(1, 0, 2)
 return x, att_weight
class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, enc_layers=1, heads = 8):
super(SAKTModel, self ).__init__()
self.encoder = Encoder(n_skill, max_seq, embed_dim, dropout, forward_expansion, num_layers=enc_layers)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
x, att_weight = self.encoder(x, question_ids)
 x = self.pred(x)
 return x.squeeze(-1), att_weight<set_options>
res = ResNet50(include_top=False, weights=None, input_shape=(32,32,3), pooling="max")
 res.summary()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )<choose_model_class>
x = res.layers[-1].output
x = Dense(10, activation='softmax', name='predictions' )(x)
 model = Model(inputs=res.layers[0].output,outputs=x)
 model.summary()
def create_model() :
return SAKTModel(n_skill, max_seq=MAX_SEQ, embed_dim=EMBED_SIZE, forward_expansion=1, enc_layers=1, heads=4, dropout=0.1)
 model = create_model()
model<train_model>
model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["categorical_accuracy"] )
model(sample_batch[0], sample_batch[1])[0]<define_variables>
epochs = 100
train_steps = int(42000*(1-val_rate)) //32
val_steps = int(42000*val_rate)//128
MODEL_PATH = '/kaggle/working/sakt_model.pt'<choose_model_class>
early_stopping = EarlyStopping(patience=10, verbose=1)
 model_checkpoint = ModelCheckpoint("best_model.h5", save_best_only=True, verbose=1)
 history = model.fit_generator(train_gen, steps_per_epoch=train_steps, epochs=epochs,
validation_data=val_gen, validation_steps=val_steps, callbacks=[model_checkpoint])
 model.save("model.h5" )
def do_train() :
optimizer = torch.optim.Adam(model.parameters() , lr=LR)
 criterion = nn.BCEWithLogitsLoss()
scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=LR,
steps_per_epoch=len(train_dataloader), epochs=EPOCHS)
 model.to(device)
 criterion.to(device)
 best_auc = 0.0
for epoch in range(EPOCHS):
train_epoch(model, train_dataloader, optimizer, criterion, scheduler, device)
 val_loss, avl_acc, val_auc = val_epoch(model, val_dataloader, criterion, device)
 print(f"epoch - {epoch + 1} val_loss - {val_loss:.3f} acc - {avl_acc:.3f} auc - {val_auc:.3f}")
 if best_auc < val_auc:
print(f'epoch - {epoch + 1} best model with val auc: {val_auc}')
 best_auc = val_auc
torch.save(model.state_dict() , MODEL_PATH )<train_model>
best_model = load_model("best_model.h5" )
LR = 2e-3
EPOCHS = 11
print(f'{LR}, {EPOCHS}')
 do_train()<train_model>
print("Evaluating performance on all training data...")
 results = best_model.evaluate(X, y, batch_size = 128)
 print("train loss, train acc:", results )
LR = 2e-4
EPOCHS = 3
print(f'{LR}, {EPOCHS}')
 do_train()<load_pretrained>
print("Predicting on all available data...")
 y_pred_one_hot = best_model.predict(X_test, verbose=1, batch_size=128 )
model = create_model()
model.load_state_dict(torch.load(MODEL_PATH))
model.to(device )<split>
sub = pd.read_csv("/kaggle/input/digit-recognizer/sample_submission.csv")
 sub
env = riiideducation.make_env()
iter_test = env.iter_test()<feature_engineering>
for i in range(len(y_pred)) :
sub.iloc[i].Label = y_pred[i]
model.eval()
prev_test_df = None
for(test_df, sample_prediction_df)in tqdm(iter_test):
if(prev_test_df is not None)&(psutil.virtual_memory().percent<90):
print(psutil.virtual_memory().percent)
 prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prev_test_df = prev_test_df[prev_test_df.content_type_id == False]
prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
prev_group_content = prev_group[prev_user_id][0]
prev_group_answered_correctly = prev_group[prev_user_id][1]
if prev_user_id in group.index:
group[prev_user_id] =(np.append(group[prev_user_id][0], prev_group_content),
np.append(group[prev_user_id][1], prev_group_answered_correctly))
else:
group[prev_user_id] =(prev_group_content, prev_group_answered_correctly)
 if len(group[prev_user_id][0])> MAX_SEQ:
new_group_content = group[prev_user_id][0][-MAX_SEQ:]
new_group_answered_correctly = group[prev_user_id][1][-MAX_SEQ:]
group[prev_user_id] =(new_group_content, new_group_answered_correctly)
 prev_test_df = test_df.copy()
test_df = test_df[test_df.content_type_id == False]
test_dataset = TestDataset(group, test_df, n_skill, max_seq=MAX_SEQ)
 test_dataloader = DataLoader(test_dataset, batch_size=len(test_df), shuffle=False)
 item = next(iter(test_dataloader))
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, _ = model(x, target_id)
 output = torch.sigmoid(output)
 output = output[:, -1]
test_df['answered_correctly'] = output.cpu().numpy()
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<install_modules>
sub.to_csv("submission.csv", index=False )