Kaggle competition: Digit Recognizer
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )<define_variables>
train_dir = ".. /input/train.csv"
test_dir = ".. /input/test.csv"
df = pd.read_csv(train_dir)
 df.info()
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 5
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=1024 
<prepare_x_and_y>
labels = df["label"].values.tolist()
labels = np.array(labels)
 n_classes = len(set(labels))
labels = keras.utils.to_categorical(labels )
def run_training(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_fn, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions 
<categorify>
df_train = df.drop(["label"], axis = 1)
 data = df_train.values.tolist()
data = np.array(data)
 data = data.astype('float32')/255.0
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<categorify>
print("Training data shape = " + str(data.shape))
print("Training labels shape = " + str(labels.shape))
SEED = [0, 1, 2, 3 ,4, 5]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions 
<merge>
gen_model = Sequential()
gen_model.add(Dense(784, activation = 'relu', input_shape =(784,)))
 gen_model.add(Dense(512, activation = 'relu'))
gen_model.add(Dense(264, activation = 'relu'))
gen_model.add(Dense(10, activation = 'softmax'))
print("STANDARD NEURAL NETWORK MODEL :-")
 gen_model.summary()
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score)
<save_to_csv>
gen_model.compile(loss = 'categorical_crossentropy', optimizer = keras.optimizers.Adadelta() , metrics = ['accuracy'] )
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )<define_variables>
gen_model_hist = gen_model.fit(data, labels, batch_size = 32, epochs = 5, validation_split = 0.1 )
TEST_MODE = False
MOCK_MODE = False
SKIP_ADD_FEATURE = False
LOCAL = False
SAINT_PICKLE_PATH = ".. /input/saint-final"
SAINT_MODEL_PATH = ".. /input/saint-final/saintv113.pth"<import_modules>
del gen_model, gen_model_hist
gc.collect()
if MOCK_MODE: 
<import_modules>
X_train_cnn = data.reshape(len(data), 28, 28, 1 )
if LOCAL is False: 
<load_pretrained>
cnn_model = Sequential()
cnn_model.add(Conv2D(32, kernel_size = [3,3], activation = 'relu', input_shape =(28,28,1)))
 cnn_model.add(Conv2D(64, kernel_size = [3,3], activation = 'relu'))
cnn_model.add(BatchNormalization())
 cnn_model.add(MaxPool2D(pool_size = [2,2], strides = 2))
cnn_model.add(Conv2D(128, kernel_size = [3,3], activation = 'relu'))
cnn_model.add(MaxPool2D(pool_size = [2,2], strides = 2))
cnn_model.add(Flatten())
 cnn_model.add(Dense(512, activation = 'relu'))
cnn_model.add(Dense(10, activation = 'softmax'))
print("CONVOLUTIONAL NEURAL NETWORK MODEL :-")
 cnn_model.summary()
def load_group() :
group = None
for i in range(10):
with open(f"{SAINT_PICKLE_PATH}/{i}groupv1.pickle", "rb")as f:
if group is None:
group = pickle.load(f)
 else:
group = pd.concat([group, pickle.load(f)])
 gc.collect()
gc.collect()
return group<groupby>
cnn_model.compile(loss = 'categorical_crossentropy', optimizer = 'adam', metrics = ['accuracy'])
 cnn_model_hist = cnn_model.fit(X_train_cnn, labels, batch_size = 32, epochs = 6, validation_split = 0.1 )
group = load_group()<define_variables>
del cnn_model, cnn_model_hist
gc.collect()
SEED = 123
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 seed_everything(SEED )<categorify>
data_aug = ImageDataGenerator(featurewise_center = False,
samplewise_center = False,
featurewise_std_normalization = False,
samplewise_std_normalization = False,
zca_whitening = False,
rotation_range = 10,
zoom_range = 0.1,
width_shift_range = 0.1,
height_shift_range = 0.1,
horizontal_flip = False,
vertical_flip = False )
MAX_SEQ = 100
n_skill = 13523
n_part = 7
n_et = 300
n_lt = 1441
n_lsi = 128
DROPOUT = 0.1
EMBED_SIZE = 256
BATCH_SIZE = 256
def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class FFN(nn.Module):
def __init__(self, state_size = 200, forward_expansion = 1, bn_size=MAX_SEQ - 1):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, forward_expansion * state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(forward_expansion * state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.relu(self.lr1(x))
x = self.lr2(x)
 return self.dropout(x)
 class Encoder(nn.Module):
def __init__(self, n_skill, n_pt=7, n_lsi=n_lsi, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, num_layers=1, heads = 8):
super(Encoder, self ).__init__()
self.n_skill, self.embed_dim = n_skill, embed_dim
self.embedding = nn.Embedding(n_skill + 1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq, embed_dim)
 self.dropout = nn.Dropout(dropout)
 self.n_pt = n_pt
self.pt_embedding = nn.Embedding(n_pt + 1, embed_dim)
 self.n_lsi = n_lsi
self.lsi_embedding = nn.Embedding(n_lsi + 1, embed_dim)
 self.layer_normal = nn.LayerNorm(embed_dim)
 def forward(self, x, question_ids, pt_x, lsi_x):
device = x.device
x = self.embedding(x)
 pt_x = self.pt_embedding(pt_x)
 lsi_x = self.lsi_embedding(lsi_x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = self.dropout(self.layer_normal(x + pos_x + pt_x + lsi_x))
return x
class Decoder(nn.Module):
def __init__(self, n_et=n_et, n_lt=n_lt, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, num_layers=1, heads = 8):
super(Decoder, self ).__init__()
self.embed_dim = embed_dim
self.pos_embedding = nn.Embedding(max_seq, embed_dim)
 self.dropout = nn.Dropout(dropout)
 self.n_response = 2
self.n_et = n_et
self.n_lt = n_lt
self.response_embedding = nn.Embedding(self.n_response + 2, embed_dim)
 self.et_embedding = nn.Embedding(self.n_et + 2, embed_dim)
 self.lt_embedding = nn.Embedding(self.n_lt + 2, embed_dim)
 self.layer_normal = nn.LayerNorm(embed_dim)
 def forward(self, c, et, lt):
device = c.device
c = self.response_embedding(c)
 pos_id = torch.arange(c.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 et = self.et_embedding(et)
 lt = self.lt_embedding(lt)
 x = self.dropout(self.layer_normal(c + pos_x + et + lt))
return x
class SAINTModel(nn.Module):
def __init__(self, n_skill, n_pt=7, n_lsi=n_lsi, n_et=n_et, n_lt=n_lt, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, enc_layers=3, dec_layers=3, heads = 8):
super(SAINTModel, self ).__init__()
self.encoder = Encoder(n_skill, n_pt, n_lsi, max_seq, embed_dim, dropout, forward_expansion, num_layers=enc_layers)
 self.decoder = Decoder(n_et, n_lt, max_seq, embed_dim, dropout, forward_expansion, num_layers=dec_layers)
 self.transformer = torch.nn.Transformer(embed_dim, heads, enc_layers, dec_layers, embed_dim*forward_expansion, dropout)
 self.ffn = FFN(embed_dim, forward_expansion = forward_expansion)
 self.pred = nn.Linear(embed_dim, 1)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.dropout = nn.Dropout(dropout)
 def forward(self, x, question_ids, pt_x, lsi_x, c, et, lt):
ex = self.encoder(x, question_ids, pt_x, lsi_x)
 dx = self.decoder(c, et, lt)
 ex = ex.permute(1, 0, 2)
 dx = dx.permute(1, 0, 2)
 device = ex.device
mask = future_mask(ex.size(0)).to(device)
 att_output = self.transformer(ex, dx, src_mask=mask, tgt_mask=mask)
 att_output = self.layer_normal(att_output)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.dropout(self.layer_normal(x + att_output))
x = self.pred(x)
 return x.squeeze(-1)
 class TestDataset(Dataset):
def __init__(self, samples, test_df, n_skill=n_skill, n_et=n_et, n_lt=n_lt, n_pt=7, max_seq=MAX_SEQ):
super(TestDataset, self ).__init__()
self.samples = samples
self.user_ids = [x for x in test_df["user_id"].unique() ]
self.test_df = test_df
self.n_skill = n_skill
self.n_et = n_et
self.n_lt = n_lt
self.n_pt = n_pt
self.max_seq = max_seq
def __len__(self):
return self.test_df.shape[0]
def __getitem__(self, index):
test_info = self.test_df.iloc[index]
user_id = test_info["user_id"]
target_id_new = test_info["content_id"]
part_new = test_info["part"]
lsi_topic_new = test_info["lsi_topic"]
content_id_seq = np.zeros(self.max_seq, dtype=int)
 answered_correctly_seq = np.zeros(self.max_seq, dtype=int)
 c_seq = np.zeros(self.max_seq, dtype=int)
 lag_seq = np.zeros(self.max_seq, dtype=int)
 elapsed_time_seq = np.zeros(self.max_seq, dtype=int)
 part_seq = np.zeros(self.max_seq, dtype=int)
 lsi_topic_seq = np.zeros(self.max_seq, dtype=int)
 if user_id in self.samples.index:
content_id, answered_correctly, lag, elapsed_time, part, lsi_topic = self.samples[user_id]
seq_len = len(content_id)
 if seq_len >= self.max_seq:
content_id_seq[:] = content_id[-self.max_seq:]
answered_correctly_seq[:] = answered_correctly[-self.max_seq:]
c_seq[:] = answered_correctly[-self.max_seq:] + 1
lag_seq[:] = lag[-self.max_seq:] + 1
elapsed_time_seq[:] = elapsed_time[-self.max_seq:] + 1
part_seq[:] = part[-self.max_seq:]
lsi_topic_seq[:] = lsi_topic[-self.max_seq:]
else:
content_id_seq[-seq_len:] = content_id
c_seq[-seq_len:] = answered_correctly[:] + 1
lag_seq[-seq_len:] = lag[:] + 1
elapsed_time_seq[-seq_len:] = elapsed_time[:] + 1
part_seq[-seq_len:] = part
lsi_topic_seq[-seq_len:] = lsi_topic
target_id = content_id_seq
label = answered_correctly_seq
x = np.append(content_id_seq.copy() [1:], target_id_new)
 pt_x = np.append(part_seq.copy() [1:], part_new)
 lsi_x = np.append(lsi_topic_seq.copy() [1:], lsi_topic_new)
 c = c_seq.copy()
et = elapsed_time_seq.copy()
lt = lag_seq.copy()
return x, target_id, pt_x, lsi_x, c, et, lt<choose_model_class>
models_ensemble = []
for i in range(7):
model = Sequential()
model.add(Conv2D(32, kernel_size = [3,3], activation = 'relu', input_shape =(28,28,1)))
 model.add(Conv2D(64, kernel_size = [3,3], activation = 'relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size = [2,2], strides = 2))
model.add(Conv2D(128, kernel_size = [3,3], activation = 'relu'))
model.add(MaxPool2D(pool_size = [2,2], strides = 2))
model.add(Flatten())
 model.add(Dense(512, activation = 'relu'))
model.add(Dense(10, activation = 'softmax'))
model.compile(loss = 'categorical_crossentropy', optimizer = 'adam', metrics = ['accuracy'])
 models_ensemble.append(model )
def create_model() :
return SAINTModel(n_skill, n_pt=7, n_lsi=n_lsi, n_et=n_et, n_lt=n_lt, max_seq=MAX_SEQ, embed_dim=EMBED_SIZE, forward_expansion=1, enc_layers=2, dec_layers=2, heads=8, dropout=0.1 )<load_pretrained>
model_histories = []
i = 1
for model in models_ensemble:
xtrain, xtest, ytrain, ytest = train_test_split(X_train_cnn, labels, test_size = 0.07)
 print("Model " +str(i)+ " : ",end="")
 model_history = model.fit_generator(data_aug.flow(xtrain, ytrain, batch_size = 64), epochs = 1, verbose = 1, validation_data =(xtest, ytest), steps_per_epoch = xtrain.shape[0])
 model_histories.append(model_history)
 i += 1
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 saint_model = create_model()
try:
saint_model.load_state_dict(torch.load(SAINT_MODEL_PATH))
except:
saint_model.load_state_dict(torch.load(SAINT_MODEL_PATH, map_location='cpu'))
saint_model.to(device)
 saint_model.eval()<categorify>
testdata = pd.read_csv(test_dir)
 testdata = testdata.values.tolist()
testdata = np.array(testdata)
 testdata_reshaped = testdata.reshape(testdata.shape[0], 28, 28, 1)
 testdata_reshaped = testdata_reshaped.astype('float')/255.0
def make_predictions_final_model(curr_model):
prediction_array = curr_model.predict_on_batch(testdata_reshaped)
 predictions = [np.argmax(i)for i in prediction_array]
return predictions
class PredictEnv:
def __init__(self, folds_path, folds):
self.conn = sqlite3.connect(':memory:')
 self.c = self.conn.cursor()
self.setup_folds(folds_path, folds)
 def setup_folds(self, folds_path, folds):
self.c.executescript(f ).fetchone()
self.group_num = 0
self.records_remaining = self.c.execute('SELECT COUNT(*)FROM b_records' ).fetchone() [0]
self.df_users = pd.read_sql('SELECT * FROM b_users', self.conn)
 def iter_test(self):
next_correct = '[]'
next_responses = '[]'
while self.records_remaining:
self.c.execute(f ).fetchone()
self.conn.commit()
df_b = pd.read_sql(f , self.conn)
 if len(df_b):
df_b['group_num'] = self.group_num
df_b['prior_group_answers_correct'] = None
df_b.at[0, 'prior_group_answers_correct'] = next_correct
df_b['prior_group_responses'] = None
df_b.at[0, 'prior_group_responses'] = next_responses
next_correct = f'[{(", " ).join(df_b.answered_correctly.astype(str)) }]'
next_responses = f'[{(", " ).join(df_b.user_answer.astype(str)) }]'
del df_b['answered_correctly']
del df_b['user_answer']
df_b = df_b.set_index('group_num')
 df_p = df_b[['row_id']].copy()
df_p['answered_correctly'] = 0.5
self.records_remaining -= len(df_b)
 yield df_b, df_p
self.group_num += 1
def predict(self, df_pred):
if(df_pred.answered_correctly == -1 ).any() :
raise
else:
df_pred.reset_index().to_sql('predictions', self.conn, if_exists='append', index=False)
 def get_predictions(self):
df_preds = pd.read_sql(, self.conn)
 self.score = roc_auc_score(df_preds.y_true, df_preds.y_pred)
 print(f'ROC AUC Score: {self.score:0.4f}')
 return df_preds<predict_on_test>
predictions_ensemble = []
for model in models_ensemble:
curr_predictions = make_predictions_final_model(model)
 predictions_ensemble.append(curr_predictions)
 prediction_per_image = []
for i in range(len(predictions_ensemble[0])) :
temppred = [predictions_ensemble[0][i], predictions_ensemble[1][i], predictions_ensemble[2][i], predictions_ensemble[3][i], predictions_ensemble[4][i], predictions_ensemble[5][i], predictions_ensemble[6][i]]
prediction_per_image.append(temppred)
 prediction_per_image = np.array(prediction_per_image)
 modes = stats.mode(prediction_per_image, axis = 1)
 final_predictions = []
for i in modes[0]:
final_predictions.append(i[0] )
if MOCK_MODE:
FOLDS = Path('.. /input/riiid-folds/riiid.db')
 env = PredictEnv(FOLDS, [0, 1])
 iter_test = env.iter_test()
else:
env = riiideducation.make_env()
iter_test = env.iter_test()
set_predict = env.predict
if TEST_MODE and type(iter_test)!= list:
list_df = []
for itr,(df_test, sample_prediction_df)in enumerate(iter_test):
df_test.loc[:, 'answered_correctly'] = 0.5
list_df.append(( df_test, None))
env.predict(df_test.loc[df_test['content_type_id'] == 0, ['row_id', 'answered_correctly']])
 iter_test = list_df
print("TEST_MODE MODE ENABLED")
 else:
print("TEST_MODE MODE DISABLED" )<define_variables>
final_csv = []
csv_title = ['ImageId', 'Label']
final_csv.append(csv_title)
 for i in range(len(final_predictions)) :
image_id = i + 1
label = final_predictions[i]
temp = [image_id, label]
final_csv.append(temp)
 print(len(final_csv))
with open('submission_csv_aug.csv', 'w')as file:
writer = csv.writer(file)
 writer.writerows(final_csv)
 file.close()