Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<load_pretrained>
import numpy as np
import pandas as pd
import keras
from keras import Model
from keras.layers import Dense, Conv2D, pooling, Activation, BatchNormalization, Input, Flatten, Dropout, MaxPooling2D
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ReduceLROnPlateau
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, f1_score
import matplotlib.pyplot as plt
import seaborn as sns
from IPython.display import clear_output
import time
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model = SAKTModel(n_skill, embed_dim=128)
 try:
model.load_state_dict(torch.load(".. /input/rishi-sakt-featureembeddings-riidchallenge/SAKT_Rishi.pt"))
except:
model.load_state_dict(torch.load(".. /input/rishi-sakt-featureembeddings-riidchallenge/SAKT_Rishi.pt", map_location='cpu'))
model.to(device )<train_model>
train_data = pd.read_csv('.. /input/train.csv')
 test_data = pd.read_csv('.. /input/test.csv' )
training_args = TrainingArguments(
 output_dir = './results',
num_train_epochs= 1,
warmup_steps = 1000,
eval_steps = 500,
max_steps = 0,
learning_rate = 1e-6,
per_device_train_batch_size= 1024,
per_device_eval_batch_size = 1024, 
)
 trainer = MyTrainer(
 model = model,
args = training_args,
train_dataset = train_dataset,
eval_dataset = eval_dataset,
)<train_model>
dataset_x = np.array(train_data.iloc[:, 1:] ).reshape(42000, 28, 28, 1)
 dataset_y = np.array(train_data.iloc[:, 0] ).ravel()
dataset_one_hot = keras.utils.to_categorical(dataset_y, num_classes=10)
 unlabelled_test_x = np.array(test_data.iloc[:, :] ).reshape(28000, 28, 28, 1)
 assert(dataset_x.shape[0] == dataset_y.size )
trainer.train()<train_model>
dataset_x = dataset_x / 255.0
unlabelled_test_x = unlabelled_test_x / 255.0
training_args = TrainingArguments(
 output_dir = './results',
num_train_epochs = 1,
warmup_steps = 100,
eval_steps = 300,
max_steps = 0,
learning_rate = 1e-7,
per_device_train_batch_size = 1024,
per_device_eval_batch_size = 1024, 
)
 trainer = MyTrainer(
 model = model,
args = training_args,
train_dataset = eval_dataset,
eval_dataset = None,
)<train_model>
train_x, test_x, train_y, test_y = train_test_split(dataset_x, dataset_one_hot, test_size=0.1 )
trainer.train()<concatenate>
transform_params = {
'featurewise_center': False,
'featurewise_std_normalization': False,
'samplewise_center': False,
'samplewise_std_normalization': False,
'rotation_range': 10,
'width_shift_range': 0.1,
'height_shift_range': 0.1,
'zoom_range': 0.1,
'validation_split': 0.15
}
img_gen = ImageDataGenerator(**transform_params )
group = train_group.append(eval_group)
 print('len(group)=', len(group))
del train_group, eval_group, train_dataset, eval_dataset, training_args, trainer
gc.collect()
group<feature_engineering>
visualizaion_flow = img_gen.flow(train_x, train_y, batch_size=1, shuffle=False)
 train_gen = img_gen.flow(x=train_x, y=train_y, subset='training', batch_size=96)
 valid_gen = img_gen.flow(x=train_x, y=train_y, subset='validation', batch_size=96 )
class TestDataset(Dataset):
def __init__(self, samples, test_df, skills, max_seq=MAX_SEQ):
super(TestDataset, self ).__init__()
self.samples = samples
self.user_ids = [x for x in test_df["user_id"].unique() ]
self.test_df = test_df
self.skills = skills
self.n_skill = max(skills)+ 1
self.max_seq = max_seq
return
def __len__(self):
return self.test_df.shape[0]
def __getitem__(self, index):
test_info = self.test_df.iloc[index]
user_id = test_info["user_id"]
target_id = test_info["content_id"]
target_tags = test_info["tags"]
target_bundle = test_info["bundle_id"]
if target_id not in known_qtn_ids:
target_id = 0
if target_bundle not in known_bundle_ids:
target_bundle = 0
for i in range(len(target_tags)) :
if target_tags[i] not in known_tags:
target_tags[i] = 0
q = np.zeros(self.max_seq, dtype=int)
 qa = np.zeros(self.max_seq, dtype=int)
 b = np.zeros(self.max_seq, dtype=int)
 t = np.array([[0]*5]*self.max_seq, dtype=int)
 if user_id in self.samples.index:
q_, qa_ = self.samples[user_id]
q_b_t_df = pd.DataFrame.from_dict({'question_id': q_})
 q_b_t_df = q_b_t_df.merge(questions_df, on='question_id', how='left')
 b_ = q_b_t_df.bundle_id.values
t_ = np.vstack(q_b_t_df.tags.values)
 del q_b_t_df
seq_len = len(q_)
 if seq_len >= self.max_seq:
q = q_[-self.max_seq:]
qa = qa_[-self.max_seq:]
b = b_[-self.max_seq:]
t = t_[-self.max_seq:]
else:
q[-seq_len:] = q_
qa[-seq_len:] = qa_
b[-seq_len:] = b_
t[-seq_len:] = t_
x = np.zeros(self.max_seq-1, dtype=int)
 x = q[1:].copy()
x +=(qa[1:] == 1)* self.n_skill
questions = np.append(q[2:], [target_id])
 bundle_x = b[1:].copy()
tags_x = t[1:].copy()
b_target = np.append(b[2:], [target_bundle])
 t_target = np.vstack(( t[2:], target_tags))
return x, questions, bundle_x, tags_x, b_target, t_target<split>
def conv_block(x, filters, kernel_size, strides, layer_no, use_pool=False, padding='same'):
x = Conv2D(filters=filters, kernel_size=kernel_size, strides=strides, name=f'conv{layer_no}',
padding=padding )(x)
 x = BatchNormalization(name=f'bn{layer_no}' )(x)
 x = Activation('relu', name=f'activation{layer_no}' )(x)
 if use_pool:
x = MaxPooling2D(pool_size=[2, 2], strides=[2, 2], name=f'pool{layer_no}', padding='same' )(x)
 return x
env = riiideducation.make_env()
iter_test = env.iter_test()<merge>
def conv_model(X):
h, w, c = X.shape[1:]
X = Input(shape=(h, w, c))
conv1 = conv_block(X, filters=8, kernel_size=[3, 3], strides=[1, 1], layer_no=1)
 conv2 = conv_block(conv1, filters=16, kernel_size=[2, 2], strides=[1, 1], layer_no=2)
 conv3 = conv_block(conv2, filters=32, kernel_size=[2, 2], strides=[1, 1], layer_no=3, use_pool=True)
 conv4 = conv_block(conv3, filters=64, kernel_size=[3, 3], strides=[2, 2], layer_no=4)
 conv5 = conv_block(conv4, filters=128, kernel_size=[2, 2], strides=[1, 1], layer_no=5)
 conv6 = conv_block(conv5, filters=256, kernel_size=[2, 2], strides=[1, 1], layer_no=6, use_pool=True)
 flat1 = Flatten(name='flatten1' )(conv6)
 drop1 = Dropout(0.35, name='Dopout1' )(flat1)
 dens1 = Dense(128, name='dense1' )(drop1)
 bn7 = BatchNormalization(name='bn7' )(dens1)
 drop2 = Dropout(0.35, name='Dopout2' )(bn7)
 relu1 = Activation('relu', name='activation7' )(drop2)
 dens1 = Dense(256, name='dense01' )(relu1)
 bn7 = BatchNormalization(name='bn07' )(dens1)
 drop2 = Dropout(0.5, name='Dopout02' )(bn7)
 relu1 = Activation('relu', name='activation07' )(drop2)
 dens2 = Dense(10, name='dense2' )(relu1)
 bn8 = BatchNormalization(name='bn8' )(dens2)
 output_layer = Activation('softmax', name='softmax' )(bn8)
 model = Model(inputs=X, outputs=output_layer)
 return model
model.eval()
prev_test_df = None
for(test_df, sample_prediction_df)in tqdm(iter_test):
test_df = test_df.merge(questions_df, left_on='content_id', right_on='question_id', how='left' ).drop(columns=['question_id'])
 if(prev_test_df is not None)&(psutil.virtual_memory().percent<90):
print(psutil.virtual_memory().percent)
 prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prev_test_df = prev_test_df[prev_test_df.content_type_id == False]
prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly', 'bundle_id', 'tags']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values,
r['bundle_id'].values,
np.vstack(list(r['tags'].values))))
for prev_user_id in prev_group.index:
prev_group_content = prev_group[prev_user_id][0]
prev_group_ac = prev_group[prev_user_id][1]
prev_group_bund_id = prev_group[prev_user_id][2]
prev_group_tags = prev_group[prev_user_id][3]
if prev_user_id in group.index:
group[prev_user_id] =(np.append(group[prev_user_id][0], prev_group_content),
np.append(group[prev_user_id][1], prev_group_ac))
else:
group[prev_user_id] =(prev_group_content, prev_group_ac)
 if len(group[prev_user_id][0])> MAX_SEQ:
new_group_content = group[prev_user_id][0][-MAX_SEQ:]
new_group_ac = group[prev_user_id][1][-MAX_SEQ:]
group[prev_user_id] =(new_group_content, new_group_ac)
 prev_test_df = test_df.copy()
test_df = test_df[test_df.content_type_id == False]
test_dataset = TestDataset(group, test_df, skills)
 test_dataloader = DataLoader(test_dataset, batch_size=1024, shuffle=False)
 outs = []
softmax = nn.Softmax(dim=1)
 for item in tqdm(test_dataloader):
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
bundle_x = item[2].to(device ).long()
tags_x = item[3].to(device ).long()
b_target = item[4].to(device ).long()
t_target = item[5].to(device ).long()
with torch.no_grad() :
pred_logits = model(x, target_id, bundle_x, tags_x, b_target, t_target)
 pred_logits = pred_logits.logits.cpu()
pred_pr = softmax(pred_logits ).numpy()
pred_pr = pred_pr[:, 1]
outs.extend(pred_pr)
 test_df['answered_correctly'] = outs
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<train_model>
model = conv_model(train_x )
print("Done !" )<import_modules>
def build_model(learning_rate=0.00065):
optimizer = keras.optimizers.RMSprop(lr=learning_rate, decay=learning_rate/2)
 model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'] )
import numpy as np
import pandas as pd
import tensorflow as tf
import pickle
import json
import typing
import math
from contextlib import contextmanager
import time
from threading import Lock 
<import_modules>
plateau_reduce = ReduceLROnPlateau(monitor='val_acc', factor=0.5,
patience=6, min_lr=0.0000001 )
from modeling import RiiidAnswerModel
from model_config import config<categorify>
use_generator = True
with open('.. /input/riiiddatamodel/data_map_512.pickle', 'rb')as f:
data_map = pickle.load(f)
 with open(
 '.. /input/riiiddatamodel/encoded_content_map.json',
'r')as f:
encoded_content_map = json.load(f)
 with open(
 '.. /input/riiiddatamodel/encoded_content_id_map.json',
'r')as f:
encoded_content_id_map = json.load(f)
 encoded_content_table = pd.DataFrame.from_dict(
 encoded_content_id_map
).astype({
'content_id': 'int16',
'content_type_id':'int8',
'encoded_content_id':'int32'
})
 max_question_id = encoded_content_table[
encoded_content_table.content_type_id == 0
].encoded_content_id.max()<define_variables>
callbacks = [plotter]
SEQUENCE_LENGTH = 512
BATCH_SIZE = 32
data_keys = ['timestamp',
'encoded_content_id',
'user_answer',
'answered_correctly',
'question_elapsed_time',
'question_had_explanation',
'non_padding_mask',
'time_lag']
data_types = [
tf.int64,
tf.int32,
tf.int32,
tf.int32,
tf.float32,
tf.int32,
tf.float32,
tf.float32
]<categorify>
if use_generator:
model.fit_generator(train_gen, validation_data=valid_gen, epochs=120,
steps_per_epoch=train_x.shape[0]*0.85//96,
validation_steps=train_x.shape[0]*0.15//96, callbacks=callbacks)
 else:
model.fit(x=train_x, y=train_y, epochs=80, batch_size=32, callbacks=callbacks,
validation_split=0.15 )
def build_test_ds_function(test_agg_df, data_map):
data = data_map['data']
index_map = data_map['index']
lock = Lock()
def process_row_i(i):
user_id = i.numpy()
row = test_agg_df.loc[user_id]
n = len(row['timestamp'])
 index = index_map.get(user_id, None)
 if index is not None:
is_question = data['encoded_content_id'][index] <= max_question_id
questions = data['timestamp'][index][is_question]
if len(questions)> 0:
last_question_timestamp = questions.max()
prior_question_cond = data['timestamp'][index] == last_question_timestamp
data['question_elapsed_time'][index][prior_question_cond] = row['prior_question_elapsed_time'][0]
data['question_had_explanation'][index][prior_question_cond] = row['prior_question_had_explanation'][0]
for k in data_keys:
data[k][index,:-n] = data[k][index, n:]
else:
lock.acquire()
index = data_map['next_index']
index_map[user_id] = index
data_map['next_index'] += 1
lock.release()
data['timestamp'][index, -n:] = row['timestamp']
data['encoded_content_id'][index,-n:] = row['encoded_content_id']
data['user_answer'][index, -n:] = 0
data['answered_correctly'][index, -n:] = 0
data['question_elapsed_time'][index, -n:] = 0
data['question_had_explanation'][index, -n:] = 0
data['non_padding_mask'][index,-n:] = 1
data['time_lag'][index, -n:] = row['timestamp'][0] - data['timestamp'][index, -(n+1)]
return [data[k][index] for k in data_keys]
def map_function(i):
x = tf.py_function(process_row_i, inp=[i], Tout=data_types)
 return {
k:v for k,v in zip(data_keys, x)
 }
return tf.data.Dataset.from_tensor_slices(test_agg_df.index ).map(
 map_function,
num_parallel_calls = tf.data.experimental.AUTOTUNE
).prefetch(
 tf.data.experimental.AUTOTUNE
).batch(BATCH_SIZE)
<categorify>
keras.models.save_model(model, f'model_{time.time() }.h5' )
def update_map_with_user_answer(data_map, prev_test_agg_df):
data = data_map['data']
for user_id, row in prev_test_agg_df.iterrows() :
n = len(row['user_answer'])
 index = data_map['index'][user_id]
data['user_answer'][index][-n:] = row['user_answer']
data['answered_correctly'][index][-n:] = row['answered_correctly'] 
<predict_on_test>
def calculate_performance(labels, pred, dataset):
pred_cat = np.argmax(pred, axis=1)
 labels_cat = np.argmax(labels, axis=1)
 corrects =(pred_cat == labels_cat)
 falses = dataset[~corrects]
falses_labels = labels_cat[~corrects]
falses_preds = pred[~corrects]
examples_num = labels.shape[0]
accuracy = np.count_nonzero(corrects)/ examples_num
return accuracy, [falses, falses_labels, falses_preds], [labels_cat, pred_cat]
model = RiiidAnswerModel(
 encoded_content_map,
**config 
)
 @tf.function(
 input_signature =({
key : tf.TensorSpec(shape=(None, SEQUENCE_LENGTH), dtype=dt)
 for key, dt in zip(data_keys, data_types)
 },)
)
 def predict_batch(test_data):
logits = model(test_data, training = False)
 logits = tf.squeeze(logits, axis =-1)
 sigmoids = tf.sigmoid(logits)
 return sigmoids
sample_data = {k:data_map['data'][k][0:BATCH_SIZE,:] for k in data_map['data'].keys() }
predict_batch(sample_data)
 model.load_weights('.. /input/riiiddatamodel/weights.h5')
<set_options>
random_idx = np.random.choice(range(test_y.shape[0]), size=10)
 examples = test_x[random_idx]
preds = model.predict(examples)
 labels = np.argmax(test_y[random_idx], axis=1)
 preds_cat = np.argmax(preds, axis=1 )
env = riiideducation.make_env()<feature_engineering>
random_idx = np.random.choice(range(test_y.shape[0]), size=10)
 examples = unlabelled_test_x[random_idx]
preds = model.predict(examples)
 preds_cat = np.argmax(preds, axis=1 )
%%time
iter_test = env.iter_test()
prev_test_df = None
for test_df, sample_prediction_df in iter_test:
if prev_test_df is not None:

 prev_test_df['user_answer'] = eval(test_df.prior_group_responses.iloc[0])
 prev_test_df['answered_correctly'] = eval(test_df.prior_group_answers_correct.iloc[0])
 prev_test_agg_df = prev_test_df.groupby(
 'user_id',
sort = False,
as_index = True 
)[['user_answer', 'answered_correctly'
]].agg(list)
 update_map_with_user_answer(data_map, prev_test_agg_df)

test_df.fillna(0, inplace = True)
 test_agg_df = test_df.merge(
 encoded_content_table,
on = ['content_id', 'content_type_id'],
how = 'left'
).fillna(0 ).groupby('user_id',
sort = False,
as_index = True)[['timestamp',
'encoded_content_id',
'prior_question_elapsed_time',
'prior_question_had_explanation',
'row_id'
]].agg(list)
 probs = predict_and_update(test_agg_df, data_map)
 extracted_probs = []
row_id = []
for prob,(_, row)in zip(probs,
test_agg_df.iterrows()):

 n = len(row['row_id'])
 extracted_probs += prob[-n:].numpy().tolist()
row_id += row['row_id']
pred_df = pd.DataFrame.from_dict({
'row_id':row_id,
'answered_correctly': extracted_probs,
})
 pred_df = sample_prediction_df[['row_id']].merge(
 pred_df,
on = 'row_id',
how = 'left'
).fillna(0.5)
 env.predict(pred_df)
 prev_test_df = test_df 
<install_modules>
falses_data = test_falses_data
falses_examples, falses_labels, falses_preds = falses_data
falses_idx = np.argmax(falses_preds, axis=1 )
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl
!pip install.. /input/pandarallel151whl/pandarallel-1.5.1-py3-none-any.whl<import_modules>
random_falses = np.random.choice(range(falses_labels.size), size=np.min(( 10, falses_labels.size)) , replace=False)
 examples = falses_examples[random_falses]
preds_probs = falses_preds[random_falses]
labels = falses_labels[random_falses]
preds_binary = np.argmax(preds_probs, axis=1 )
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 session_conf = tf.compat.v1.ConfigProto(
 intra_op_parallelism_threads=1,
inter_op_parallelism_threads=1 
)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=session_conf)
 tf.compat.v1.keras.backend.set_session(sess)
 warnings.simplefilter('ignore')
 pd.set_option("max_columns", 150)
 pd.set_option('display.max_rows', 150)
 seed_everything(42)
 pandarallel.initialize()<define_variables>
pred = model.predict(unlabelled_test_x)
 pred_cat = np.argmax(pred,axis = 1)
 pred_series = pd.Series(pred_cat,name='Label')
 id_series = pd.Series(range(1, unlabelled_test_x.shape[0]+1), name='ImageId')
 submission = pd.concat(( id_series, pred_series), axis=1)
 print(submission.head(5))
<load_from_csv><EOS> 
submission.to_csv('submission.csv', index=False )