Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<import_modules>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from keras.utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Conv2D, MaxPool2D, Flatten, BatchNormalization, Dropout
from keras.callbacks import EarlyStopping, ReduceLROnPlateau
from keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
import itertools
class Predictor(object):

 def __init__(self, model, model_metadata, ohe_categorical_index_vocab,
mhe_categorical_index_vocab):

 self._model = model
self._model_metadata = model_metadata
self._ohe_categorical_index_vocab = ohe_categorical_index_vocab
self._mhe_categorical_index_vocab = mhe_categorical_index_vocab
self._model_type = None
self._label_col = None
self._feature_name_to_index_map = {}
self._feature_names = []
self._class_names = []
def _extract_model_metadata(self):

 if 'model_type' not in self._model_metadata or self._model_metadata[
'model_type'] not in [
'boosted_tree_regressor', 'boosted_tree_classifier'
]:
raise ValueError('Invalid model_type in model_metadata')
 self._model_type = self._model_metadata['model_type']
if 'label_col' not in self._model_metadata:
raise ValueError('label_col not found in model_metadata')
 self._label_col = self._model_metadata['label_col']
if not self._model_metadata['features']:
raise ValueError('No feature found in model_metadata')
 self._feature_names = self._model_metadata['feature_names']
if self._model_type == 'boosted_tree_classifier':
if 'class_names' not in self._model_metadata or not self._model_metadata[
'class_names']:
raise ValueError('No class_names found in model_metadata')
 self._class_names = self._model_metadata['class_names']
for feature_index in range(len(self._feature_names)) :
feature_name = self._feature_names[feature_index]
self._feature_name_to_index_map[feature_name] = feature_index
feature_metadata = self._model_metadata['features'][feature_name]
if 'encode_type' not in feature_metadata or not feature_metadata[
'encode_type']:
continue
elif feature_metadata['encode_type'] == 'ohe':
if feature_index not in self._ohe_categorical_index_vocab:
raise ValueError(
 'feature_index %d missing in _ohe_categorical_index_vocab' %
feature_index)
 elif feature_metadata['encode_type'] == 'mhe':
if feature_index not in self._mhe_categorical_index_vocab:
raise ValueError(
 'feature_index %d missing in _mhe_categorical_index_vocab' %
feature_index)
 else:
raise ValueError('Invalid encode_type %s for feature %s' % 
(feature_metadata['encode_type'], feature_name))
def _preprocess(self, data):

 self._extract_model_metadata()
preprocessed_data = []
for row_index in range(len(data)) :
row = data[row_index]
sorted_data_feature_names = sorted(row.keys())
 sorted_model_feature_names = sorted(self._feature_names)
 if sorted_data_feature_names != sorted_model_feature_names:
raise ValueError(
 'Row %d has different features %s than the model features %s' % 
(row_index, ','.join(sorted_data_feature_names),
','.join(sorted_model_feature_names)))
 encoded_row = []
for feature_name in self._feature_names:
col = row[feature_name]
feature_index = self._feature_name_to_index_map[feature_name]
if feature_index in self._ohe_categorical_index_vocab:
vocab = self._ohe_categorical_index_vocab[feature_index]
col_value = str(col)
 if col_value in vocab:
encoded_row.append(float(vocab.index(col_value)))
 else:
encoded_row.append(None)
 elif feature_index in self._mhe_categorical_index_vocab:
vocab = self._mhe_categorical_index_vocab[feature_index]
mhe_list = [0.0] * len(vocab)
 try:
for item in col:
item_value = str(item)
 if item_value in vocab:
mhe_list[vocab.index(item_value)] = 1.0
encoded_row.extend(mhe_list)
 except ValueError:
raise ValueError('The feature %s in row %d is not an array' % 
(feature_name, row_index))
else:
try:
encoded_row.append(float(col))
except ValueError:
raise ValueError(
 'The feature %s in row %d cannot be converted to float' % 
(feature_name, row_index))
preprocessed_data.append(encoded_row)
 return preprocessed_data
def predict(self, instances, **kwargs):

 del kwargs
encoded = self._preprocess(instances)
 prediction_input = xgb.DMatrix(
 np.array(encoded ).reshape(( len(instances), -1)) , missing=None)
 if self._model_type == 'boosted_tree_classifier':
outputs = self._model.predict(prediction_input)
 final_outputs = []
for np_output in outputs:
output = np_output.tolist()
final_output = {}
final_output['predicted_{}'.format(
 self._label_col)] = self._class_names[output.index(max(output)) ]
final_output['{}_values'.format(self._label_col)] = self._class_names
final_output['{}_probs'.format(self._label_col)] = output
final_outputs.append(final_output)
 return final_outputs
else:
return {
'predicted_' + self._label_col:
self._model.predict(prediction_input ).tolist()
}
@classmethod
def from_path(cls, model_dir, model_name="model.bst", meta_name="model_metadata.json"):

 model_path = os.path.join(model_dir, model_name)
 model = xgb.Booster(model_file=model_path)
 assets_path = model_dir
model_metadata_path = os.path.join(assets_path, meta_name)
 with open(model_metadata_path)as f:
model_metadata = json.load(f)
 txt_list = glob.glob(assets_path + '/*.txt')
 ohe_categorical_index_vocab = {}
mhe_categorical_index_vocab = {}
for txt_file in txt_list:
ohe_feature_found = re.search(r'(\d+ ).txt', txt_file)
 mhe_feature_found = re.search(r'(\d+)_array.txt', txt_file)
 if ohe_feature_found:
feature_index = int(ohe_feature_found.group(1))
with open(txt_file)as f:
ohe_categorical_index_vocab[feature_index] = f.read().splitlines()
elif mhe_feature_found:
feature_index = int(mhe_feature_found.group(1))
with open(txt_file)as f:
mhe_categorical_index_vocab[feature_index] = f.read().splitlines()
return cls(model, model_metadata, ohe_categorical_index_vocab,
mhe_categorical_index_vocab )<feature_engineering>
train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv' )
class Iter_Valid(object):
def __init__(self, df, max_user=1000):
df = df.reset_index(drop=True)
 self.df = df
self.user_answer = df['user_answer'].astype(str ).values
self.answered_correctly = df['answered_correctly'].astype(str ).values
df['prior_group_responses'] = "[]"
df['prior_group_answers_correct'] = "[]"
self.sample_df = df[df['content_type_id'] == 0][['row_id']]
self.sample_df['answered_correctly'] = 0
self.len = len(df)
 self.user_id = df.user_id.values
self.task_container_id = df.task_container_id.values
self.content_type_id = df.content_type_id.values
self.max_user = max_user
self.current = 0
self.pre_user_answer_list = []
self.pre_answered_correctly_list = []
def __iter__(self):
return self
def fix_df(self, user_answer_list, answered_correctly_list, pre_start):
df= self.df[pre_start:self.current].copy()
sample_df = self.sample_df[pre_start:self.current].copy()
df.loc[pre_start,'prior_group_responses'] = '[' + ",".join(self.pre_user_answer_list)+ ']'
df.loc[pre_start,'prior_group_answers_correct'] = '[' + ",".join(self.pre_answered_correctly_list)+ ']'
self.pre_user_answer_list = user_answer_list
self.pre_answered_correctly_list = answered_correctly_list
return df, sample_df
def __next__(self):
added_user = set()
pre_start = self.current
pre_added_user = -1
pre_task_container_id = -1
pre_content_type_id = -1
user_answer_list = []
answered_correctly_list = []
while self.current < self.len:
crr_user_id = self.user_id[self.current]
crr_task_container_id = self.task_container_id[self.current]
crr_content_type_id = self.content_type_id[self.current]
if crr_user_id in added_user and(crr_user_id != pre_added_user or(crr_task_container_id != pre_task_container_id and crr_content_type_id == 0 and pre_content_type_id == 0)) :
return self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 if len(added_user)== self.max_user:
if crr_user_id == pre_added_user and(crr_task_container_id == pre_task_container_id or crr_content_type_id == 1):
user_answer_list.append(self.user_answer[self.current])
 answered_correctly_list.append(self.answered_correctly[self.current])
 self.current += 1
continue
else:
return self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 added_user.add(crr_user_id)
 pre_added_user = crr_user_id
pre_task_container_id = crr_task_container_id
pre_content_type_id = crr_content_type_id
user_answer_list.append(self.user_answer[self.current])
 answered_correctly_list.append(self.answered_correctly[self.current])
 self.current += 1
if pre_start < self.current:
return self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 else:
raise StopIteration()<choose_model_class>
X_train = train.iloc[:,1:]
y_train = train.iloc[:,0]
MAX_SEQ = 240
ACCEPTED_USER_CONTENT_SIZE = 2
EMBED_SIZE = 256
BATCH_SIZE = 64+32
DROPOUT = 0.1
class FFN(nn.Module):
def __init__(self, state_size = 200, forward_expansion = 1, bn_size = MAX_SEQ - 1, dropout=0.2):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, forward_expansion * state_size)
 self.relu = nn.ReLU()
self.bn = nn.BatchNorm1d(bn_size)
 self.lr2 = nn.Linear(forward_expansion * state_size, state_size)
 self.dropout = nn.Dropout(dropout)
 def forward(self, x):
x = self.relu(self.lr1(x))
x = self.bn(x)
 x = self.lr2(x)
 return self.dropout(x)
 class FFN0(nn.Module):
def __init__(self, state_size = 200, forward_expansion = 1, bn_size = MAX_SEQ - 1, dropout=0.2):
super(FFN0, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, forward_expansion * state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(forward_expansion * state_size, state_size)
 self.layer_normal = nn.LayerNorm(state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 x=self.layer_normal(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask =(np.triu(np.ones([seq_length, seq_length]), k = 1)).astype('bool')
 return torch.from_numpy(future_mask)
 class TransformerBlock(nn.Module):
def __init__(self, embed_dim, heads = 8, dropout = DROPOUT, forward_expansion = 1):
super(TransformerBlock, self ).__init__()
self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=heads, dropout=dropout)
 self.dropout = nn.Dropout(dropout)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim, forward_expansion = forward_expansion, dropout=dropout)
 self.ffn0 = FFN0(embed_dim, forward_expansion = forward_expansion, dropout=dropout)
 self.layer_normal_2 = nn.LayerNorm(embed_dim)
 def forward(self, value, key, query, att_mask):
att_output, att_weight = self.multi_att(value, key, query, attn_mask=att_mask)
 att_output = self.dropout(self.layer_normal(att_output + value))
att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x1 = self.ffn0(att_output)
 x = self.dropout(self.layer_normal_2(x + x1 + att_output))
return x.squeeze(-1), att_weight
class Encoder(nn.Module):
def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, num_layers=1, heads = 8):
super(Encoder, self ).__init__()
self.n_skill, self.embed_dim = n_skill, embed_dim
self.embedding = nn.Embedding(2 * n_skill + 1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq - 1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.layers = nn.ModuleList([TransformerBlock(embed_dim, forward_expansion = forward_expansion)for _ in range(num_layers)])
 self.dropout = nn.Dropout(dropout)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = self.dropout(x + pos_x)
 x = x.permute(1, 0, 2)
 e = self.e_embedding(question_ids)
 e = e.permute(1, 0, 2)
 for layer in self.layers:
att_mask = future_mask(e.size(0)).to(device)
 x, att_weight = layer(e, x, x, att_mask=att_mask)
 x = x.permute(1, 0, 2)
 x = x.permute(1, 0, 2)
 return x, att_weight
class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, enc_layers=1, heads = 8):
super(SAKTModel, self ).__init__()
self.encoder = Encoder(n_skill, max_seq, embed_dim, dropout, forward_expansion, num_layers=enc_layers)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
x, att_weight = self.encoder(x, question_ids)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
class TestDataset(Dataset):
def __init__(self, samples, test_df, n_skill, max_seq=100):
super(TestDataset, self ).__init__()
self.samples, self.user_ids, self.test_df = samples, [x for x in test_df["user_id"].unique() ], test_df
self.n_skill, self.max_seq = n_skill, max_seq
def __len__(self):
return self.test_df.shape[0]
def __getitem__(self, index):
test_info = self.test_df.iloc[index]
user_id = test_info['user_id']
target_id = test_info['content_id']
content_id_seq = np.zeros(self.max_seq, dtype=int)
 answered_correctly_seq = np.zeros(self.max_seq, dtype=int)
 if user_id in self.samples.index:
content_id, answered_correctly = self.samples[user_id]
seq_len = len(content_id)
 if seq_len >= self.max_seq:
content_id_seq = content_id[-self.max_seq:]
answered_correctly_seq = answered_correctly[-self.max_seq:]
else:
content_id_seq[-seq_len:] = content_id
answered_correctly_seq[-seq_len:] = answered_correctly
x = content_id_seq[1:].copy()
x +=(answered_correctly_seq[1:] == 1)* self.n_skill
questions = np.append(content_id_seq[2:], [target_id])
 return x, questions 
<define_variables>
X_train = X_train.values.reshape(-1, 28, 28, 1)/255.
 test = test.values.reshape(-1, 28, 28, 1)/255.
 y_train = to_categorical(y_train, 10 )
class config:
FOLD = 0
ROOT_PATH = "/kaggle/input/riiid-xgboost-model-and-features"
MODEL_NAME = "xgb_v17_06_f0"
validaten_flg = False
DDOF = 1<load_pretrained>
random_seed = 0
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=random_seed )
model_path = f"{config.ROOT_PATH}/{config.MODEL_NAME}/{config.MODEL_NAME}"
model_name = f"{config.MODEL_NAME}_model.bst"
model_meta = f"{config.MODEL_NAME}_assets_model_metadata.json"
model = Predictor.from_path(model_path, model_name=model_name, meta_name=model_meta)
 model._extract_model_metadata()
feature_names = model._feature_names
categorical_features = [feature_names[i] for i in model._ohe_categorical_index_vocab.keys() ]
print("features:", len(feature_names))
categorical_features
print(model._mhe_categorical_index_vocab)
 assert len(model._mhe_categorical_index_vocab)== 0<load_pretrained>
datagen = ImageDataGenerator(
 rotation_range=10,
width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=0.1
)
group = joblib.load("/kaggle/input/riiid-sakt-model/group.pkl.zip")
 n_skill = joblib.load("/kaggle/input/riiid-sakt-model/skills.pkl.zip")
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 def create_model() :
return SAKTModel(n_skill, max_seq=MAX_SEQ, embed_dim=EMBED_SIZE, forward_expansion=1, enc_layers=1, heads=4, dropout=0.1)
 sakt_model = create_model()
sakt_model.load_state_dict(torch.load("/kaggle/input/riiid-sakt-model/sakt_model.pt"))
sakt_model.to(device)
 sakt_model.eval()
sakt_model_b = create_model()
sakt_model_b.load_state_dict(torch.load("/kaggle/input/riiid-sakt-model/best_sakt_model_1.pt"))
sakt_model_b.to(device)
 sakt_model_b.eval()
print("all model loaded" )<load_from_csv>
model = Sequential()
model.add(Conv2D(32,(5,5), padding='same', input_shape=X_train.shape[1:], activation='relu'))
model.add(Conv2D(32,(5,5), padding='same', activation='relu'))
model.add(MaxPool2D(2,2))
model.add(Conv2D(64,(3,3), padding='same', activation='relu'))
model.add(Conv2D(64,(3,3), padding='same', activation='relu'))
model.add(MaxPool2D(2,2))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
model.summary()
content_agg_feats = pd.read_csv(f"{config.ROOT_PATH}/content_agg_feats.csv")
 question_tags_ohe = pd.read_csv(f"{config.ROOT_PATH}/question_tags_ohe.csv")
 lecture_tags_ohe = pd.read_csv(f"{config.ROOT_PATH}/lecture_tags_ohe.csv")
 questions = pd.read_csv("/kaggle/input/riiid-test-answer-prediction/questions.csv")
 question_tags_ohe = question_tags_ohe.rename(columns={'question_id':'content_id'})
 lecture_tags_ohe = lecture_tags_ohe.rename(columns={'lecture_id':'content_id'})
 questions = questions.rename(columns={'question_id':'content_id'} ).drop(["bundle_id", "correct_answer", "tags"], axis=1)
 content_agg_feats = content_agg_feats.merge(question_tags_ohe, how="left", on="content_id")
 content_agg_feats = content_agg_feats.merge(questions, how="left", on="content_id")
 content_agg_feats = content_agg_feats.merge(lecture_tags_ohe, how="outer", on="content_id")
 content_agg_feats = content_agg_feats.fillna(0)
 content_agg_feats_v = content_agg_feats.values
content_agg_feats_c = content_agg_feats.columns.values[1:]
q_ohe_dic = {i: v for i, v in enumerate(question_tags_ohe.set_index("content_id" ).values)}
l_ohe_dic = {i: row.values for i, row in lecture_tags_ohe.drop("type_of", axis=1 ).set_index("content_id" ).iterrows() }
del lecture_tags_ohe, question_tags_ohe, questions, content_agg_feats
gc.collect()<load_from_csv>
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'] )
user_agg_feats_even = pd.read_csv(f"{config.ROOT_PATH}/user_agg_feat_even.csv")
 user_agg_feats_odd = pd.read_csv(f"{config.ROOT_PATH}/user_agg_feat_odd.csv")
 user_agg_feats_df = pd.concat([user_agg_feats_even, user_agg_feats_odd])
 user_agg_feats_v = user_agg_feats_df.values
del user_agg_feats_df, user_agg_feats_even, user_agg_feats_odd
gc.collect()<load_from_csv>
EPOCHS = 30
BATCH_SIZE = 20
callback_list = [
ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=1),
EarlyStopping(monitor='val_loss', min_delta=0.0005, patience=4)
 ]
history = model.fit(datagen.flow(X_train, y_train, batch_size=BATCH_SIZE),
epochs=EPOCHS,
callbacks=callback_list,
validation_data=(X_val, y_val),
steps_per_epoch=X_train.shape[0] // BATCH_SIZE )
user_last_timestamp = pd.read_csv(f"{config.ROOT_PATH}/user_last_timestamp.csv")
 last_timestamp_dic = {k: v for k, v in user_last_timestamp.values}
del user_last_timestamp
gc.collect()<load_pretrained>
results = model.predict(test)
 results = np.argmax(results, axis=1)
 results = pd.Series(results, name='Label')
 submission = pd.concat([pd.Series(range(1,28001), name='ImageID'), results], axis=1)
 submission.to_csv('submission.csv', index=False )