Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<count_values>
%matplotlib inline
print('Number of outliers:', len(train_data)-mask.sum() )<split>
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
import keras
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size = 0.2, random_state=2021 )<choose_model_class>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
model = XGBRegressor(objective='reg:squarederror',
booster = "gbtree",
eval_metric = "rmse",
tree_method = "gpu_hist",
n_estimators = 600,
learning_rate = 0.04,
eta = 0.1,
max_depth = 7,
subsample=0.85,
colsample_bytree = 0.85,
colsample_bylevel = 0.8,
alpha = 0,
random_state = 2021)
 model.fit(X_train, y_train)
 y_val_pred = model.predict(X_val)
 print('Score validation data:', np.sqrt(mean_squared_error(y_val, y_val_pred)) )<predict_on_test>
Y_train = train["label"]
X_train = train.drop(labels = ["label"],axis = 1)
y_test = model.predict(X_test )<prepare_output>
X_train = X_train / 255.0
test = test / 255.0
output = samp_subm.copy()
output['target'] = y_test<save_to_csv>
Y_train = to_categorical(Y_train, num_classes = 10 )
output.to_csv('submission.csv', index=False )<import_modules>
X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1 )
import os
import sys
import math
import pickle
import psutil
import random
import json
import numpy as np
import torch
from torch import nn
import torch.nn.functional as F
from torch.utils.data import Dataset, DataLoader
import pandas as pd
import riiideducation<define_variables>
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5),activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 32, kernel_size =(5,5),activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))
seed = 0
random.seed(seed)
 torch.random.manual_seed(seed)
 n_workers = os.cpu_count()
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 cfg_path = '/kaggle/input/riiid-mydata/cfg.json'
train_path = '/kaggle/input/riiid-mydata/train.pkl'
tag_path = '/kaggle/input/riiid-mydata/tags.csv'
states_path = '/kaggle/input/riiid-mydata/states.pickle'
model_path = '/kaggle/input/riiid-mydata/AIKT_08-12_17-21.pt'
B = 512
MAX_LEN = 128
N_LEVELS = 21
MAX_LAG = 30 * 7 * 24 * 60
N_LAYERS = 1
D_MODEL = 256
IS_LITE = True<choose_model_class>
model.compile(optimizer = 'adam' , loss = "categorical_crossentropy", metrics=["accuracy"] )
class FFN(nn.Module):
def __init__(self, d_model, dropout=0.0):
super().__init__()
self.lr1 = nn.Linear(d_model, d_model)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(d_model, d_model)
 self.dropout = nn.Dropout(dropout)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.dropout(x)
 x = self.lr2(x)
 return x
class AIKTMultiheadAttention(nn.Module):
def __init__(self, d_model, n_heads=8, d_qkv=None, use_proj=True, dropout=0.1):
super().__init__()
if d_qkv is None or not use_proj:
assert d_model % n_heads == 0
d_qkv = d_model // n_heads
d_inner = d_qkv * n_heads
self.scale = d_model **(-0.5)
 if use_proj:
self.Q = nn.Linear(d_model, d_inner)
 self.K = nn.Linear(d_model, d_inner)
 self.V = nn.Linear(d_model, d_inner)
 self.out_proj = nn.Linear(d_inner, d_model)
 else:
self.Q = self.K = self.V = self.out_proj = lambda x: x
self.reshape_for_attn = lambda x: x.reshape(*x.shape[:-1], n_heads, d_qkv)
 self.recover_from_attn = lambda x: x.reshape(*x.shape[:-2], d_inner)
 self.dropout = nn.Dropout(dropout)
 def forward(self, query, key, value, attn_mask=None, rel_pos_embd=None):
query, key, value =(
 self.reshape_for_attn(query),
self.reshape_for_attn(key),
self.reshape_for_attn(value)
)
 attn_scores = torch.einsum('ibnd,jbnd->ijbn',(query, key)) * self.scale
if rel_pos_embd is not None:
attn_scores += rel_pos_embd
if attn_mask is not None:
assert attn_mask.dtype == torch.bool, 'Only bool type is supported for masks.'
assert attn_mask.ndim == 2, 'Only 2D attention mask is supported'
assert attn_mask.shape == attn_scores.shape[:2], 'Incorrect mask shape: {}.Expect: {}'.format(attn_mask.shape, attn_scores.shape[:2])
 mask = torch.zeros_like(attn_mask, dtype=torch.float)
 mask.masked_fill_(attn_mask, float('-inf'))
mask = mask.view(*mask.shape, 1, 1)
 attn_scores += mask
attn_weights = torch.softmax(attn_scores, dim=1)
 attn_weights = self.dropout(attn_weights)
 out = torch.einsum('ijbn,jbnd->ibnd',(attn_weights, value))
out = self.recover_from_attn(out)
 out = self.out_proj(out)
 attn_weights = attn_weights.mean(-1 ).permute(-1, 0, 1)
 return out, attn_weights
class AIKTModel(nn.Module):
def __init__(
 self, n_exercises, n_levels, max_lag, n_layers, d_model,
n_heads=8, is_lite=True, dropout=0.1, lmda=1 
):
super().__init__()
self.exercise_embd = nn.Embedding(n_exercises, d_model)
 self.rate_to_level = lambda p: torch.round(p *(n_levels - 1)).long()
self.level_embd = nn.Embedding(n_levels, d_model)
 self.correct_embd = nn.Embedding(2, d_model)
 self.max_lag = max_lag
self.n_lag_buckets = 2 * math.ceil(math.log(max_lag))
self.rel_lag_embd = nn.Embedding(self.n_lag_buckets, n_heads)
 self.lmda = lmda
self.enc = nn.ModuleList([
nn.ModuleList([
AIKTMultiheadAttention(d_model, n_heads, use_proj=not is_lite, dropout=dropout),
None if is_lite else nn.LayerNorm(d_model),
None if is_lite else FFN(d_model, dropout=dropout),
None if is_lite else nn.LayerNorm(d_model)
 ])for _ in range(n_layers)
 ])
 self.dec = nn.ModuleList([
nn.ModuleList([
AIKTMultiheadAttention(d_model, n_heads, use_proj=not is_lite, dropout=dropout),
None if is_lite else nn.LayerNorm(d_model),
None if is_lite else FFN(d_model, dropout=dropout),
None if is_lite else nn.LayerNorm(d_model)
 ])for _ in range(n_layers)
 ])
 self.is_lite = is_lite
self.predict = nn.Linear(d_model, 1)
 self.dropout = nn.Dropout(dropout)
 def lag_to_bucket(self, lag_time):
n_exact = self.n_lag_buckets // 2
acc_lag_time = torch.cumsum(lag_time, dim=-1 ).unsqueeze(-1)
 rel_lag_time = torch.clamp(
 acc_lag_time - acc_lag_time.transpose(-1, -2), min=0, max=self.max_lag 
)
 rel_lag_time = torch.cat(
 [rel_lag_time[:, :, :1], rel_lag_time[:, :, :-1]], dim=-1 
)
 buckets_for_long_lag = n_exact - 1 + torch.ceil(
 torch.log(rel_lag_time / n_exact)/ math.log(self.max_lag / n_exact)*(self.n_lag_buckets - n_exact)
)
 buckets = torch.where(rel_lag_time < n_exact, rel_lag_time, buckets_for_long_lag.long())
 return buckets.permute(1, 2, 0)
 def forward(self, e, c, r, lt, attn_mask=None):
src = self.exercise_embd(e)
 src = src.transpose(0, 1)
 enc_attn_weights = []
for self_attn, ln1, ffn, ln2 in self.enc:
out, attn_weights = self_attn(src, src, src, attn_mask=attn_mask)
 if self.is_lite:
src = self.dropout(out)
 else:
src = ln1(src + self.dropout(out))
out = ffn(src)
 src = ln2(src + self.dropout(out))
enc_attn_weights.append(attn_weights)
 kc = src.transpose(0, 1)
 src = kc + self.correct_embd(c)
 src = F.pad(src[:, :-1, :], [0, 0, 1, 0] ).transpose(0, 1)
 tgt = self.exercise_embd(e)+ self.level_embd(self.rate_to_level(r))
tgt = tgt.transpose(0, 1)
 rel_pos_embd = self.rel_lag_embd(self.lag_to_bucket(lt))
dec_attn_weights = []
for cross_attn, ln1, ffn, ln2 in self.dec:
out, attn_weights = cross_attn(tgt, src, src, attn_mask=attn_mask, rel_pos_embd=rel_pos_embd)
 if self.is_lite:
tgt = self.dropout(out)
 else:
tgt = ln1(tgt + self.dropout(out))
out = ffn(tgt)
 tgt = ln2(tgt + self.dropout(out))
dec_attn_weights.append(attn_weights)
 tgt = tgt.transpose(0, 1)
 logit = self.predict(tgt)
 return logit.squeeze(-1),(enc_attn_weights, dec_attn_weights )<load_from_csv>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )
class KaggleOnlineDataset(Dataset):
def __init__(self, train_path, tag_path, states_path, n_exercises, cols, max_len):
super().__init__()
self.df = pd.read_pickle(train_path)
 self.test_df = None
tag_df = pd.read_csv(tag_path, usecols=['exercise_id', 'bundle_id', 'part', 'correct_rate', 'frequency'])
 assert np.all(tag_df['exercise_id'].values == np.arange(n_exercises))
self.parts, self.correct_rate, self.frequency = tag_df[['part', 'correct_rate', 'frequency']].values.T
self.lag_info = pickle.load(open(states_path, 'rb'))
self.n_exercises = n_exercises
self.cols = cols
self.max_len = max_len
def __len__(self):
assert self.test_df is not None, 'Please call update() first'
return len(self.test_df)
 def __getitem__(self, idx):
new_observation = self.test_df.iloc[idx]
user_id = new_observation['user_id']
new_data = {col: np.array([new_observation.get(col, 0)])for col in self.cols}
if user_id in self.df.index:
old_items = self.df[user_id]
old_len = min(len(old_items[0]), self.max_len - 1)
 data = {key: np.append(old_item[-old_len:], new_data[key])for key, old_item in zip(self.cols, old_items)}
else:
old_len = 0
data = new_data
seq_len = old_len + 1
data['part'] = self.parts[data['exercise_id']]
data['correct_rate'] = self.correct_rate[data['exercise_id']]
dtype_map = {key: int for key in self.cols + ['part']}
dtype_map['correct_rate'] = float
data = KaggleOnlineDataset._postpad_and_asdtype(data, self.max_len - seq_len, dtype_map)
 data['valid_len'] = np.array([seq_len], dtype=int)
 return data
@staticmethod
def _postpad_and_asdtype(data, pad, dtype_map):
return {
key: np.pad(item, [[0, pad]] ).astype(dtype_map[key])for key, item in data.items()
}
def update(self, test_df):
if self.test_df is not None and psutil.virtual_memory().percent < 90:
prev_df = self.test_df
prev_df['correct'] = np.array(eval(test_df.iloc[0]['prior_group_answers_correct'])) [self.was_exercise]
user_df = prev_df.groupby('user_id' ).apply(lambda udf: tuple(udf[col].values for col in self.cols))
for user_id, new_items in user_df.iteritems() :
if user_id in self.df.index:
self.df[user_id] = tuple(map(
 lambda old_item, new_item: np.append(old_item, new_item)[-min(self.max_len, len(old_item)+ 1):],
self.df[user_id],
new_items 
))
else:
self.df[user_id] = tuple(new_item for new_item in new_items)
 is_exercise =(test_df['content_type_id'] == 0)
 test_df = test_df[is_exercise]
test_df = test_df.rename(columns={'content_id': 'exercise_id', 'prior_question_elapsed_time': 'prior_elapsed'})
 test_df['prior_elapsed'] = test_df['prior_elapsed'].fillna(0 ).astype(int)
 lag = self._compute_new_lag(test_df)
 test_df['lag'] = np.where(
 np.logical_and(0 < lag, lag < 60 * 1000), 1, np.round(lag /(1000 * 60))
).astype(int)
 prior_elapsed = test_df['prior_elapsed'].values
test_df['prior_elapsed'] = np.where(
 np.logical_and(0 < prior_elapsed, prior_elapsed < 1000), 1, np.round(prior_elapsed / 1000)
 ).astype(int)
 test_df.reset_index(drop=True, inplace=True)
 self.test_df = test_df
self.was_exercise = is_exercise.values
return test_df
def _update_correct_rate(self, prev_df):
exercise_df = prev_df.groupby('exercise_id' ).aggregate({'correct': [sum, len]})['correct']
n_correct = np.arange(self.n_exercises)
 np.put(n_correct, exercise_df.index, exercise_df['sum'].values)
 n_correct = n_correct + np.round(self.correct_rate * self.frequency)
 more_frequency = np.arange(self.n_exercises)
 np.put(more_frequency, exercise_df.index, exercise_df['len'].values)
 self.frequency += more_frequency
correct_rate = n_correct / self.frequency
self.correct_rate = np.where(np.isfinite(correct_rate), correct_rate, 0.5)
 def _compute_new_lag(self, df):
last_states, exercise_id_to_bundle, bundle_id_to_size = self.lag_info
lag = np.zeros(len(df))
for i,(user_id, curr_timestamp, curr_exercise_id, prior_elapsed)in enumerate(
 df[['user_id', 'timestamp', 'exercise_id', 'prior_elapsed']].values 
):
curr_bundle_id = exercise_id_to_bundle[curr_exercise_id]
last_state = last_states.get(user_id, None)
 if last_state is None:
last_states[user_id] =(curr_timestamp, curr_bundle_id)
 lag[i] = 0
else:
last_timestamp, last_bundle_id = last_state
if curr_bundle_id == last_bundle_id:
lag[i] = 0
else:
last_states[user_id] =(curr_timestamp, curr_bundle_id)
 elapsed_offset = bundle_id_to_size[last_bundle_id] * prior_elapsed
lag[i] = curr_timestamp - last_timestamp - elapsed_offset
lag = np.clip(lag, a_min=0, a_max=None)
 self.lag_info =(last_states, exercise_id_to_bundle, bundle_id_to_size)
 return lag<categorify>
epochs = 40
batch_size = 80
def truncate_and_prepare_masks(items, valid_len, need_pad_mask=True, need_attn_mask=True):
max_len = valid_len.max()
device = max_len.device
out = [None if item is None else item[:, :max_len] for item in items]
pad_mask = torch.arange(max_len, device=device)>= valid_len if need_pad_mask else None
attn_mask = torch.triu(torch.ones(max_len, max_len), diagonal=1 ).to(device, torch.bool)if need_attn_mask else None
return out, pad_mask, attn_mask<load_pretrained>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train)
cfg = json.load(open(cfg_path, 'r'))
model = AIKTModel(cfg['n_exercises'], N_LEVELS, MAX_LAG, N_LAYERS, D_MODEL, is_lite=IS_LITE ).to(device)
 model.load_state_dict(torch.load(model_path))
model.eval()
testset = KaggleOnlineDataset(train_path, tag_path, states_path, cfg['n_exercises'], cfg['cols'], MAX_LEN)
 env = riiideducation.make_env()
iter_test = env.iter_test()
for test_df, _ in iter_test:
test_df = testset.update(test_df)
 testloader = DataLoader(testset, batch_size=B, shuffle=False, num_workers=n_workers, drop_last=False)
 outs = np.array([], dtype='float32')
 for data in testloader:
valid_len = data['valid_len'].to(device, torch.long)
 inputs, _, attn_mask = truncate_and_prepare_masks(
 [
data['exercise_id'].to(device, torch.long),
data['correct'].to(device, torch.long),
data['correct_rate'].to(device, torch.float),
data['lag'].to(device, torch.long)
 ],
valid_len,
need_pad_mask=False 
)
 out, _ = model(*inputs, attn_mask=attn_mask)
 out = torch.gather(out, 1, valid_len - 1 ).squeeze(-1)
 outs = np.append(outs, torch.sigmoid(out ).detach().cpu().numpy())
 test_df['answered_correctly'] = outs
env.predict(test_df[['row_id', 'answered_correctly']] )<load_from_csv>
history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_val,Y_val),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size
, callbacks=[learning_rate_reduction] )
session=pd.read_csv("/kaggle/input/airbnb-recruiting-new-user-bookings/sessions.csv.zip")
 print(session.shape)
 session.head()<load_from_csv>
results = model.predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )
train_user=pd.read_csv("/kaggle/input/airbnb-recruiting-new-user-bookings/train_users_2.csv.zip")
 print(train_user.shape)
 train_user.head()<load_from_csv>
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False )