Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<feature_engineering>
import os
import random
import numpy as np
import pandas as pd
import tensorflow as tf
from sklearn.metrics import log_loss
from sklearn.model_selection import KFold
def get_time_mean_diff(test):
time_mean_diff = []
user_list = []
time_adm_list = []
for user_id, timestamp_diff1 in test[['user_id', 'timestamp_diff1']].values:
time_adm = min(100000, timestamp_diff1)
 user_list.append(user_id)
 time_adm_list.append(time_adm)
 if user_id in time_adm_dict:
time_mean_diff.append(time_adm - time_adm_dict[user_id] / user_dict_count[user_id])
 else:
time_mean_diff.append(0)
 for u, t in zip(user_list, time_adm_list):
time_adm_dict[u] += t
return time_mean_diff<define_variables>
SEED = 9
def all_seeds(s):
random.seed(s)
 os.environ['PYTHONHASHSEED'] = str(s)
 np.random.seed(s)
 tf.random.set_seed(s)
 all_seeds(SEED )
env = riiideducation.make_env()
iter_test = env.iter_test()
prior_test_df = None
model = lgb.Booster(model_file='.. /input/riiid-model-42v1/model42v1.txt' )<set_options>
test_df = pd.read_csv('.. /input/lish-moa/test_features.csv')
 train_df = pd.read_csv('.. /input/lish-moa/train_features.csv')
 tr_target_df = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 sample_df = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 target_cols = tr_target_df.columns[1:]
N_TARGETS = len(target_cols )
gc.collect()<prepare_output>
def preprocess_df(df):
df['cp_type'] =(df['cp_type'] == 'trt_cp' ).astype(int)
 df['cp_dose'] =(df['cp_dose'] == 'D2' ).astype(int)
 return df
x_train = preprocess_df(train_df.drop(columns="sig_id"))
y_train = tr_target_df.drop(columns="sig_id")
 x_test = preprocess_df(test_df.drop(columns="sig_id"))
N_FEATURES = x_train.shape[1]
%%time
for(test_df, sample_prediction_df)in iter_test:
if prior_test_df is not None:
prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)
 for user_id, content_id, part, cluster_id, answered_correctly, content_mean in prior_test_df[['user_id', 'content_id', 'part', 'cluster_id', 'answered_correctly', 'content_mean']].values:
user_dict_sum[user_id] += answered_correctly
user_dict_count[user_id] += 1
part_dict_sum[(user_id, part)] += answered_correctly
part_dict_count[(user_id, part)] += 1
cluster_dict_sum[(user_id, cluster_id)] += answered_correctly
cluster_dict_count[(user_id, cluster_id)] += 1
lag_dict[user_id] = answered_correctly
content_mean_sum_dict[user_id] += content_mean
test_df = pd.merge(test_df, questions_df, left_on='content_id', right_on='question_id', how='left')
 test_df = pd.merge(test_df, lectures_df, on=['content_id', 'content_type_id'], how='left')
 prior_test_df = test_df.copy()
test_df['prior_tag'] = test_df.groupby('user_id')['lecture_tag'].shift()
test_df['prior_tag'].fillna(-1, inplace=True)
 test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df['user_correctness'], test_df['cumcount_u'], \
test_df['part_avg'], test_df['cumcount_p'], \
test_df['cluster0_avg'], test_df['cluster1_avg'], test_df['cluster2_avg'], test_df['cumcount_cl'], \
test_df['target_lag'], test_df['user_rating'] = get_user_data(test_df)
 test_df['timestamp_diff1'], test_df['timestamp_diff2'] = get_timestamp_diff(test_df)
 test_df['attempt'] = get_attempt(test_df)
 test_df['prior_tag'] = get_prior_tag(test_df)
 update_last_lecture_dict(prior_test_df)
 test_df['timestamp_diff1'] = test_df['timestamp_diff1'] / test_df['task_num']
test_df['timestamp_diff2'] = test_df['timestamp_diff2'] / test_df['task_num']
test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df['prior_question_elapsed_time'].fillna(22000., inplace=True)
 test_df['timestamp_diff1'].fillna(25572., inplace=True)
 test_df['timestamp_diff2'].fillna(53309., inplace=True)
 test_df['time_mean_diff'] = get_time_mean_diff(test_df)
 test_df = test_df.astype(features_dtypes)
 test_df['answered_correctly'] = model.predict(test_df[features].values)
 env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<import_modules>
EPOCHS = 64
BATCH_SIZE = 128
FOLDS = 5
REPEATS = 5
LR = 0.0008
N_TARGETS = len(target_cols)
 def multi_log_loss(y_true, y_pred):
losses = []
for col in y_true.columns:
losses.append(log_loss(y_true.loc[:, col], y_pred.loc[:, col]))
return np.mean(losses )
import psutil
import joblib
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader<split>
def my_model() :
model = tf.keras.Sequential([
tf.keras.layers.Input(N_FEATURES),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.1),
tf.keras.layers.Dense(3500, activation="relu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(1750, activation="relu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(875, activation="relu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.1),
tf.keras.layers.Dense(412, activation="relu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(N_TARGETS, activation="sigmoid")
 ])
 model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate = LR),
loss = 'binary_crossentropy',
metrics = ["accuracy"])
 return model
env = riiideducation.make_env()
iter_test = env.iter_test()<init_hyperparams>
def train(resume_models = None, repeat_number = 0, folds = 5, skip_folds = 0):
models = []
oof_pred = y_train.copy()
kfold = KFold(folds, shuffle = True, random_state = 9)
 for fold,(i_tr, i_va)in enumerate(kfold.split(x_train)) :
print('-'*85)
 print(f'Repeat {repeat_number}, Fold {fold}')
 cb_lr_schedule = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_loss',
factor = 0.4,
patience = 1,
verbose = 2,
min_delta = 0.0001,
mode = 'auto')
 cb_early_stop = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss',
min_delta = 0,
patience = 4,
verbose = 1,
mode = 'min')
 checkpt_path = f'repeat:{repeat_number}_Fold:{fold}.hdf5'
cb_checkpt = tf.keras.callbacks.ModelCheckpoint(checkpt_path,
monitor = 'val_loss',
verbose = 2,
save_best_only = True,
save_weights_only = True,
mode = 'min')
 model = my_model()
model.fit(x_train.values[i_tr], y_train.values[i_tr],
validation_data =(x_train.values[i_va], y_train.values[i_va]),
callbacks = [cb_lr_schedule, cb_early_stop, cb_checkpt],
epochs = EPOCHS,
batch_size = BATCH_SIZE,
verbose = 2)
 model.load_weights(checkpt_path)
 oof_pred.loc[i_va, :] = model.predict(x_train.values[i_va])
 models.append(model)
 return models, oof_pred
MAX_SEQ = 100
n_part = 7
D_MODEL = 256
N_LAYER = 2
DROPOUT = 0.1<feature_engineering>
models = []
oof_pred = []
for i in range(REPEATS):
m, oof = train(repeat_number = i, folds = FOLDS)
 print('-'*85)
 models = models + m
oof_pred.append(oof )
def feature_time_lag(df, time_dict):
tt = np.zeros(len(df), dtype=np.int64)
 for ind, row in enumerate(df[['user_id','timestamp','task_container_id']].values):
if row[0] in time_dict.keys() :
if row[2]-time_dict[row[0]][1] == 0:
tt[ind] = time_dict[row[0]][2]
else:
t_last = time_dict[row[0]][0]
task_ind_last = time_dict[row[0]][1]
tt[ind] = row[1]-t_last
time_dict[row[0]] =(row[1], row[2], tt[ind])
 else:
time_dict[row[0]] =(row[1], row[2], -1)
 tt[ind] = 0
df["time_lag"] = tt
return df 
<define_search_model>
mean_oof_pred = y_train.copy()
mean_oof_pred.loc[:, target_cols] = 0
for i, p in enumerate(oof_pred):
print(f"Repeat {i} OOF Log Loss: {multi_log_loss(y_train, p)}")
 mean_oof_pred.loc[:, target_cols] += p[target_cols]
mean_oof_pred.loc[:, target_cols] /= len(oof_pred)
 print(f"Mean OOF Log Loss: {multi_log_loss(y_train, mean_oof_pred)}")
 mean_oof_pred.loc[x_train['cp_type'] == 0, target_cols] = 0
print(f"Mean OOF Log Loss(ctl adjusted): {multi_log_loss(y_train, mean_oof_pred)}" )
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(DROPOUT)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAINTModel(nn.Module):
def __init__(self, n_skill, n_part, max_seq=MAX_SEQ, embed_dim= 128, elapsed_time_cat_flag = True):
super(SAINTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.n_cat = n_part
self.elapsed_time_cat_flag = elapsed_time_cat_flag
self.e_embedding = nn.Embedding(self.n_skill+1, embed_dim)
 self.c_embedding = nn.Embedding(self.n_cat+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.res_embedding = nn.Embedding(2+1, embed_dim)
 if self.elapsed_time_cat_flag == True:
self.elapsed_time_embedding = nn.Embedding(300+1, embed_dim)
 self.lag_embedding1 = nn.Embedding(300+1, embed_dim)
 self.lag_embedding2 = nn.Embedding(1440+1, embed_dim)
 self.lag_embedding3 = nn.Embedding(365+1, embed_dim)
 else:
self.elapsed_time_embedding = nn.Linear(1, embed_dim, bias=False)
 self.lag_embedding = nn.Linear(1, embed_dim, bias=False)
 self.exp_embedding = nn.Embedding(2+1, embed_dim)
 self.transformer = nn.Transformer(nhead=8, d_model = embed_dim, num_encoder_layers= N_LAYER, num_decoder_layers= N_LAYER, dropout = DROPOUT)
 self.dropout = nn.Dropout(DROPOUT)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, question, part, response, elapsed_time, lag_time, exp):
device = question.device
question = self.e_embedding(question)
 part = self.c_embedding(part)
 pos_id = torch.arange(question.size(1)).unsqueeze(0 ).to(device)
 pos_id = self.pos_embedding(pos_id)
 res = self.res_embedding(response)
 exp = self.exp_embedding(exp)
 if self.elapsed_time_cat_flag == True:
elapsed_time = torch.true_divide(elapsed_time, 1000)
 elapsed_time = torch.round(elapsed_time)
 elapsed_time = torch.where(elapsed_time.float() <= 300, elapsed_time, torch.tensor(300.0 ).to(device)).long()
elapsed_time = self.elapsed_time_embedding(elapsed_time)
 lag_time = torch.true_divide(lag_time, 1000)
 lag_time = torch.round(lag_time)
 lag_time1 = torch.where(lag_time.float() <= 300, lag_time, torch.tensor(300.0 ).to(device)).long()
lag_time = torch.true_divide(lag_time, 60)
 lag_time = torch.round(lag_time)
 lag_time2 = torch.where(lag_time.float() <= 1440, lag_time, torch.tensor(1440.0 ).to(device)).long()
lag_time = torch.true_divide(lag_time, 1440)
 lag_time = torch.round(lag_time)
 lag_time3 = torch.where(lag_time.float() <= 365, lag_time, torch.tensor(365.0 ).to(device)).long()
lag_time1 = self.lag_embedding1(lag_time1)
 lag_time2 = self.lag_embedding2(lag_time2)
 lag_time3 = self.lag_embedding3(lag_time3)
 else:
elapsed_time = elapsed_time.view(-1,1)
 elapsed_time = self.elapsed_time_embedding(elapsed_time)
 elapsed_time = elapsed_time.view(-1, MAX_SEQ-1, self.embed_dim)
 lag_time = lag_time.view(-1,1)
 lag_time = self.lag_embedding(lag_time)
 lag_time = lag_time.view(-1, MAX_SEQ-1, self.embed_dim)
 enc = question + part + pos_id + exp
dec = pos_id + res + elapsed_time + lag_time1 + lag_time2 + lag_time3
enc = enc.permute(1, 0, 2)
 dec = dec.permute(1, 0, 2)
 mask = future_mask(enc.size(0)).to(device)
 att_output = self.transformer(enc, dec, src_mask=mask, tgt_mask=mask, memory_mask = mask)
 att_output = self.layer_normal(att_output)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 x = self.pred(x)
 return x.squeeze(-1 )<load_pretrained>
test_pred = sample_df.copy()
test_pred[target_cols] = 0
for model in models:
test_pred.loc[:,target_cols] += model.predict(x_test)
 test_pred.loc[:,target_cols] /= len(models)
 test_pred.loc[x_test['cp_type'] == 0, target_cols] = 0
test_pred.to_csv('submission.csv', index=False )