Kaggle competition: Riiid Answer Correctness Prediction
all_data['NoLowQual']=(all_data['LowQualFinSF']==0)
 all_data['NoOpenPorch']=(all_data['OpenPorchSF']==0)
 all_data['NoWoodDeck']=(all_data['WoodDeckSF']==0)
 all_data['NoGarage']=(all_data['GarageArea']==0 )<categorify>
import gc
import random
from tqdm import tqdm
from sklearn.model_selection import train_test_split
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np
import random
import psutil
import datatable as dt
from collections import namedtuple
import os
import seaborn as sns
from sklearn.utils import shuffle
import datetime
import time
from sklearn.metrics import accuracy_score, precision_recall_fscore_support, matthews_corrcoef, roc_auc_score
import torch
import torch.nn as nn
import torch.nn.utils.rnn as rnn_utils
from torch.autograd import Variable
from torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler
from transformers import TrainingArguments
from transformers import AdamW, BertConfig
from transformers import get_linear_schedule_with_warmup
Bsmt=Bsmt.replace(to_replace='Po', value=1)
 Bsmt=Bsmt.replace(to_replace='Fa', value=2)
 Bsmt=Bsmt.replace(to_replace='TA', value=3)
 Bsmt=Bsmt.replace(to_replace='Gd', value=4)
 Bsmt=Bsmt.replace(to_replace='Ex', value=5)
 Bsmt=Bsmt.replace(to_replace='None', value=0)
 Bsmt=Bsmt.replace(to_replace='No', value=1)
 Bsmt=Bsmt.replace(to_replace='Mn', value=2)
 Bsmt=Bsmt.replace(to_replace='Av', value=3)
 Bsmt=Bsmt.replace(to_replace='Gd', value=4)
 Bsmt=Bsmt.replace(to_replace='Unf', value=1)
 Bsmt=Bsmt.replace(to_replace='LwQ', value=2)
 Bsmt=Bsmt.replace(to_replace='Rec', value=3)
 Bsmt=Bsmt.replace(to_replace='BLQ', value=4)
 Bsmt=Bsmt.replace(to_replace='ALQ', value=5)
 Bsmt=Bsmt.replace(to_replace='GLQ', value=6 )<feature_engineering>
MAX_SEQ = 256
Bsmt['BsmtScore']= Bsmt['BsmtQual'] * Bsmt['BsmtCond'] * Bsmt['TotalBsmtSF']
all_data['BsmtScore']=Bsmt['BsmtScore']<feature_engineering>
%%time
dtype = {'timestamp' :'int64',
'user_id' :'int32' ,
'content_id' :'int16',
'content_type_id' :'int8',
'answered_correctly':'int8'}
train_df = pd.read_csv('.. /input/riiid-test-answer-prediction/train.csv', usecols=[1, 2, 3, 4, 7], dtype=dtype)
 train_df.head()
Bsmt['BsmtFin'] =(Bsmt['BsmtFinSF1'] * Bsmt['BsmtFinType1'])+(Bsmt['BsmtFinSF2'] * Bsmt['BsmtFinType2'])
 all_data['BsmtFinScore']=Bsmt['BsmtFin']
all_data['BsmtDNF']=(all_data['BsmtFinScore']==0 )<define_variables>
dtype = {
'question_id' :'int16',
'bundle_id' :'int16',
'tags' :'str',
}
questions_df = pd.read_csv('.. /input/riiid-test-answer-prediction/questions.csv', usecols=[0, 1, 4], dtype=dtype)
 questions_df['tags'] = questions_df['tags'].map(lambda s: [int(v)for v in s.split() ], 'ignore')
 questions_df['tags'] = questions_df['tags'].fillna(0)
 questions_df['tags'] = questions_df['tags'].map(lambda s: [0]*5 if s==0 else s*(5//len(s)) +s[:5%len(s)] if len(s)<5 else s[:5])
 questions_df.head()
lot=['LotFrontage', 'LotArea','LotConfig','LotShape']
Lot=all_data[lot]
Lot.head()<categorify>
train_df = train_df[train_df.content_type_id == False]
train_df = train_df.sort_values(['timestamp'], ascending=True ).reset_index(drop = True )
garage=['GarageArea','GarageCars','GarageCond','GarageFinish','GarageQual','GarageType','GarageYrBlt']
Garage=all_data[garage]
Garage=Garage.replace(to_replace='Po', value=1)
 Garage=Garage.replace(to_replace='Fa', value=2)
 Garage=Garage.replace(to_replace='TA', value=3)
 Garage=Garage.replace(to_replace='Gd', value=4)
 Garage=Garage.replace(to_replace='Ex', value=5)
 Garage=Garage.replace(to_replace='None', value=0)
 Garage=Garage.replace(to_replace='Unf', value=1)
 Garage=Garage.replace(to_replace='RFn', value=2)
 Garage=Garage.replace(to_replace='Fin', value=3)
 Garage=Garage.replace(to_replace='CarPort', value=1)
 Garage=Garage.replace(to_replace='Basment', value=4)
 Garage=Garage.replace(to_replace='Detchd', value=2)
 Garage=Garage.replace(to_replace='2Types', value=3)
 Garage=Garage.replace(to_replace='Basement', value=5)
 Garage=Garage.replace(to_replace='Attchd', value=6)
 Garage=Garage.replace(to_replace='BuiltIn', value=7)
 Garage.head()<categorify>
known_bundle_ids = set(questions_df["bundle_id"].unique())
 print("number bundles =", len(known_bundle_ids))
n_bundles = max(known_bundle_ids)+ 1
n_bundles
def onehot(col_list):
global all_data
while len(col_list)!=0:
col=col_list.pop(0)
 data_encoded=pd.get_dummies(all_data[col], prefix=col)
 all_data=pd.merge(all_data, data_encoded, on='Id')
 all_data=all_data.drop(columns=col)
 print(all_data.shape )<categorify>
known_qtn_ids = set(questions_df["question_id"].unique())
 print("number questions =", len(known_qtn_ids))
n_skill = max(known_qtn_ids)+ 1
n_skill
onehot(list(non_numeric))<categorify>
known_tags = questions_df["tags"].apply(pd.Series ).unstack().reset_index().dropna()
known_tags = set(known_tags[0].astype('int16' ).unique())
 n_tags = max(known_tags)+ 1
n_tags, len(known_tags )
def log_transform(col_list):
transformed_col=[]
while len(col_list)!=0:
col=col_list.pop(0)
 if all_data[col].skew() > 0.5:
all_data[col]=np.log(all_data[col]+1)
 transformed_col.append(col)
 else:
pass
print(f"{len(transformed_col)} features had been tranformed")
 print(all_data.shape )<categorify>
gc.collect()
train_df.head()
numeric=all_data.select_dtypes(include=np.number)
 log_transform(list(numeric))<split>
train_df_1 = train_df[:train_df.shape[0]//2]
train_df_2 = train_df[train_df.shape[0]//2:]
del train_df; gc.collect()
group = train_df_1[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values, 
))
del train_df_1
gc.collect()
group
train=all_data[:len(train)]
test=all_data[len(train):]<import_modules>
group_2 = train_df_2[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values, 
))
del train_df_2
gc.collect()
group_2
from sklearn.linear_model import ElasticNet, Lasso
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.model_selection import cross_val_score
from sklearn.metrics import make_scorer
from sklearn import linear_model, model_selection, ensemble, preprocessing
from sklearn.tree import DecisionTreeRegressor
from sklearn.linear_model import LinearRegression, Ridge, Lasso, ElasticNet,SGDRegressor
from sklearn.ensemble import RandomForestRegressor,BaggingRegressor, GradientBoostingRegressor, AdaBoostRegressor
from sklearn.model_selection import GridSearchCV, KFold, cross_val_score
from sklearn.neighbors import KNeighborsRegressor
import xgboost as xgb
import lightgbm as lgb<compute_test_metric>
group_2_set = set(group_2.index)
 for k in group.index:
if k in group_2_set:
group[k] =(np.append(group[k][0], group_2[k][0]), np.append(group[k][1], group_2[k][1]))
group_2.pop(k)
 gc.collect()
group = group.append(group_2)
 del group_2; gc.collect()
print('len(group)=', len(group))
group
def rmse(predict, actual):
predict = np.array(predict)
 actual = np.array(actual)
 distance = predict - actual
square_distance = distance ** 2
mean_square_distance = square_distance.mean()
score = np.sqrt(mean_square_distance)
 return score
rmse_score = make_scorer(rmse)
 rmse_score<prepare_x_and_y>
random.seed(1 )
feature_names=list(all_data)
 X_train = train[feature_names]
X_test = test[feature_names]
y_train = saleprices['log_SalePrice']<compute_train_metric>
class SAKTDataset(Dataset):
def __init__(self, group, n_skill, max_seq=MAX_SEQ):
super(SAKTDataset, self ).__init__()
self.max_seq = max_seq
self.n_skill = n_skill
self.n_bundles = n_bundles
self.n_tags = n_tags
self.samples = group
self.user_ids = []
for user_id in group.index:
q, qa = group[user_id]
if len(q)< 2:
continue
self.user_ids.append(user_id)
 return
def __len__(self):
return len(self.user_ids)
 def __getitem__(self, index):
user_id = self.user_ids[index]
q_, qa_ = self.samples[user_id]
q_b_t_df = pd.DataFrame.from_dict({'question_id': q_})
 q_b_t_df = q_b_t_df.merge(questions_df, on='question_id', how='left')
 b_ = q_b_t_df.bundle_id.values
t_ = np.vstack(q_b_t_df.tags.values)
 del q_b_t_df
seq_len = len(q_)
 q = np.zeros(self.max_seq, dtype=int)
 qa = np.zeros(self.max_seq, dtype=int)
 b = np.zeros(self.max_seq, dtype=int)
 t = np.array([[0]*5]*self.max_seq, dtype=int)
 if seq_len >= self.max_seq:
if random.random() >0.1:
start = random.randint(0,(seq_len-self.max_seq))
end = start + self.max_seq
q[:] = q_[start:end]
qa[:] = qa_[start:end]
b[:] = b_[start:end]
t[:] = t_[start:end]
else:
q[:] = q_[-self.max_seq:]
qa[:] = qa_[-self.max_seq:]
b[:] = b_[-self.max_seq:]
t[:] = t_[-self.max_seq:]
else:
if random.random() >0.1:
start = 0
end = random.randint(2,seq_len)
 seq_len = end - start
q[-seq_len:] = q_[0:seq_len]
qa[-seq_len:] = qa_[0:seq_len]
b[-seq_len:] = b_[0:seq_len]
t[-seq_len:] = t_[0:seq_len]
else:
q[-seq_len:] = q_
qa[-seq_len:] = qa_
b[-seq_len:] = b_
t[-seq_len:] = t_
target_id = q[1:]
label = qa[1:]
x = np.zeros(self.max_seq-1, dtype=int)
 x = q[:-1].copy()
x +=(qa[:-1] == 1)* self.n_skill
bundle_x = b[:-1].copy()
tags_x = t[:-1].copy()
b_target = b[1:]
t_target = t[1:]
return x, target_id, label, bundle_x, tags_x, b_target, t_target
def score(model):
score = cross_val_score(model, X_train, y_train, cv=5, scoring=rmse_score ).mean()
return score<set_options>
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128):
super(SAKTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.embedding = nn.Embedding(2*n_skill+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.bundle_embedding = nn.Embedding(n_bundles, embed_dim)
 self.tag_embedding = nn.Embedding(n_tags, embed_dim)
 self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.outDense = nn.Linear(embed_dim, 2)
 self.outActivtn = nn.LogSoftmax(dim=1)
 self.NLLLoss = nn.NLLLoss()
return
def getTagsEmb(self, tags):

 t_emb = self.tag_embedding(tags)
 t_emb = torch.mean(t_emb, dim=-2)
 return t_emb
def forward(self, x, question_ids, bundle_x, tags_x, b_target, t_target, labels=None):

 device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 bundle_x = self.bundle_embedding(bundle_x)
 b_target = self.bundle_embedding(b_target)
 tags_x = self.getTagsEmb(tags_x)
 t_target = self.getTagsEmb(t_target)
 x = x + pos_x + bundle_x + tags_x
e = self.e_embedding(question_ids)+ b_target + t_target
x = x.permute(1, 0, 2);
e = e.permute(1, 0, 2)
 att_mask = future_mask(x.size(0)).to(device)
 att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
 att_output = self.layer_normal(att_output + e)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 out_logits = self.outDense(x)
 if labels is None:
out_logits = out_logits[:, -1]
Logits = namedtuple('Logits',['logits'])
 out_logits = Logits(out_logits)
 return out_logits
out_logits = out_logits.view(out_logits.shape[0] * out_logits.shape[1], -1)
 log_ps = self.outActivtn(out_logits)
 labels = torch.squeeze(labels.view(labels.shape[0] * labels.shape[1], -1), -1)
 batchLoss = self.NLLLoss(log_ps, labels)
 Loss_Logits = namedtuple('Loss_Logits',['loss','logits'])
 loss_logits = Loss_Logits(batchLoss, out_logits)
 return loss_logits
YouTubeVideo('EuBBz3bI-aA',width=700, height=350 )<choose_model_class>
frac = 0.05
temp = list(group.index)
 temp = shuffle(temp)
 frac = int(len(temp)* frac)
 temp2 = temp[:frac]
temp = temp[frac:]
train_group = group[temp]
eval_group = group[temp2]
del group
gc.collect()
print('len(train_group)=', len(train_group))
print('len(eval_group)=', len(eval_group))
train_group
model_Lasso= make_pipeline(RobustScaler() , Lasso(alpha =0.000327, random_state=18))
model_ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.00052, l1_ratio=0.70654, random_state=18))
model_GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =18)
 model_XGB=xgb.XGBRegressor(n_jobs=-1, n_estimators=849, learning_rate=0.015876,
max_depth=58, colsample_bytree=0.599653, colsample_bylevel=0.287441, subsample=0.154134, seed=18)
 model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 forest_reg = RandomForestRegressor(bootstrap=False, criterion='mse', max_depth=None,
max_features=60, max_leaf_nodes=None, min_impurity_decrease=0.0,
min_impurity_split=None, min_samples_leaf=1,
min_samples_split=2, min_weight_fraction_leaf=0.0,
n_estimators=70, n_jobs=1, oob_score=False, random_state=42,
verbose=0, warm_start=False)
<train_model>
def format_time(elapsed):

 elapsed_rounded = int(round(( elapsed)))
 return str(datetime.timedelta(seconds=elapsed_rounded))
def compute_metrics(labels, pred_logits):
preds = pred_logits.argmax(-1)
 precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='macro')
 acc = accuracy_score(labels, preds)
 mcc = matthews_corrcoef(labels, preds)
 softmax = nn.Softmax(dim=1)
 pred_pr = softmax(torch.tensor(pred_logits))
auc = roc_auc_score(labels, pred_pr[:, 1])
 metrics = {
'mcc' : mcc,
'accuracy' : acc,
'f1' : f1,
'precision': precision,
'recall' : recall,
'auc' : auc
}
return metrics
model_Lasso.fit(X_train, y_train)
 Lasso_Predictions=np.exp(model_Lasso.predict(X_test)) -1
model_ENet.fit(X_train, y_train)
 ENet_Predictions=np.exp(model_ENet.predict(X_test)) -1
model_XGB.fit(X_train, y_train)
 XGB_Predictions=np.exp(model_XGB.predict(X_test)) -1
model_GBoost.fit(X_train, y_train)
 GBoost_Predictions=np.exp(model_GBoost.predict(X_test)) -1
model_lgb.fit(X_train, y_train)
 lgb_Predictions=np.exp(model_lgb.predict(X_test)) -1
forest_reg.fit(X_train, y_train)
 forest_reg_Predictions=np.exp(forest_reg.predict(X_test)) -1 
<find_best_params>
class MyTrainer:
def __init__(self, model, args, train_dataset, eval_dataset, compute_metrics=compute_metrics):
self.model = model
self.args = args
self.train_dataset = train_dataset
self.eval_dataset = eval_dataset
self.compute_metrics = compute_metrics
self.isTrained = False
self.device = self.get_device_type()
self.optimizer = AdamW(model.parameters() ,
lr = args.learning_rate,
eps = args.adam_epsilon 
)
 self.epochs = self.args.num_train_epochs
self.train_dataloader, self.validation_dataloader, self.lr_scheduler, self.num_training_steps = self.get_dataLoaders()
return
def get_device_type(self):
if torch.cuda.is_available() :
device = torch.device("cuda")
 print('There are %d GPU(s)available.' % torch.cuda.device_count())
 print('We will use the GPU:', torch.cuda.get_device_name(0))
else:
print('No GPU available, using the CPU instead.')
 device = torch.device("cpu")
 return device
def get_dataLoaders(self):
if isinstance(self.train_dataset, torch.utils.data.IterableDataset):
train_sampler = None
else:
train_sampler = SequentialSampler(self.train_dataset)
 train_dataloader = DataLoader(
 self.train_dataset,
sampler = train_sampler,
batch_size = self.args.per_device_train_batch_size,
num_workers = 8 
)
 validation_dataloader = None
if self.eval_dataset:
validation_dataloader = DataLoader(
 self.eval_dataset,
sampler = SequentialSampler(self.eval_dataset),
batch_size = self.args.per_device_eval_batch_size,
num_workers = 8 
)
 num_training_steps = len(train_dataloader)* self.epochs
lr_scheduler = get_linear_schedule_with_warmup(self.optimizer,
num_warmup_steps = self.args.warmup_steps,
num_training_steps = num_training_steps)
 return train_dataloader, validation_dataloader, lr_scheduler, num_training_steps
def test_iterate_dataloader() :
for step, batch in enumerate(self.train_dataloader):
print(step)
 print(batch)
 break
return
def train(self):
seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 training_stats = []
total_t0 = time.time()
min_val_loss = 9999
min_train_loss = 9999
step = 0
for epoch_i in range(0, self.epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, self.epochs))
print('Training...')
 t0 = time.time()
total_train_loss = 0
self.model.train()
for stp, batch in enumerate(self.train_dataloader):
step += 1
if step % 50 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(self.train_dataloader), elapsed))
if(self.args.max_steps > 0 and self.args.max_steps < step)or \ 
(self.args.eval_steps> 0 and step % self.args.eval_steps==0 and step>0):
avg_train_loss = total_train_loss / step
training_time = format_time(time.time() - t0)
 if self.validation_dataloader:
print("Running Validation...")
 avg_val_loss, avg_val_f1, avg_val_mcc, avg_val_auc, avg_val_precision, avg_val_recall, avg_val_accuracy, validation_time = self.evaluate()
training_stats.append({
'epoch' : epoch_i + 1,
'training_loss' : avg_train_loss,
'eval_loss' : avg_val_loss,
'eval_f1' : avg_val_f1,
'eval_mcc' : avg_val_mcc,
'eval_precision': avg_val_precision,
'eval_recall' : avg_val_recall,
'eval_auc' : avg_val_auc,
'eval_accuracy' : avg_val_accuracy,
'training_time' : training_time,
'eval_time' : validation_time
})
 if avg_val_loss < min_val_loss:
min_val_loss = avg_val_loss
torch.save(model.state_dict() , "SAKT_Rishi.pt")
 self.model.zero_grad()
', batch)
 x = batch[0].to(device ).long()
target_id = batch[1].to(device ).long()
labels = batch[2].to(device ).long()
bundle_x = batch[3].to(device ).long()
tags_x = batch[4].to(device ).long()
b_target = batch[5].to(device ).long()
t_target = batch[6].to(device ).long()
output = self.model(x, target_id, bundle_x, tags_x, b_target, t_target, labels)
 loss = output.loss
logits = output.logits
total_train_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(self.model.parameters() , 1.0)
 self.optimizer.step()
self.lr_scheduler.step()
avg_train_loss = total_train_loss / len(self.train_dataloader)
 training_time = format_time(time.time() - t0)
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epcoh took: {:}".format(training_time))
if self.validation_dataloader:
print("
Running Validation...")
 avg_val_loss, avg_val_f1, avg_val_mcc, avg_val_auc, avg_val_precision, avg_val_recall, avg_val_accuracy, validation_time = self.evaluate()
training_stats.append({
'epoch' : epoch_i + 1,
'training_loss' : avg_train_loss,
'eval_loss' : avg_val_loss,
'eval_f1' : avg_val_f1,
'eval_mcc' : avg_val_mcc,
'eval_precision': avg_val_precision,
'eval_recall' : avg_val_recall,
'eval_auc' : avg_val_auc,
'eval_accuracy' : avg_val_accuracy,
'training_time' : training_time,
'eval_time' : validation_time
})
 if avg_val_loss < min_val_loss:
min_val_loss = avg_val_loss
torch.save(model.state_dict() , "SAKT_Rishi.pt")
 else:
training_stats.append({
'epoch' : epoch_i + 1,
'training_loss' : avg_train_loss,
'training_time' : training_time,
})
 if avg_train_loss < min_train_loss:
min_train_loss = avg_train_loss
torch.save(model.state_dict() , "SAKT_Rishi.pt")
 print("")
 print("Training complete!")
 print("Total training took {:}(h:mm:ss)".format(format_time(time.time() -total_t0)))
 self.isTrained = True
self.plot_train_stats(training_stats)
 return training_stats
def evaluate(self):
t0 = time.time()
self.model.eval()
total_eval_mcc = 0
total_eval_f1 = 0
total_eval_precision = 0
total_eval_recall = 0
total_eval_auc = 0
total_eval_accuracy = 0
total_eval_loss = 0
nb_eval_steps = 0
for batch in self.validation_dataloader:
with torch.no_grad() :
x = batch[0].to(device ).long()
target_id = batch[1].to(device ).long()
labels = batch[2].to(device ).long()
bundle_x = batch[3].to(device ).long()
tags_x = batch[4].to(device ).long()
b_target = batch[5].to(device ).long()
t_target = batch[6].to(device ).long()
output = self.model(x, target_id, bundle_x, tags_x, b_target, t_target, labels)
 loss = output.loss
logits = output.logits
total_eval_loss += loss.item()
logits = logits.detach().cpu().numpy()
label_ids = labels.to('cpu' ).numpy()
label_ids = np.squeeze(label_ids.reshape(( label_ids.shape[0] * label_ids.shape[1], -1)) , -1)
 metrics = self.compute_metrics(label_ids, logits)
 total_eval_mcc += metrics['mcc']
total_eval_f1 += metrics['f1']
total_eval_precision += metrics['precision']
total_eval_recall += metrics['recall']
total_eval_auc += metrics['auc']
total_eval_accuracy += metrics['accuracy']
avg_val_f1 = total_eval_f1 / len(self.validation_dataloader)
 print(" F1: {0:.3f}".format(avg_val_f1))
avg_val_mcc = total_eval_mcc / len(self.validation_dataloader)
 print(" MCC: {0:.3f}".format(avg_val_mcc))
avg_val_precision = total_eval_precision / len(self.validation_dataloader)
 print(" Precision: {0:.3f}".format(avg_val_precision))
avg_val_recall = total_eval_recall / len(self.validation_dataloader)
 print(" Recall: {0:.3f}".format(avg_val_recall))
avg_val_auc = total_eval_auc / len(self.validation_dataloader)
 print(" AUC: {0:.3f}".format(avg_val_auc))
avg_val_accuracy = total_eval_accuracy / len(self.validation_dataloader)
 print(" Accuracy: {0:.3f}".format(avg_val_accuracy))
avg_val_loss = total_eval_loss / len(self.validation_dataloader)
 validation_time = format_time(time.time() - t0)
 print(" Validation Loss: {0:.2f}".format(avg_val_loss))
print(" Validation took: {:}".format(validation_time))
return avg_val_loss, avg_val_f1, avg_val_mcc, avg_val_auc, avg_val_precision, avg_val_recall, avg_val_accuracy, validation_time
def plot_train_stats(self, training_stats):

 mccs = accuracies = f1_scores = precisions = recalls = auc = losses = epochs = -1
epochs = training_stats[-1]['epoch']
if 'eval_mcc' in training_stats[0]:
mccs = [e['eval_mcc'] for e in training_stats]
sns.lineplot(x=np.arange(1, epochs + 1), y=mccs, label='val_mcc')
 if 'eval_accuracy' in training_stats[0]:
accuracies = [e['eval_accuracy'] for e in training_stats]
sns.lineplot(x=np.arange(1, epochs + 1), y=accuracies, label='val_accuracy')
 if 'eval_f1' in training_stats[0]:
f1_scores = [e['eval_f1'] for e in training_stats]
sns.lineplot(x=np.arange(1, epochs + 1), y=f1_scores, label='val_f1')
 if 'eval_precision' in training_stats[0]:
precisions = [e['eval_precision'] for e in training_stats]
sns.lineplot(x=np.arange(1, epochs + 1), y=precisions, label='val_precision')
 if 'eval_recall' in training_stats[0]:
recalls = [e['eval_recall'] for e in training_stats]
sns.lineplot(x=np.arange(1, epochs + 1), y=recalls, label='val_recall')
 if 'eval_auc' in training_stats[0]:
auc = [e['eval_auc'] for e in training_stats]
sns.lineplot(x=np.arange(1, epochs + 1), y=mccs, label='val_auc')
 if 'eval_loss' in training_stats[0]:
losses = [e['eval_loss'] for e in training_stats]
if 'training_loss' in training_stats[0]:
tr_losses = [e['training_loss'] for e in training_stats]
sns.lineplot(x=np.arange(1, epochs + 1), y=tr_losses, label='tr_losses')
 plt.show()
print('mccs :', mccs)
 print('accuracies :', accuracies)
 print('precisions :', precisions)
 print('recalls :', recalls)
 print('f1_scores :', f1_scores)
 print('auc :', auc)
 print('losses :', losses)
 print('tr_losses :', tr_losses)
 return
def getTrainedModel(self):
if self.isTrained:
return self.model
return None
def predict(self, prediction_dataset, isRemoveLabels=True):

 prediction_sampler = SequentialSampler(prediction_dataset)
 prediction_dataloader = DataLoader(prediction_dataset, sampler=prediction_sampler, batch_size=self.args.per_device_eval_batch_size)
 print('Predicting labels for {:,} test sentences...'.format(len(prediction_dataset)))
 self.model.eval()
predictions = []
for batch in prediction_dataloader:
batch = {t:batch[t].to(self.device)for t in batch}
with torch.no_grad() :
if isRemoveLabels:
batch.pop('labels')
 for k in batch:
batch[k] = batch[k].to(self.device)
 outputs = model(**batch)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
predictions.append(logits)
 print('Done predictions for ', len(predictions), '/', len(prediction_dataloader), 'batches')
 print('Done prediction')
 pred_logits = np.concatenate(predictions, axis=0)
 return pred_logits, None, None
scores ={}
scores.update({'Lasso':score(model_Lasso)})
 scores.update({"Elastic Net":score(model_ENet)})
 scores.update({"XGB":score(model_XGB)})
 scores.update({"Gradient Boost":score(model_GBoost)})
 scores.update({"lgb":score(model_lgb)})
 scores.update({"Random Forest":score(forest_reg)} )<create_dataframe>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model = SAKTModel(n_skill, embed_dim=128)
 try:
model.load_state_dict(torch.load(".. /input/rishi-sakt-featureembeddings-riidchallenge/SAKT_Rishi.pt"))
except:
model.load_state_dict(torch.load(".. /input/rishi-sakt-featureembeddings-riidchallenge/SAKT_Rishi.pt", map_location='cpu'))
model.to(device )
scores_df =pd.DataFrame(list(scores.items()),columns=['Model','Score'])
 scores_df.sort_values(['Score'] )<import_modules>
training_args = TrainingArguments(
 output_dir = './results',
num_train_epochs= 1,
warmup_steps = 1000,
eval_steps = 500,
max_steps = 0,
learning_rate = 1e-6,
per_device_train_batch_size= 1024,
per_device_eval_batch_size = 1024, 
)
 trainer = MyTrainer(
 model = model,
args = training_args,
train_dataset = train_dataset,
eval_dataset = eval_dataset,
)
Image(url= "https://blogs.sas.com/content/subconsciousmusings/files/2017/05/weighted-unweighted.png" )<define_variables>
trainer.train()
ensemble =(Lasso_Predictions + XGB_Predictions + lgb_Predictions + ENet_Predictions +
forest_reg_Predictions)/5
ensemble<load_from_csv>
training_args = TrainingArguments(
 output_dir = './results',
num_train_epochs = 1,
warmup_steps = 100,
eval_steps = 300,
max_steps = 0,
learning_rate = 1e-7,
per_device_train_batch_size = 1024,
per_device_eval_batch_size = 1024, 
)
 trainer = MyTrainer(
 model = model,
args = training_args,
train_dataset = eval_dataset,
eval_dataset = None,
)
submission=pd.read_csv('.. /input/sample_submission.csv')
 submission['SalePrice']= ensemble<save_to_csv>
trainer.train()
submission.to_csv('submission.csv',index=False )<set_options>
group = train_group.append(eval_group)
 print('len(group)=', len(group))
del train_group, eval_group, train_dataset, eval_dataset, training_args, trainer
gc.collect()
group
warnings.filterwarnings('ignore')
 %matplotlib inline<import_modules>
class TestDataset(Dataset):
def __init__(self, samples, test_df, skills, max_seq=MAX_SEQ):
super(TestDataset, self ).__init__()
self.samples = samples
self.user_ids = [x for x in test_df["user_id"].unique() ]
self.test_df = test_df
self.skills = skills
self.n_skill = max(skills)+ 1
self.max_seq = max_seq
return
def __len__(self):
return self.test_df.shape[0]
def __getitem__(self, index):
test_info = self.test_df.iloc[index]
user_id = test_info["user_id"]
target_id = test_info["content_id"]
target_tags = test_info["tags"]
target_bundle = test_info["bundle_id"]
if target_id not in known_qtn_ids:
target_id = 0
if target_bundle not in known_bundle_ids:
target_bundle = 0
for i in range(len(target_tags)) :
if target_tags[i] not in known_tags:
target_tags[i] = 0
q = np.zeros(self.max_seq, dtype=int)
 qa = np.zeros(self.max_seq, dtype=int)
 b = np.zeros(self.max_seq, dtype=int)
 t = np.array([[0]*5]*self.max_seq, dtype=int)
 if user_id in self.samples.index:
q_, qa_ = self.samples[user_id]
q_b_t_df = pd.DataFrame.from_dict({'question_id': q_})
 q_b_t_df = q_b_t_df.merge(questions_df, on='question_id', how='left')
 b_ = q_b_t_df.bundle_id.values
t_ = np.vstack(q_b_t_df.tags.values)
 del q_b_t_df
seq_len = len(q_)
 if seq_len >= self.max_seq:
q = q_[-self.max_seq:]
qa = qa_[-self.max_seq:]
b = b_[-self.max_seq:]
t = t_[-self.max_seq:]
else:
q[-seq_len:] = q_
qa[-seq_len:] = qa_
b[-seq_len:] = b_
t[-seq_len:] = t_
x = np.zeros(self.max_seq-1, dtype=int)
 x = q[1:].copy()
x +=(qa[1:] == 1)* self.n_skill
questions = np.append(q[2:], [target_id])
 bundle_x = b[1:].copy()
tags_x = t[1:].copy()
b_target = np.append(b[2:], [target_bundle])
 t_target = np.vstack(( t[2:], target_tags))
return x, questions, bundle_x, tags_x, b_target, t_target
pd.__version__<import_modules>
env = riiideducation.make_env()
iter_test = env.iter_test()
np.__version__<import_modules>
model.eval()
prev_test_df = None
for(test_df, sample_prediction_df)in tqdm(iter_test):
test_df = test_df.merge(questions_df, left_on='content_id', right_on='question_id', how='left' ).drop(columns=['question_id'])
 if(prev_test_df is not None)&(psutil.virtual_memory().percent<90):
print(psutil.virtual_memory().percent)
 prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prev_test_df = prev_test_df[prev_test_df.content_type_id == False]
prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly', 'bundle_id', 'tags']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values,
r['bundle_id'].values,
np.vstack(list(r['tags'].values))))
for prev_user_id in prev_group.index:
prev_group_content = prev_group[prev_user_id][0]
prev_group_ac = prev_group[prev_user_id][1]
prev_group_bund_id = prev_group[prev_user_id][2]
prev_group_tags = prev_group[prev_user_id][3]
if prev_user_id in group.index:
group[prev_user_id] =(np.append(group[prev_user_id][0], prev_group_content),
np.append(group[prev_user_id][1], prev_group_ac))
else:
group[prev_user_id] =(prev_group_content, prev_group_ac)
 if len(group[prev_user_id][0])> MAX_SEQ:
new_group_content = group[prev_user_id][0][-MAX_SEQ:]
new_group_ac = group[prev_user_id][1][-MAX_SEQ:]
group[prev_user_id] =(new_group_content, new_group_ac)
 prev_test_df = test_df.copy()
test_df = test_df[test_df.content_type_id == False]
test_dataset = TestDataset(group, test_df, skills)
 test_dataloader = DataLoader(test_dataset, batch_size=1024, shuffle=False)
 outs = []
softmax = nn.Softmax(dim=1)
 for item in tqdm(test_dataloader):
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
bundle_x = item[2].to(device ).long()
tags_x = item[3].to(device ).long()
b_target = item[4].to(device ).long()
t_target = item[5].to(device ).long()
with torch.no_grad() :
pred_logits = model(x, target_id, bundle_x, tags_x, b_target, t_target)
 pred_logits = pred_logits.logits.cpu()
pred_pr = softmax(pred_logits ).numpy()
pred_pr = pred_pr[:, 1]
outs.extend(pred_pr)
 test_df['answered_correctly'] = outs
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )
<load_from_csv><EOS> 
print("Done !" )