Kaggle competition: Riiid Answer Correctness Prediction
def argmax(iterable):
return max(enumerate(iterable), key=lambda x: x[1])[0]<predict_on_test>
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl
def custom_pred(bc,lgb):
preds = list()
diff_index = [i for i, x in enumerate(y_pred_bc==y_pred_lgb)if not x]
for index,(i,j)in enumerate(zip(bc,lgb)) :
if index in diff_index:
preds.append(y_pred_bc[index])
 else:
preds.append(argmax(i+j)%2)
 return preds 
<predict_on_test>
import gc
import random
from tqdm.notebook import tqdm
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split
import seaborn as sns
import matplotlib.pyplot as plt
import joblib
import torch
import torch.nn as nn
import torch.nn.utils.rnn as rnn_utils
from torch.autograd import Variable
from torch.utils.data import Dataset, DataLoader
from pathlib import Path
import datatable as dt
preds = custom_pred(Proba_bc,Proba_lgb )<save_to_csv>
path = Path('/kaggle/input')
 assert path.exists()
submission = pd.DataFrame({
"PassengerId": ID,
"Survived": preds
})
 submission.to_csv('submission.csv', index=False )<import_modules>
%%time
data_types_dict = {
'content_type_id': 'bool',
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'
train_df = dt.fread(path/'riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()
import pandas as pd
import numpy as np<import_modules>
%%time
train_df = train_df[train_df.content_type_id == False]
train_df = train_df.sort_values(['timestamp'], ascending=True ).reset_index(drop = True )
import pandas as pd
import numpy as np<load_from_csv>
del train_df['timestamp']
del train_df['content_type_id']
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print("Train Shape: " + str(train.shape))
print("Test Shape: " + str(test.shape))<count_unique_values>
n_skill = train_df["content_id"].nunique()
print("number skills", n_skill )
unique_vals_per_col = train.T.apply(lambda x: x.nunique() , axis=1)
 print(unique_vals_per_col.head(5))<drop_column>
joblib.dump(n_skill, "skills.pkl.zip" )
train = train.drop(train[(train['GrLivArea']>4000)&(train['SalePrice']<300000)].index)
<prepare_x_and_y>
%%time
group = train_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(r['content_id'].values, r['answered_correctly'].values))
del train_df
y = train['SalePrice']
train.drop("SalePrice", axis = 1, inplace = True )<categorify>
joblib.dump(group, "group.pkl.zip" )
ntrain = train.shape[0]
all_data = pd.concat(( train, test)).reset_index(drop=True)
 print("all_data size is : {}".format(all_data.shape))
def ordinal_encode(df, col, order_list):
df[col] = df[col].astype('category', ordered=True, categories=order_list ).cat.codes
return df
def label_encode(df, col):
for c in col:
encoder = preprocessing.LabelEncoder()
df[c] = encoder.fit_transform(df[c].astype(str))
return df
def split_all_data(all_data, ntrain):
print(( 'Split all_data back to train and test: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())))
 train_df = all_data[:ntrain]
test_df = all_data[ntrain:]
return train_df, test_df

 print(( 'Ordinal Encoding: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())))
 order_list = ['Ex', 'Gd', 'TA', 'Fa', 'Po']
cols = ['KitchenQual', 'ExterQual', 'ExterCond', 'HeatingQC']
for col in cols:
all_data = ordinal_encode(all_data, col, order_list)
 order_list = ['Ex', 'Gd', 'TA', 'Fa', 'Po', 'NA']
cols = ['BsmtQual', 'BsmtCond']
for col in cols:
all_data = ordinal_encode(all_data, col, order_list)
 order_list = ['Gd', 'Av', 'Mn', 'No', 'NA']
cols = ['BsmtExposure', 'FireplaceQu', 'GarageQual', 'GarageCond']
for col in cols:
all_data = ordinal_encode(all_data, col, order_list)
 order_list = ['Typ', 'Min1', 'Min2', 'Mod', 'Maj1', 'Maj2', 'Sev', 'Sal']
cols = ['Functional']
for col in cols:
all_data = ordinal_encode(all_data, col, order_list)
 order_list = ['Fin', 'RFn', 'Unf', 'NA']
cols = ['GarageFinish']
for col in cols:
all_data = ordinal_encode(all_data, col, order_list)
 order_list = ['Ex', 'Gd', 'TA', 'Fa', 'NA']
cols = ['PoolQC']
for col in cols:
all_data = ordinal_encode(all_data, col, order_list)

print(( 'Label Encoding: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())))
 cols_to_label_encode = ['MSSubClass', 'MSZoning', 'Street', 'Alley', 'LotShape', 'LandContour', 'Utilities', 'LotConfig', 'LandSlope', 'Neighborhood', 'Condition1', 'Condition2',
'BldgType', 'HouseStyle', 'RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'Foundation', 'BsmtFinType1', 'BsmtFinType2', 'Heating', 'CentralAir',
'Electrical', 'GarageType', 'PavedDrive', 'Fence', 'MiscFeature', 'SaleType', 'SaleCondition']
all_data = label_encode(all_data, cols_to_label_encode)
 train, test = split_all_data(all_data, ntrain)
 print("Train Shape: " + str(train.shape))
print("Test Shape: " + str(test.shape)) 
<compute_test_metric>
MAX_SEQ = 240
ACCEPTED_USER_CONTENT_SIZE = 2
EMBED_SIZE = 256
BATCH_SIZE = 64+32
DROPOUT = 0.1
print(f"{MAX_SEQ} {ACCEPTED_USER_CONTENT_SIZE} {EMBED_SIZE} {BATCH_SIZE} {DROPOUT}" )
def score_transformer(y):
y = np.log(y)
 return y
y = score_transformer(y )<feature_engineering>
class SAKTDataset(Dataset):
def __init__(self, group, n_skill, max_seq=100):
super(SAKTDataset, self ).__init__()
self.samples, self.n_skill, self.max_seq = {}, n_skill, max_seq
self.user_ids = []
for i, user_id in enumerate(group.index):
if(i % 10000 == 0):
print(f'Processed {i} users')
 content_id, answered_correctly = group[user_id]
if len(content_id)>= ACCEPTED_USER_CONTENT_SIZE:
if len(content_id)> self.max_seq:
total_questions = len(content_id)
 last_pos = total_questions // self.max_seq
for seq in range(last_pos):
index = f"{user_id}_{seq}"
self.user_ids.append(index)
 start = seq * self.max_seq
end =(seq + 1)* self.max_seq
self.samples[index] =(content_id[start:end], answered_correctly[start:end])
 if len(content_id[end:])>= ACCEPTED_USER_CONTENT_SIZE:
index = f"{user_id}_{last_pos + 1}"
self.user_ids.append(index)
 self.samples[index] =(content_id[end:], answered_correctly[end:])
 else:
index = f'{user_id}'
self.user_ids.append(index)
 self.samples[index] =(content_id, answered_correctly)
 def __len__(self):
return len(self.user_ids)
 def __getitem__(self, index):
user_id = self.user_ids[index]
content_id, answered_correctly = self.samples[user_id]
seq_len = len(content_id)
 content_id_seq = np.zeros(self.max_seq, dtype=int)
 answered_correctly_seq = np.zeros(self.max_seq, dtype=int)
 if seq_len >= self.max_seq:
content_id_seq[:] = content_id[-self.max_seq:]
answered_correctly_seq[:] = answered_correctly[-self.max_seq:]
else:
content_id_seq[-seq_len:] = content_id
answered_correctly_seq[-seq_len:] = answered_correctly
target_id = content_id_seq[1:]
label = answered_correctly_seq[1:]
x = content_id_seq[:-1].copy()
x +=(answered_correctly_seq[:-1] == 1)* self.n_skill
return x, target_id, label
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data['fe.sum.GrLivArea_BsmtFinSF1_BsmtFinSF2'] = all_data['GrLivArea'] + all_data['BsmtFinSF1'] + all_data['BsmtFinSF2']
all_data['fe.sum.OverallQual_Overall_Cond'] = all_data['OverallQual'] + all_data['OverallCond']
all_data['fe.mult.OverallQual_Overall_Cond'] = all_data['OverallQual'] * all_data['OverallCond']
all_data['fe.sum.KitchenQual_ExterQual'] = all_data['KitchenQual'] + all_data['ExterQual']
all_data['fe.mult.OverallQual_Overall_Cond'] = all_data['OverallQual'] * all_data['OverallCond']
all_data['fe.ratio.1stFlrSF_2ndFlrSF'] = all_data['1stFlrSF'] / all_data['2ndFlrSF']
all_data['fe.ratio.BedroomAbvGr_GrLivArea'] = all_data['BedroomAbvGr'] / all_data['GrLivArea'] 
<split>
TEST_SIZE = 0.05
train, val = train_test_split(group, test_size = TEST_SIZE, random_state=42 )
train_features = list(all_data)
 train_features = [e for e in train_features if e not in('ExterQual', 'Condition2', 'GarageCond', 'Street', 'Alley', 'PoolArea', 'PoolQC', 'Utilities',
'GarageQual', 'MiscVal', 'MiscFeature')]
train, test = split_all_data(all_data, ntrain)
 train_features.remove('Id')
<define_variables>
train_dataset = SAKTDataset(train, n_skill, max_seq=MAX_SEQ)
 train_dataloader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=8)
 del train
nfolds=5
kf = KFold(n_splits=nfolds, shuffle=True, random_state=37)
 y_valid_pred = 0*y
y_valid_pred_cat = 0*y
fold_scores = [0] * nfolds
fold_scores_cat = [0] * nfolds
importances = pd.DataFrame()
oof_reg_preds = np.zeros(train.shape[0])
 sub_reg_preds = np.zeros(test.shape[0])
 sub_reg_preds_cat = np.zeros(test.shape[0])
 for fold_,(train_index, val_index)in enumerate(kf.split(train, y)) :
trn_x, trn_y = train[train_features].iloc[train_index], y.iloc[train_index]
val_x, val_y = train[train_features].iloc[val_index], y.iloc[val_index]
reg = LGBMRegressor(
 num_leaves=15,
max_depth=3,
min_child_weight=50,
learning_rate=0.04,
n_estimators=1000,
reg_alpha=0.01,
reg_lambda=5,
subsample=1,
colsample_bytree=0.21,
random_state=2 
)
 reg.fit(
 trn_x, trn_y,
eval_set=[(val_x, val_y)],
early_stopping_rounds=20,
verbose=100,
eval_metric='rmse' 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = train_features
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
importances = pd.concat([importances, imp_df], axis=0, sort=False)
 y_valid_pred.iloc[val_index] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 y_valid_pred[y_valid_pred < 0] = 0
fold_score = reg.best_score_['valid_0']['rmse']
fold_scores[fold_] = fold_score
_preds = reg.predict(test[train_features], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_reg_preds += _preds / nfolds
print("LightGBM CV RMSE: " + str(mean_squared_error(y, y_valid_pred)**.5))
print("LightGBM CV standard deviation: " + str(np.std(fold_scores)))
<save_to_csv>
val_dataset = SAKTDataset(val, n_skill, max_seq=MAX_SEQ)
 val_dataloader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=8)
 del val
sub_reg_preds = np.exp(sub_reg_preds)
 test.is_copy = False
test.loc[:,'SalePrice'] = sub_reg_preds
test[['Id', 'SalePrice']].to_csv("tutorial_sub.csv", float_format='%.8f', index=False)
<load_from_csv>
df = pd.read_csv('/kaggle/input/titanic/train.csv')
 df2 = pd.read_csv('/kaggle/input/titanic/test.csv')
 pd.options.display.max_columns = None
df.head()<load_from_csv>
class FFN(nn.Module):
def __init__(self, state_size = 200, forward_expansion = 1, bn_size = MAX_SEQ - 1, dropout=0.2):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, forward_expansion * state_size)
 self.relu = nn.ReLU()
self.bn = nn.BatchNorm1d(bn_size)
 self.lr2 = nn.Linear(forward_expansion * state_size, state_size)
 self.dropout = nn.Dropout(dropout)
 def forward(self, x):
x = self.relu(self.lr1(x))
x = self.bn(x)
 x = self.lr2(x)
 return self.dropout(x)
 class FFN0(nn.Module):
def __init__(self, state_size = 200, forward_expansion = 1, bn_size = MAX_SEQ - 1, dropout=0.2):
super(FFN0, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, forward_expansion * state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(forward_expansion * state_size, state_size)
 self.layer_normal = nn.LayerNorm(state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 x=self.layer_normal(x)
 return self.dropout(x )
df3 = pd.read_csv('/kaggle/input/titanic/gender_submission.csv')
 df3.head()<create_dataframe>
def future_mask(seq_length):
future_mask =(np.triu(np.ones([seq_length, seq_length]), k = 1)).astype('bool')
 return torch.from_numpy(future_mask)
 future_mask(5 )
percent_missing = df.isnull().sum() * 100 / len(df)
 missing_values = pd.DataFrame({'percent_missing': percent_missing})
 missing_values.sort_values(by ='percent_missing' , ascending=False )<create_dataframe>
class TransformerBlock(nn.Module):
def __init__(self, embed_dim, heads = 8, dropout = DROPOUT, forward_expansion = 1):
super(TransformerBlock, self ).__init__()
self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=heads, dropout=dropout)
 self.dropout = nn.Dropout(dropout)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim, forward_expansion = forward_expansion, dropout=dropout)
 self.ffn0 = FFN0(embed_dim, forward_expansion = forward_expansion, dropout=dropout)
 self.layer_normal_2 = nn.LayerNorm(embed_dim)
 def forward(self, value, key, query, att_mask):
att_output, att_weight = self.multi_att(value, key, query, attn_mask=att_mask)
 att_output = self.dropout(self.layer_normal(att_output + value))
att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x1 = self.ffn0(att_output)
 x = self.dropout(self.layer_normal_2(x + x1 + att_output))
return x.squeeze(-1), att_weight
class Encoder(nn.Module):
def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, num_layers=1, heads = 8):
super(Encoder, self ).__init__()
self.n_skill, self.embed_dim = n_skill, embed_dim
self.embedding = nn.Embedding(2 * n_skill + 1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq - 1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.layers = nn.ModuleList([TransformerBlock(embed_dim, forward_expansion = forward_expansion)for _ in range(num_layers)])
 self.dropout = nn.Dropout(dropout)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = self.dropout(x + pos_x)
 x = x.permute(1, 0, 2)
 e = self.e_embedding(question_ids)
 e = e.permute(1, 0, 2)
 for layer in self.layers:
att_mask = future_mask(e.size(0)).to(device)
 x, att_weight = layer(e, x, x, att_mask=att_mask)
 x = x.permute(1, 0, 2)
 x = x.permute(1, 0, 2)
 return x, att_weight
class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, enc_layers=1, heads = 8):
super(SAKTModel, self ).__init__()
self.encoder = Encoder(n_skill, max_seq, embed_dim, dropout, forward_expansion, num_layers=enc_layers)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
x, att_weight = self.encoder(x, question_ids)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
percent_missing = df2.isnull().sum() * 100 / len(df2)
 missing_values = pd.DataFrame({'percent_missing': percent_missing})
 missing_values.sort_values(by ='percent_missing' , ascending=False )<count_values>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )
df['Survived'].value_counts()<prepare_x_and_y>
def create_model() :
return SAKTModel(n_skill, max_seq=MAX_SEQ, embed_dim=EMBED_SIZE, forward_expansion=1, enc_layers=1, heads=4, dropout=0.1)
 model = create_model()
model
X = df.drop(['Cabin', 'Name', 'PassengerId', 'Survived', 'Ticket'], axis = 1)
 Y = df['Survived']
Test_Data = df2.drop(['Cabin', 'Name', 'PassengerId', 'Ticket'], axis = 1 )<categorify>
model(sample_batch[0], sample_batch[1])[0]
X = pd.get_dummies(X, prefix_sep='_')
 Test_Data = pd.get_dummies(Test_Data, prefix_sep='_')
 X.head()<prepare_output>
MODEL_PATH = '/kaggle/working/sakt_model.pt'
Test_Data['Embarked_0'] = 0
Test_Data = Test_Data[['Pclass', 'Age','SibSp','Parch','Fare','Sex_female', 'Sex_male', 'Embarked_C', 'Embarked_Q', 'Embarked_S']]
Test_Data.head()<categorify>
def do_train() :
optimizer = torch.optim.Adam(model.parameters() , lr=LR)
 criterion = nn.BCEWithLogitsLoss()
scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=LR,
steps_per_epoch=len(train_dataloader), epochs=EPOCHS)
 model.to(device)
 criterion.to(device)
 best_auc = 0.0
for epoch in range(EPOCHS):
train_epoch(model, train_dataloader, optimizer, criterion, scheduler, device)
 val_loss, avl_acc, val_auc = val_epoch(model, val_dataloader, criterion, device)
 print(f"epoch - {epoch + 1} val_loss - {val_loss:.3f} acc - {avl_acc:.3f} auc - {val_auc:.3f}")
 if best_auc < val_auc:
print(f'epoch - {epoch + 1} best model with val auc: {val_auc}')
 best_auc = val_auc
torch.save(model.state_dict() , MODEL_PATH )
Y = LabelEncoder().fit_transform(Y)
 Y<split>
LR = 2e-3
EPOCHS = 11
print(f'{LR}, {EPOCHS}')
 do_train()
X2 = StandardScaler().fit_transform(X)
 Test_Data = StandardScaler().fit_transform(Test_Data)
 X_Train, X_Test, Y_Train, Y_Test = train_test_split(X2, Y, test_size = 0.20, random_state = 101 )<compute_train_metric>
LR = 2e-4
EPOCHS = 3
print(f'{LR}, {EPOCHS}')
 do_train()
start = time.process_time()
trainedmodel = LogisticRegression().fit(X_Train,Y_Train)
 print(time.process_time() - start)
 predictions =trainedmodel.predict(X_Test)
 print(confusion_matrix(Y_Test,predictions))
print(classification_report(Y_Test,predictions))<compute_train_metric>
model = create_model()
model.load_state_dict(torch.load(MODEL_PATH))
model.to(device )
start = time.process_time()
trainedsvm = svm.LinearSVC().fit(X_Train, Y_Train)
 print(time.process_time() - start)
 predictionsvm = trainedsvm.predict(X_Test)
 print(confusion_matrix(Y_Test,predictionsvm))
print(classification_report(Y_Test,predictionsvm))<compute_train_metric>
env = riiideducation.make_env()
iter_test = env.iter_test()
<find_best_model_class><EOS> 
model.eval()
prev_test_df = None
for(test_df, sample_prediction_df)in tqdm(iter_test):
if(prev_test_df is not None)&(psutil.virtual_memory().percent<90):
print(psutil.virtual_memory().percent)
 prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prev_test_df = prev_test_df[prev_test_df.content_type_id == False]
prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
prev_group_content = prev_group[prev_user_id][0]
prev_group_answered_correctly = prev_group[prev_user_id][1]
if prev_user_id in group.index:
group[prev_user_id] =(np.append(group[prev_user_id][0], prev_group_content),
np.append(group[prev_user_id][1], prev_group_answered_correctly))
else:
group[prev_user_id] =(prev_group_content, prev_group_answered_correctly)
 if len(group[prev_user_id][0])> MAX_SEQ:
new_group_content = group[prev_user_id][0][-MAX_SEQ:]
new_group_answered_correctly = group[prev_user_id][1][-MAX_SEQ:]
group[prev_user_id] =(new_group_content, new_group_answered_correctly)
 prev_test_df = test_df.copy()
test_df = test_df[test_df.content_type_id == False]
test_dataset = TestDataset(group, test_df, n_skill, max_seq=MAX_SEQ)
 test_dataloader = DataLoader(test_dataset, batch_size=len(test_df), shuffle=False)
 item = next(iter(test_dataloader))
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, _ = model(x, target_id)
 output = torch.sigmoid(output)
 output = output[:, -1]
test_df['answered_correctly'] = output.cpu().numpy()
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )