Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<choose_model_class>
%matplotlib inline
warnings.filterwarnings("ignore" )
class TransformerBlock(nn.Module):
def __init__(self, embed_dim, heads = 8, dropout = DROPOUT, forward_expansion = 1):
super(TransformerBlock, self ).__init__()
self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=heads, dropout=dropout)
 self.dropout = nn.Dropout(dropout)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim, forward_expansion = forward_expansion, dropout=dropout)
 self.layer_normal_2 = nn.LayerNorm(embed_dim)
 def forward(self, value, key, query, att_mask):
att_output, att_weight = self.multi_att(value, key, query, attn_mask=att_mask)
 att_output = self.dropout(self.layer_normal(att_output + value))
att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.dropout(self.layer_normal_2(x + att_output))
return x.squeeze(-1), att_weight
class Encoder(nn.Module):
def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, num_layers=1, heads = 8):
super(Encoder, self ).__init__()
self.n_skill, self.embed_dim = n_skill, embed_dim
self.embedding = nn.Embedding(2 * n_skill + 1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq - 1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.layers = nn.ModuleList([TransformerBlock(embed_dim, forward_expansion = forward_expansion)for _ in range(num_layers)])
 self.dropout = nn.Dropout(dropout)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = self.dropout(x + pos_x)
 x = x.permute(1, 0, 2)
 e = self.e_embedding(question_ids)
 e = e.permute(1, 0, 2)
 for layer in self.layers:
att_mask = future_mask(e.size(0)).to(device)
 x, att_weight = layer(e, x, x, att_mask=att_mask)
 x = x.permute(1, 0, 2)
 x = x.permute(1, 0, 2)
 return x, att_weight
class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, enc_layers=1, heads = 8):
super(SAKTModel, self ).__init__()
self.encoder = Encoder(n_skill, max_seq, embed_dim, dropout, forward_expansion, num_layers=enc_layers)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
x, att_weight = self.encoder(x, question_ids)
 x = self.pred(x)
 return x.squeeze(-1), att_weight<set_options>
train = pd.read_csv("/kaggle/input/titanic/train.csv")
 df_test = pd.read_csv("/kaggle/input/titanic/test.csv")
 test = df_test.copy()
gender_submission = pd.read_csv("/kaggle/input/titanic/gender_submission.csv" )
device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )<choose_model_class>
train.isnull().sum().sort_values(ascending = False )
def create_model() :
return SAKTModel(n_skill, max_seq=MAX_SEQ, embed_dim=EMBED_SIZE, forward_expansion=1, enc_layers=1, heads=8, dropout=0.1)
 model = create_model()
model<train_model>
test.isnull().sum().sort_values(ascending = False )
model(sample_batch[0], sample_batch[1])[0]<define_variables>
train.Pclass.isnull().sum()
LR = 2e-3
EPOCHS = 10
MODEL_PATH = '/kaggle/working/sakt.pth'<choose_model_class>
train.Name.value_counts()
def do_train() :
optimizer = torch.optim.Adam(model.parameters() , lr=LR)
 criterion = nn.BCEWithLogitsLoss()
scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=LR,
steps_per_epoch=len(train_dataloader), epochs=EPOCHS)
 model.to(device)
 criterion.to(device)
 best_auc = 0.0
for epoch in range(EPOCHS):
train_epoch(model, train_dataloader, optimizer, criterion, scheduler, device)
 val_loss, avl_acc, val_auc = val_epoch(model, val_dataloader, criterion, device)
 print(f"epoch - {epoch + 1} val_loss - {val_loss:.3f} acc - {avl_acc:.3f} auc - {val_auc:.3f}")
 if best_auc < val_auc:
print(f'epoch - {epoch + 1} best model with val auc: {val_auc}')
 best_auc = val_auc
torch.save(model.state_dict() , MODEL_PATH )<train_model>
train.drop(columns = ["Name","PassengerId"], axis = 1, inplace = True)
 test.drop(columns = ["Name","PassengerId"], axis = 1, inplace = True )
do_train()<train_model>
train.Sex.isnull().sum()
LR = 2e-4
EPOCHS = 3
do_train()<load_pretrained>
train.Age.isnull().sum()
model = create_model()
model.load_state_dict(torch.load(MODEL_PATH))
model.to(device )<split>
train.Ticket.value_counts()
env = riiideducation.make_env()
iter_test = env.iter_test()<feature_engineering>
print("There are {} unique Ticket values.".format(len(train.Ticket.unique())) )
model.eval()
prev_test_df = None
for(test_df, sample_prediction_df)in tqdm(iter_test):
if(prev_test_df is not None)&(psutil.virtual_memory().percent<90):
print(psutil.virtual_memory().percent)
 prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prev_test_df = prev_test_df[prev_test_df.content_type_id == False]
prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
prev_group_content = prev_group[prev_user_id][0]
prev_group_answered_correctly = prev_group[prev_user_id][1]
if prev_user_id in group.index:
group[prev_user_id] =(np.append(group[prev_user_id][0], prev_group_content),
np.append(group[prev_user_id][1], prev_group_answered_correctly))
else:
group[prev_user_id] =(prev_group_content, prev_group_answered_correctly)
 if len(group[prev_user_id][0])> MAX_SEQ:
new_group_content = group[prev_user_id][0][-MAX_SEQ:]
new_group_answered_correctly = group[prev_user_id][1][-MAX_SEQ:]
group[prev_user_id] =(new_group_content, new_group_answered_correctly)
 prev_test_df = test_df.copy()
test_df = test_df[test_df.content_type_id == False]
test_dataset = TestDataset(group, test_df, n_skill, max_seq=MAX_SEQ)
 test_dataloader = DataLoader(test_dataset, batch_size=len(test_df), shuffle=False)
 item = next(iter(test_dataloader))
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, _ = model(x, target_id)
 output = torch.sigmoid(output)
 output = output[:, -1]
test_df['answered_correctly'] = output.cpu().numpy()
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<create_dataframe>
train.drop("Ticket", axis = 1, inplace = True)
 test.drop("Ticket", axis = 1, inplace = True )
test_dataset = TestDataset(group, test_df, n_skill, max_seq=MAX_SEQ )<load_pretrained>
train.drop("Fare", axis = 1, inplace = True)
 test.drop("Fare", axis = 1, inplace = True )
group.to_pickle('/kaggle/working/group.pkl' )<install_modules>
train.Cabin.value_counts()
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1<set_options>
train.drop("Cabin", axis = 1, inplace = True)
 test.drop("Cabin", axis =1, inplace = True )
_ = np.seterr(divide='ignore', invalid='ignore' )<define_variables>
train.Embarked.value_counts()
data_types_dict = {
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id':'int8',
'task_container_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'<load_from_csv>
train["Embarked"].isnull().sum()
train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()<load_from_csv>
print(len(train))
train = train.dropna(subset=['Embarked'])
 print(len(train))
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )<categorify>
train['Age'] = train.groupby("Pclass")['Age'].transform(lambda x: x.fillna(x.median()))
test['Age'] = test.groupby("Pclass")['Age'].transform(lambda x: x.fillna(x.median()))
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])
 part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]<merge>
print(train["Age"].isnull().sum())
 print(test["Age"].isnull().sum() )
train_lectures = train_df[train_df.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left' )<groupby>
train = pd.get_dummies(data = train, columns = ["Sex", "Embarked", "Pclass"])
 test = pd.get_dummies(data = test, columns = ["Sex", "Embarked", "Pclass"] )
user_lecture_stats_part = train_lectures.groupby('user_id',as_index = False)[part_lectures_columns + types_of_lectures_columns].sum()<data_type_conversions>
X_train = train.drop("Survived", axis = 1)
 y_train = train["Survived"]
lecturedata_types_dict = {
'user_id': 'int32',
'part_1': 'int8',
'part_2': 'int8',
'part_3': 'int8',
'part_4': 'int8',
'part_5': 'int8',
'part_6': 'int8',
'part_7': 'int8',
'type_of_concept': 'int8',
'type_of_intention': 'int8',
'type_of_solving_question': 'int8',
'type_of_starter': 'int8'
}
user_lecture_stats_part = user_lecture_stats_part.astype(lecturedata_types_dict )<data_type_conversions>
def fit_model(algo, X_train, y_train, cv):
model = algo.fit(X_train, y_train)
 y_pred = algo.predict(X_train)
 accuracy = round(accuracy_score(y_train, y_pred)* 100 , 2)
 y_pred_cv = cross_val_predict(algo, X_train, y_train, cv = cv)
 accuracy_cv = round(accuracy_score(y_train, y_pred_cv)* 100 , 2)
 return y_pred_cv, accuracy, accuracy_cv
for column in user_lecture_stats_part.columns:
if(column !='user_id'):
user_lecture_stats_part[column] =(user_lecture_stats_part[column] > 0 ).astype('int8' )<filter>
y_pred_cv_lr, accuracy_lr, accuracy_cv_lr = fit_model(LogisticRegression(random_state = 3),
X_train, y_train, 10)
 print("Accuracy : ",accuracy_lr)
 print("Accuracy CV :",accuracy_cv_lr )
train_lectures[train_lectures.user_id==5382]<filter>
rf = RandomForestClassifier(n_estimators = 100, random_state = 3)
 rf.fit(X_train, y_train)
 y_train_pred = rf.predict(X_train)
 print('Confusion Matrix : ','
', confusion_matrix(y_train, y_train_pred))
print()
print("Accuracy : ", round(accuracy_score(y_train, y_train_pred)* 100, 2))
user_lecture_stats_part[user_lecture_stats_part.user_id==5382]<drop_column>
rfc = RandomForestClassifier(random_state=3)
 params = {'n_estimators' : sp_randint(50,200),
'max_depth' : sp_randint(2,100),
'max_depth' : sp_randint(2,100),
'min_samples_split' : sp_randint(2,100),
'min_samples_leaf' : sp_randint(1,200),
'criterion' : ['gini', 'entropy']}
rsearch_rfc = RandomizedSearchCV(rfc, param_distributions = params, n_iter = 100, cv = 3, scoring = 'roc_auc', n_jobs = -1,\
return_train_score = True, random_state = 3)
 rsearch_rfc.fit(X_train,y_train )
del(train_lectures )<data_type_conversions>
rsearch_rfc.best_params_
cum = train_df.groupby('user_id')['content_type_id'].agg(['cumsum', 'cumcount'])
 train_df['user_lecture_cumsum'] = cum['cumsum']
train_df['user_lecture_lv'] = cum['cumsum'] / cum['cumcount']
train_df.user_lecture_lv=train_df.user_lecture_lv.astype('float16')
 train_df.user_lecture_cumsum=train_df.user_lecture_cumsum.astype('int8')
 user_lecture_agg = train_df.groupby('user_id')['content_type_id'].agg(['sum', 'count'] )<data_type_conversions>
rfc = RandomForestClassifier(**rsearch_rfc.best_params_, random_state = 3)
 rfc.fit(X_train, y_train)
 y_train_pred = rfc.predict(X_train)
 print('Confusion Matrix : ','
', confusion_matrix(y_train, y_train_pred))
print()
print("Accuracy : ", round(accuracy_score(y_train, y_train_pred)* 100, 2))
train_df['prior_question_had_explanation'].fillna(False, inplace=True)
 train_df = train_df.astype(data_types_dict)
 train_df = train_df[train_df[target] != -1].reset_index(drop=True)
 content_explation_agg=train_df[["content_id","prior_question_had_explanation",target]].groupby(["content_id","prior_question_had_explanation"])[target].agg(['mean'])
 content_explation_agg=content_explation_agg.unstack()
content_explation_agg=content_explation_agg.reset_index()
content_explation_agg.columns = ['content_id', 'content_explation_false_mean','content_explation_true_mean']
content_explation_agg.content_id=content_explation_agg.content_id.astype('int16')
 content_explation_agg.content_explation_false_mean=content_explation_agg.content_explation_false_mean.astype('float16')
 content_explation_agg.content_explation_true_mean=content_explation_agg.content_explation_true_mean.astype('float16')
 prior_question_elapsed_time_mean=train_df['prior_question_elapsed_time'].mean()
train_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True )<groupby>
imp = pd.DataFrame(rfc.feature_importances_, index = X_train.columns, columns = ['imp'])
 imp = imp.sort_values(by ='imp', ascending = False)
 imp
max_timestamp_u = train_df[['user_id','timestamp']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u.columns = ['user_id', 'max_time_stamp']<data_type_conversions>
y_test_pred = rfc.predict(test )
train_df['lagtime'] = train_df.groupby('user_id')['timestamp'].shift()
train_df['lagtime']=train_df['timestamp']-train_df['lagtime']
lagtime_mean=train_df['lagtime'].mean()
train_df['lagtime'].fillna(lagtime_mean, inplace=True)
 train_df.lagtime=train_df.lagtime.astype('int32')
<categorify>
submission = pd.DataFrame()
submission['PassengerId'] = df_test['PassengerId']
submission['Survived'] = y_test_pred
submission.head()
lagtime_agg = train_df.groupby('user_id')['lagtime'].agg(['mean'])
 train_df['lagtime_mean'] = train_df['user_id'].map(lagtime_agg['mean'])
 train_df.lagtime_mean=train_df.lagtime_mean.astype('int32' )<groupby>
submission.to_csv('rf_submission.csv', index=False)
 print('Submission CSV is ready!' )
train_df[['user_id','prior_question_elapsed_time']].groupby(['user_id'] ).head()<groupby>
submissions_check = pd.read_csv("rf_submission.csv")
 submissions_check.head()