Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<data_type_conversions>
import numpy as np
import pandas as pd
import seaborn as sns
import os
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split, cross_val_score, StratifiedKFold, GridSearchCV
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import GradientBoostingClassifier
questions_df_dict = {
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
}
questions_df = questions_df.astype(questions_df_dict )<drop_column>
ROOT_DIR = '/kaggle/input/titanic'
train_df = pd.read_csv(os.path.join(ROOT_DIR, 'train.csv'))
test_df = pd.read_csv(os.path.join(ROOT_DIR, 'test.csv'))
test_id = test_df['PassengerId']
print('Train shape: ', train_df.shape)
 print('Test shape: ', test_df.shape )
questions_df.drop(columns=['tags'], inplace=True )<data_type_conversions>
def show_null_values(df):
null_df = pd.DataFrame(
 {'Column': [col for col in df.columns if df[col].isna().sum() ],
'Ratio': [df[col].isna().sum() /df.shape[0] for col in df.columns if df[col].isna().sum() ]}, 
)
 return null_df.sort_values(by=['Ratio'], ascending=False )
questions_df['part_bundle_id']=questions_df['part']*100000+questions_df['bundle_id']
questions_df.part_bundle_id=questions_df.part_bundle_id.astype('int32')
<load_from_csv>
show_null_values(train_df )
<rename_columns>
show_null_values(test_df )
questions_df.rename(columns={'question_id':'content_id'}, inplace=True )<merge>
train_df.drop(columns=['Cabin'], inplace=True)
 test_df.drop(columns=['Cabin'], inplace=True )
questions_df = pd.merge(questions_df, content_explation_agg, on='content_id', how='left',right_index=True)
<drop_column>
train_size = train_df.shape[0]
del content_explation_agg<data_type_conversions>
y_train = train_df['Survived']
all_data = pd.concat([train_df.drop(columns=['Survived']), test_df] )
questions_df['content_correctness'] = questions_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 questions_df.content_correctness=questions_df.content_correctness.astype('float16')
 questions_df['content_correctness_std'] = questions_df['content_id'].map(content_agg['var'])
 questions_df.content_correctness_std=questions_df.content_correctness_std.astype('float16')
 questions_df['content_uncorrect_count'] = questions_df['content_id'].map(content_agg['count']-content_agg['sum'] ).astype('int32')
 questions_df['content_correct_count'] = questions_df['content_id'].map(content_agg['sum'] ).astype('int32' )<data_type_conversions>
all_data['Age'].fillna(train_df['Age'].median() , inplace=True )
questions_df['content_elapsed_time_mean'] = questions_df['content_id'].map(content_elapsed_time_agg['mean'])
 questions_df.content_elapsed_time_mean=questions_df.content_elapsed_time_mean.astype('float16')
 questions_df['content_had_explanation_mean'] = questions_df['content_id'].map(content_had_explanation_agg['mean'])
 questions_df.content_had_explanation_mean=questions_df.content_had_explanation_mean.astype('float16' )<drop_column>
all_data['Embarked'].fillna(train_df['Embarked'].mode() [0], inplace=True )
del content_elapsed_time_agg
del content_had_explanation_agg
gc.collect()<categorify>
all_data['Fare'].fillna(train_df['Fare'].mean() , inplace=True )
part_agg = questions_df.groupby('part')['content_correctness'].agg(['mean', 'var'])
 questions_df['part_correctness_mean'] = questions_df['part'].map(part_agg['mean'])
 questions_df['part_correctness_std'] = questions_df['part'].map(part_agg['var'])
 questions_df.part_correctness_mean=questions_df.part_correctness_mean.astype('float16')
 questions_df.part_correctness_std=questions_df.part_correctness_std.astype('float16' )<data_type_conversions>
show_null_values(all_data )
part_agg = questions_df.groupby('part')['content_uncorrect_count'].agg(['sum'])
 questions_df['part_uncor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32')
 part_agg = questions_df.groupby('part')['content_correct_count'].agg(['sum'])
 questions_df['part_cor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32' )<categorify>
all_data.drop(columns=['Name', 'Ticket'], inplace=True)
 all_data.drop(columns=['PassengerId'], inplace=True )
bundle_agg = questions_df.groupby('bundle_id')['content_correctness'].agg(['mean'])
 questions_df['bundle_correctness_mean'] = questions_df['bundle_id'].map(bundle_agg['mean'])
 questions_df.bundle_correctness_mean=questions_df.bundle_correctness_mean.astype('float16')
<data_type_conversions>
for col in ['Sex', 'Embarked', 'Pclass']:
dummies = pd.get_dummies(all_data[col], prefix=col)
 all_data = pd.concat([all_data, dummies], axis=1)
 all_data.drop(columns=col, inplace=True )
<drop_column>
X_train = all_data[:train_size]
X_test = all_data[train_size:]
del content_agg
del bundle_agg
del part_agg
gc.collect()<define_variables>
scaled_features = ['Age', 'Fare']
scaler = StandardScaler()
X_train_scaled = X_train.copy()
X_test_scaled = X_test.copy()
X_train_scaled[scaled_features] = scaler.fit_transform(X_train_scaled[scaled_features])
 X_test_scaled[scaled_features] = scaler.transform(X_test_scaled[scaled_features] )
features_dict = {
'timestamp':'float16',
'user_interaction_count':'int16',
'user_interaction_timestamp_mean':'float32',
'lagtime':'float32',
'lagtime2':'float32',
'lagtime3':'float32',
'content_id':'int16',
'task_container_id':'int16',
'user_lecture_sum':'int16',
'user_lecture_lv':'float16',
'prior_question_elapsed_time':'float32',
'delta_prior_question_elapsed_time':'int32',
'user_correctness':'float16',
'user_uncorrect_count':'int16',
'user_correct_count':'int16',
'content_correctness_std':'float16',
'content_correct_count':'int32',
'content_uncorrect_count':'int32',
'content_elapsed_time_mean':'float16',
'content_had_explanation_mean':'float16',
'content_explation_false_mean':'float16',
'content_explation_true_mean':'float16',
'task_container_correctness':'float16',
'task_container_std':'float16',
'task_container_cor_count':'int32',
'task_container_uncor_count':'int32',
'attempt_no':'int8',
'part':'int8',
'part_correctness_mean':'float16',
'part_correctness_std':'float16',
'part_uncor_count':'int32',
'part_cor_count':'int32',
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
'part_bundle_id':'int32',
'content_sub_bundle':'int8',
'prior_question_had_explanation':'int8',
'explanation_mean':'float16',
'explanation_false_count':'int16',
'explanation_true_count':'int16',
}
categorical_columns= [
'content_id',
'task_container_id',
'part',
'tags0',
'tags1',
'tags2',
'tags3',
'tags4',
'tags5',
'part_bundle_id',
'content_sub_bundle',
'prior_question_had_explanation',
]
features=list(features_dict.keys())
<init_hyperparams>
logistic_regr = LogisticRegression(max_iter=500)
 param_grid = {'C': [0.01, 0.05, 0.1, 0.5], 'penalty': ['l1', 'l2']}
logistic_gcv = GridSearchCV(logistic_regr, param_grid=param_grid, scoring='accuracy', n_jobs=-1, cv=5, return_train_score=True)
 logistic_gcv.fit(X_train_scaled, y_train )
flag_lgbm=True
clfs = list()
params = {
'num_leaves': 200,
'max_bin':450,
'feature_fraction': 0.52,
'bagging_fraction': 0.52,
'objective': 'binary',
'learning_rate': 0.05,
"boosting_type": "gbdt",
"metric": 'auc',
}
trains=list()
valids=list()
num=1
for i in range(0,num):
train_df_clf=train_df[1200*10000:2*1400*10000]
print('sample end')
 del train_df
users=train_df_clf['user_id'].drop_duplicates()
users=users.sample(frac=0.08)
 users_df=pd.DataFrame()
users_df['user_id']=users.values
valid_df_newuser = pd.merge(train_df_clf, users_df, on=['user_id'], how='inner',right_index=True)
 del users_df
del users
gc.collect()
train_df_clf.drop(valid_df_newuser.index, inplace=True)
 print('pd.merge(train_df_clf, questions_df)')
 train_df_clf = pd.merge(train_df_clf, questions_df, on='content_id', how='left',right_index=True)
 valid_df_newuser = pd.merge(valid_df_newuser, questions_df, on='content_id', how='left',right_index=True)
 print('valid_df')
 valid_df=train_df_clf.sample(frac=0.1)
 train_df_clf.drop(valid_df.index, inplace=True)
 valid_df = valid_df.append(valid_df_newuser)
 del valid_df_newuser
gc.collect()
trains.append(train_df_clf)
 valids.append(valid_df)
 print('train_df_clf length：',len(train_df_clf))
print('valid_df length：',len(valid_df)) 
<drop_column>
print('Logistic best params: ', logistic_gcv.best_params_)
 print('Logistic Regression best score: ', logistic_gcv.best_score_)
 print(logistic_gcv.cv_results_['mean_train_score'])
 print(logistic_gcv.cv_results_['mean_test_score'] )
del train_df_clf
del valid_df
gc.collect()<prepare_x_and_y>
dt = DecisionTreeClassifier(random_state=2)
 dt_params = {'max_depth': [3, 5, 6],
}
dt_gcv = GridSearchCV(dt, param_grid=dt_params, scoring='accuracy', n_jobs=-1, cv=5, return_train_score=True)
 dt_gcv.fit(X_train, y_train )
for i in range(0,num):
X_train_np = trains[i][features].values.astype(np.float32)
 X_valid_np = valids[i][features].values.astype(np.float32)
 tr_data = lgb.Dataset(X_train_np, label=trains[i][target], feature_name=list(features))
va_data = lgb.Dataset(X_valid_np, label=valids[i][target], feature_name=list(features))
del trains
del valids
del X_train_np
del X_valid_np
gc.collect()
model1 = joblib.load('.. /input/5-start-777/lgbm775_model.pkl')
 model2 = joblib.load('.. /input/777-val-lgbm/lgbm775_model.pkl')
 model3 = joblib.load('.. /input/3-start-1200/lgbm775_model.pkl')
 clfs.append(model1)
 clfs.append(model2)
 clfs.append(model3)
 del model1
del model2
del model3
del tr_data
del va_data
gc.collect() 
<choose_model_class>
print('Decision Tree best params: ', dt_gcv.best_params_)
 print('Decision Tree best score: ', dt_gcv.best_score_)
 print(dt_gcv.cv_results_['mean_train_score'])
 print(dt_gcv.cv_results_['mean_test_score'])
 print(dt_gcv.cv_results_['std_train_score'])
 print(dt_gcv.cv_results_['std_test_score'] )
MAX_SEQ = 280
ACCEPTED_USER_CONTENT_SIZE = 7
EMBED_SIZE = 128
BATCH_SIZE = 64
DROPOUT = 0.1
class FFN(nn.Module):
def __init__(self, state_size = 200, forward_expansion = 1, bn_size=MAX_SEQ - 1, dropout=0.2):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, forward_expansion * state_size)
 self.relu = nn.ReLU()
self.bn = nn.BatchNorm1d(bn_size)
 self.lr2 = nn.Linear(forward_expansion * state_size, state_size)
 self.dropout = nn.Dropout(dropout)
 def forward(self, x):
x = self.relu(self.lr1(x))
x = self.bn(x)
 x = self.lr2(x)
 return self.dropout(x)
 FFN()<categorify>
svm = SVC()
svm_params = {'C': [0.01, 0.05, 0.1, 1]}
svm_gcv = GridSearchCV(svm, param_grid=svm_params, scoring='accuracy', n_jobs=-1, cv=5, return_train_score=True)
 svm_gcv.fit(X_train_scaled, y_train )
def future_mask(seq_length):
future_mask =(np.triu(np.ones([seq_length, seq_length]), k = 1)).astype('bool')
 return torch.from_numpy(future_mask)
 future_mask(5 )<choose_model_class>
print('SVM best params: ', svm_gcv.best_params_)
 print('SVM best score: ', svm_gcv.best_score_)
 print(svm_gcv.cv_results_['mean_train_score'])
 print(svm_gcv.cv_results_['mean_test_score'] )
class TransformerBlock(nn.Module):
def __init__(self, embed_dim, heads = 8, dropout = DROPOUT, forward_expansion = 1):
super(TransformerBlock, self ).__init__()
self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=heads, dropout=dropout)
 self.dropout = nn.Dropout(dropout)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim, forward_expansion = forward_expansion, dropout=dropout)
 self.layer_normal_2 = nn.LayerNorm(embed_dim)
 def forward(self, value, key, query, att_mask):
att_output, att_weight = self.multi_att(value, key, query, attn_mask=att_mask)
 att_output = self.dropout(self.layer_normal(att_output + value))
att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.dropout(self.layer_normal_2(x + att_output))
return x.squeeze(-1), att_weight
class Encoder(nn.Module):
def __init__(self, n_skill, max_seq=280, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, num_layers=1, heads = 8):
super(Encoder, self ).__init__()
self.n_skill, self.embed_dim = n_skill, embed_dim
self.embedding = nn.Embedding(2 * n_skill + 1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.layers = nn.ModuleList([TransformerBlock(embed_dim, forward_expansion = forward_expansion)for _ in range(num_layers)])
 self.dropout = nn.Dropout(dropout)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = self.dropout(x + pos_x)
 x = x.permute(1, 0, 2)
 e = self.e_embedding(question_ids)
 e = e.permute(1, 0, 2)
 for layer in self.layers:
att_mask = future_mask(e.size(0)).to(device)
 x, att_weight = layer(e, x, x, att_mask=att_mask)
 x = x.permute(1, 0, 2)
 x = x.permute(1, 0, 2)
 return x, att_weight
class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=280, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, enc_layers=1, heads = 8):
super(SAKTModel, self ).__init__()
self.encoder = Encoder(n_skill, max_seq, embed_dim, dropout, forward_expansion, num_layers=enc_layers)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
x, att_weight = self.encoder(x, question_ids)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
skills = joblib.load("/kaggle/input/riiid-sakt-model-dataset-public/skills.pkl.zip")
 n_skill = len(skills)
 group = joblib.load(".. /input/riiid-sakt-model-dataset-public/group.pkl.zip")
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 def create_model() :
return SAKTModel(n_skill, max_seq=MAX_SEQ, embed_dim=EMBED_SIZE, forward_expansion=1, enc_layers=1, heads=8, dropout=0.1)
 nn_model = create_model()
nn_model
try:
nn_model.load_state_dict(torch.load(".. /input/real-777/sakt.pth"))
except:
nn_model.load_state_dict(torch.load(".. /input/real-777/sakt.pth", map_location='cpu'))
nn_model.to(device)
 nn_model.eval()<data_type_conversions>
gb = GradientBoostingClassifier()
gb_params = {'n_estimators': [30, 50, 60, 100]}
gb_gcv = GridSearchCV(gb, param_grid=gb_params, scoring='accuracy', n_jobs=-1, cv=5, return_train_score=True)
 gb_gcv.fit(X_train_scaled, y_train )
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int)) 
<data_type_conversions>
print('Gradient Boosting best params: ', gb_gcv.best_params_)
 print('Gradient Boosting best score: ', gb_gcv.best_score_)
 print(gb_gcv.cv_results_['mean_train_score'])
 print(gb_gcv.cv_results_['mean_test_score'] )
del user_agg
gc.collect()
task_container_sum_dict = task_container_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
task_container_count_dict = task_container_agg['count'].astype('int32' ).to_dict(defaultdict(int))
task_container_std_dict = task_container_agg['var'].astype('float16' ).to_dict(defaultdict(int))
explanation_sum_dict = explanation_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
explanation_count_dict = explanation_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del task_container_agg
del explanation_agg
gc.collect()<data_type_conversions>
gb = GradientBoostingClassifier(n_estimators=50)
 gb.fit(X_train_scaled, y_train)
 predict = gb.predict(X_test_scaled )
user_lecture_sum_dict = user_lecture_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_lecture_count_dict = user_lecture_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del user_lecture_agg
gc.collect()<categorify>
output = pd.DataFrame({'PassengerId': test_id, 'Survived': predict})
 output.to_csv('output.csv', index=False )