Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<import_modules>
InteractiveShell.ast_node_interactivity = "all"
import pandas as pd
import numpy as np
import gc
from sklearn.metrics import roc_auc_score
from collections import defaultdict
from tqdm.notebook import tqdm
import lightgbm as lgb<define_variables>
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 df_full = [df_train, df_test]
train_pickle = '.. /input/pickle1/cv1_train.pickle'
valid_pickle = '.. /input/pickle1/cv1_valid.pickle'
question_file = '.. /input/riiid-test-answer-prediction/questions.csv'
debug = False
validaten_flg = False<categorify>
table1 = pd.pivot_table(df_train, values='Survived', index=['Pclass'])
 table1
train = pd.read_pickle(train_pickle)
 valid = pd.read_pickle(valid_pickle )<load_pretrained>
for dataset in df_full:
family_size = dataset.SibSp + dataset.Parch +1
dataset['FamilySize'] = family_size
table2 = pd.pivot_table(df_train, values = 'Survived', index= ['FamilySize'])
 table2
question_df = pd.read_pickle('.. /input/questionspickle/question.pickle' )<load_from_csv>
port_mode = df_train.Embarked.mode() [0]
df_train['Embarked'] = df_train['Embarked'].fillna(port_mode)
 fare_median = df_test.Fare.median()
df_test['Fare'] = df_test['Fare'].fillna(fare_median )
<prepare_x_and_y>
for dataset in df_full:
dataset['Sex'] = dataset['Sex'].map({'female': 1, 'male': 0} ).astype(int )
TARGET = 'answered_correctly'
FEATS = ['answered_correctly_avg_u','content_id', 'answered_correctly_sum_u', 'count_u', 'answered_correctly_avg_c', 'prior_question_had_explanation', 'prior_question_elapsed_time']
dro_cols = list(set(train.columns)- set(FEATS))
y_tr = train[TARGET]
y_va = valid[TARGET]
train.drop(dro_cols, axis=1, inplace=True)
 valid.drop(dro_cols, axis=1, inplace=True)
 _=gc.collect()<create_dataframe>
for dataset in df_full:
dataset['Embarked'] = dataset['Embarked'].map({'S':0 , 'C':1 , 'Q':2} ).astype(int )
lgb_train = lgb.Dataset(train[FEATS], y_tr)
 lgb_valid = lgb.Dataset(valid[FEATS], y_va)
 del train, y_tr,valid,y_va
_=gc.collect()<train_model>
for dataset in df_full:
dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.', expand=False )
model = lgb.train(
 {'objective': 'binary',
lgb_train,
valid_sets=[lgb_train, lgb_valid],
verbose_eval=10,
num_boost_round=10000,
early_stopping_rounds=10 
)
 _ = lgb.plot_importance(model )<data_type_conversions>
all_titles = df_test['Title'].append(df_train['Title'])
 pd.crosstab(all_titles,'count' )
def add_user_feats_without_update(df, answered_correctly_sum_u_dict, count_u_dict):
acsu = np.zeros(len(df), dtype=np.int32)
 cu = np.zeros(len(df), dtype=np.int32)
 for cnt,row in enumerate(df[['user_id']].values):
acsu[cnt] = answered_correctly_sum_u_dict[row[0]]
cu[cnt] = count_u_dict[row[0]]
user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu})
 user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']
df = pd.concat([df, user_feats_df], axis=1)
 return df<load_pretrained>
for dataset in df_full:
dataset['Title'] = dataset['Title'].replace(['Mlle','Ms'],'Miss')
 dataset['Title'] = dataset['Title'].replace(['Mme'], 'Mrs')
 dataset['Title'] = dataset['Title'].replace(['Capt','Col','Don','Jonkheer','Major','Sir','Rev','Dr'],'Raremale')
 dataset['Title'] = dataset['Title'].replace(['Countess','Dona','Lady'],'Rarefemale' )
content_df = pd.read_pickle('.. /input/pickle1/content.pickle' )<merge>
pd.pivot_table(df_train, index = df_train['Title'], values = 'Survived' )
env = riiideducation.make_env()
iter_test = env.iter_test()
set_predict = env.predict
for(test_df, sample_prediction_df)in iter_test:
previous_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df = add_user_feats_without_update(test_df, answered_correctly_sum_u_dict, count_u_dict)
 test_df = pd.merge(test_df, content_df, on='content_id', how="left")
 test_df = pd.merge(test_df, questions_df, left_on='content_id', right_on='question_id', how='left')
 test_df['prior_question_elapsed_time_mean'] = test_df.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 test_df[TARGET] = model.predict(test_df[FEATS])
 set_predict(test_df[['row_id', TARGET]] )<save_to_csv>
title_map = {"Master":1, "Miss":2, "Mr":3, "Mrs":4, "Rarefemale":5, "Raremale":6}
for dataset in df_full:
dataset['Title'] = dataset['Title'].map(title_map)
answered_correctly_sum_u_dict.to_csv('answered_correctly_sum_u_dict.csv' )<save_to_csv>
for dataset in df_full:
dataset.drop(['Name','SibSp','Parch','Ticket','Cabin','Fare'], axis= 1, inplace = True )
count_u_dict.to_csv('count_u_dict.csv' )<save_model>
for dataset in df_full:
new_df = dataset[['PassengerId','Pclass','Sex','Age','Embarked','FamilySize','Fareband','Title']]
filled = KNN(k=3 ).complete(new_df)
 filled = pd.DataFrame(filled, columns =['PassengerId','Pclass','Sex','Age','Embarked','FamilySize','Fareband','Title'])
 dataset['Age'] = filled['Age']
dataset.head()
dataset.isnull().sum()
model.save_model('modelfeats7.txt' )<import_modules>
for dataset in df_full:
dataset.drop("Age", axis= 1, inplace = True)
 df_train.head()
df_test.head()
import gc
import random
from tqdm.notebook import tqdm
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split
import seaborn as sns
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.nn.utils.rnn as rnn_utils
from torch.autograd import Variable
from torch.utils.data import Dataset, DataLoader
from pathlib import Path
import datatable as dt<define_variables>
X_train = df_train.drop(["Survived","PassengerId"], axis=1)
 Y_train = df_train["Survived"]
X_test = df_test.drop(["PassengerId"], axis=1 ).copy()
path = Path('/kaggle/input')
 assert path.exists()<load_from_csv>
logreg = LogisticRegression()
logreg.fit(X_train, Y_train)
 Y_pred = logreg.predict(X_test)
 acc_log = round(logreg.score(X_train, Y_train)* 100, 2)
 acc_log
%%time
data_types_dict = {
'content_type_id': 'bool',
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'
train_df = dt.fread(path/'riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()<sort_values>
svc = SVC()
svc.fit(X_train, Y_train)
 Y_pred = svc.predict(X_test)
 acc_svc = round(svc.score(X_train, Y_train)* 100, 2)
 acc_svc
%%time
train_df = train_df[train_df.content_type_id == False]
train_df = train_df.sort_values(['timestamp'], ascending=True ).reset_index(drop = True )<drop_column>
decision_tree = DecisionTreeClassifier()
decision_tree.fit(X_train, Y_train)
 Y_pred = decision_tree.predict(X_test)
 acc_decision_tree = round(decision_tree.score(X_train, Y_train)* 100, 2)
 acc_decision_tree
del train_df['timestamp']
del train_df['content_type_id']<count_unique_values>
knn = KNeighborsClassifier(n_neighbors = 3)
 knn.fit(X_train, Y_train)
 Y_pred = knn.predict(X_test)
 acc_knn = round(knn.score(X_train, Y_train)* 100, 2)
 acc_knn
n_skill = train_df["content_id"].nunique()
print("number skills", n_skill )<groupby>
gaussian = GaussianNB()
gaussian.fit(X_train, Y_train)
 Y_pred = gaussian.predict(X_test)
 acc_gaussian = round(gaussian.score(X_train, Y_train)* 100, 2)
 acc_gaussian
%%time
group = train_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(r['content_id'].values, r['answered_correctly'].values))
del train_df<define_variables>
perceptron = Perceptron()
perceptron.fit(X_train, Y_train)
 Y_pred = perceptron.predict(X_test)
 acc_perceptron = round(perceptron.score(X_train, Y_train)* 100, 2)
 acc_perceptron
MAX_SEQ = 200
ACCEPTED_USER_CONTENT_SIZE = 4
EMBED_SIZE = 128
BATCH_SIZE = 64
DROPOUT = 0.1<create_dataframe>
sgd = SGDClassifier()
sgd.fit(X_train, Y_train)
 Y_pred = sgd.predict(X_test)
 acc_sgd = round(sgd.score(X_train, Y_train)* 100, 2)
 acc_sgd
class SAKTDataset(Dataset):
def __init__(self, group, n_skill, max_seq=100):
super(SAKTDataset, self ).__init__()
self.samples, self.n_skill, self.max_seq = {}, n_skill, max_seq
self.user_ids = []
for i, user_id in enumerate(group.index):
if(i % 10000 == 0):
print(f'Processed {i} users')
 content_id, answered_correctly = group[user_id]
if len(content_id)>= ACCEPTED_USER_CONTENT_SIZE:
if len(content_id)> self.max_seq:
total_questions = len(content_id)
 last_pos = total_questions // self.max_seq
for seq in range(last_pos):
index = f"{user_id}_{seq}"
self.user_ids.append(index)
 start = seq * self.max_seq
end =(seq + 1)* self.max_seq
self.samples[index] =(content_id[start:end], answered_correctly[start:end])
 if len(content_id[end:])>= ACCEPTED_USER_CONTENT_SIZE:
index = f"{user_id}_{last_pos + 1}"
self.user_ids.append(index)
 self.samples[index] =(content_id[end:], answered_correctly[end:])
 else:
index = f'{user_id}'
self.user_ids.append(index)
 self.samples[index] =(content_id, answered_correctly)
 def __len__(self):
return len(self.user_ids)
 def __getitem__(self, index):
user_id = self.user_ids[index]
content_id, answered_correctly = self.samples[user_id]
seq_len = len(content_id)
 content_id_seq = np.zeros(self.max_seq, dtype=int)
 answered_correctly_seq = np.zeros(self.max_seq, dtype=int)
 if seq_len >= self.max_seq:
content_id_seq[:] = content_id[-self.max_seq:]
answered_correctly_seq[:] = answered_correctly[-self.max_seq:]
else:
content_id_seq[-seq_len:] = content_id
answered_correctly_seq[-seq_len:] = answered_correctly
target_id = content_id_seq[1:]
label = answered_correctly_seq[1:]
x = content_id_seq[:-1].copy()
x +=(answered_correctly_seq[:-1] == 1)* self.n_skill
return x, target_id, label<split>
X_train.info()
xgb = XGBClassifier()
xgb.fit(X_train,Y_train)
 y_pred = xgb.predict(X_test)
 acc_xgb = round(sgd.score(X_train, Y_train)* 100, 2)
 acc_xgb
TEST_SIZE = 0.1
train, val = train_test_split(group, test_size = TEST_SIZE )<create_dataframe>
random_forest = RandomForestClassifier(n_estimators=100)
 random_forest.fit(X_train, Y_train)
 Y_pred = random_forest.predict(X_test)
 Y_pred
random_forest.score(X_train, Y_train)
 acc_random_forest = round(random_forest.score(X_train, Y_train)* 100, 2)
 acc_random_forest
<create_dataframe><EOS> 
models = pd.DataFrame({
'Model': ['Support Vector Machines', 'KNN', 'Logistic Regression',
'Random Forest', 'Naive Bayes', 'Perceptron',
'Stochastic Gradient Decent', 'XGBoost',
'Decision Tree'],
'Score': [acc_svc, acc_knn, acc_log,
acc_random_forest, acc_gaussian, acc_perceptron,
acc_sgd, acc_xgb, acc_decision_tree]})
 models.sort_values(by='Score', ascending=False )