Kaggle competition: Titanic - Machine Learning from Disaster
import pandas as pd
import numpy as np
import re
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score
import torch
from torch import nn
from torch import optim
import torch.nn.functional as F
from torch.utils.data import TensorDataset, DataLoader
import spacy
from gensim.models import KeyedVectors
from nltk.stem import PorterStemmer, SnowballStemmer
from nltk.stem.lancaster import LancasterStemmer
import time
import gc
from tqdm import tqdm<load_from_csv>
df = pd.read_csv("/kaggle/input/titanic/train.csv", index_col="PassengerId")
 rows, cols = df.shape
print(f"Original DataFrame has {rows} rows and {cols} columns")
 df.head()
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 labels = np.array(train_df.target, dtype=int )<string_transform>
label_encoder = LabelEncoder()
df["Sex"] = label_encoder.fit_transform(df["Sex"])
 df["Family_Size"] = df["SibSp"] + df["Parch"] + 1
df["Fare_Per_Person"] = df["Fare"] / df["Family_Size"]
df.tail()
def n_upper(sentence):
return len(re.findall(r'[A-Z]',sentence))
def n_unique_words(sentence):
return len(set(sentence.split()))
def n_question_mark(sentence):
return len(re.findall(r'[?]',sentence))
def n_exclamation_mark(sentence):
return len(re.findall(r'[!]',sentence))
def n_asterisk(sentence):
return len(re.findall(r'[*]',sentence))
def n_parentheses(sentence):
return len(re.findall(r'[() ]',sentence))
def n_brackets(sentence):
return len(re.findall(r'[\[\]]',sentence))
def n_braces(sentence):
return len(re.findall(r'[{}]',sentence))
def n_quotes(sentence):
return len(re.findall(r'["]',sentence))
def n_ampersand(sentence):
return len(re.findall(r'[&]',sentence))
def n_dash(sentence):
return len(re.findall(r'[-]',sentence))
n_stats = 11
def get_stat(questions_list):

 stat_feat = np.zeros(( len(questions_list), n_stats), dtype=int)
 for i,question in tqdm(enumerate(questions_list)) :
stat_feat[i,0] = n_upper(question)
 stat_feat[i,1] = n_unique_words(question)
 stat_feat[i,2] = n_question_mark(question)
 stat_feat[i,3] = n_exclamation_mark(question)
 stat_feat[i,4] = n_asterisk(question)
 stat_feat[i,5] = n_parentheses(question)
 stat_feat[i,6] = n_brackets(question)
 stat_feat[i,7] = n_braces(question)
 stat_feat[i,8] = n_quotes(question)
 stat_feat[i,9] = n_ampersand(question)
 stat_feat[i,10] = n_dash(question)
 return stat_feat<split>
set1 = df.copy()
set1 = set1[["Survived", "Pclass", "Sex", "Age", "SibSp", "Parch", "Fare"]]
set1.dropna(axis=0, inplace=True)
 rows, cols = set1.shape
print(f"set1 DataFrame has {rows} rows and {cols} columns")
 set1.tail()
train_stat = get_stat(train_df.question_text)
 test_stat = get_stat(test_df.question_text )<string_transform>
y = set1["Age"]
X = set1[["Survived", "Pclass", "Sex", "SibSp", "Parch", "Fare"]]
train_list = list(train_df.question_text.apply(lambda s: s.lower()))
test_list = list(test_df.question_text.apply(lambda s: s.lower()))
train_text = ' '.join(train_list)
 test_text = ' '.join(test_list )<load_pretrained>
train_X, val_X, train_y, val_y = train_test_split(X, y, train_size=0.8, test_size=0.2, random_state=0)
 print(f"Training features shape: {train_X.shape}")
 print(f"Training labels shape: {train_y.shape}")
 print(f"Testing features shape: {val_X.shape}")
 print(f"Testing labels shape: {val_y.shape}" )
nlp = spacy.load("en", disable=['tagger','parser','ner','textcat'] )<feature_engineering>
def get_mae1(max_leaf_nodes, train_X, val_X, train_y, val_y):
age_model = DecisionTreeRegressor(max_leaf_nodes=max_leaf_nodes, random_state=0)
 age_model.fit(train_X, train_y)
 val_predictions = age_model.predict(val_X)
 return mean_absolute_error(val_y, val_predictions)
 for max_leaf_nodes in [5, 10, 15, 20, 50, 100]:
mae = get_mae1(max_leaf_nodes, train_X, val_X, train_y, val_y)
 print(f"Max leaf nodes: {max_leaf_nodes} \t\t Mean Absolute Error: {mae}")
vocab = {}
lemma_vocab = {}
word_idx = 1
train_tokens = []
for doc in tqdm(nlp.pipe(train_list)) :
curr_tokens = []
for token in doc:
if token.text not in vocab:
vocab[token.text] = word_idx
lemma_vocab[token.text] = token.lemma_
word_idx += 1
curr_tokens.append(vocab[token.text])
 train_tokens.append(np.array(curr_tokens, dtype=int))
test_tokens = []
for doc in tqdm(nlp.pipe(test_list)) :
curr_tokens = []
for token in doc:
if token.text not in vocab:
vocab[token.text] = word_idx
lemma_vocab[token.text] = token.lemma_
word_idx += 1
curr_tokens.append(vocab[token.text])
 test_tokens.append(np.array(curr_tokens, dtype=int))<categorify>
age_model = DecisionTreeRegressor(max_leaf_nodes=10, random_state=0)
 age_model.fit(X, y )
def pad(questions, seq_length):

 features = np.zeros(( len(questions), seq_length+1), dtype=int)
 for i, sentence in enumerate(questions):
if len(sentence)==0:
continue
features[i, 0] = len(sentence)
 features[i, -len(sentence):] = sentence
return features<define_variables>
set2 = df.copy()
columns = ["Survived", "Pclass", "Sex", "Age", "SibSp", "Parch", "Fare"]
set2 = set2.loc[set2["Age"].isnull() , columns]
rows, cols = set2.shape
print(f"set2 DataFrame has {rows} rows and {cols} columns")
 set2.tail()
seq_length = max(max(map(len, train_tokens)) , max(map(len, test_tokens)) )<categorify>
X = set2[["Survived", "Pclass", "Sex", "SibSp", "Parch", "Fare"]]
set2["Age"] = age_model.predict(X)
 set2.head()
train_tokens = pad(train_tokens, seq_length)
 test_tokens = pad(test_tokens, seq_length )<categorify>
set1 = set1.append(set2)
 rows, cols = set1.shape
print(f"set1 DataFrame has {rows} rows and {cols} columns")
 set1.head()
def get_embeddings(file):
embeddings = {}
with open(file, encoding="utf8", errors='ignore')as f:
for line in tqdm(f):
line_list = line.split(" ")
 if len(line_list)> 100:
embeddings[line_list[0]] = np.array(line_list[1:], dtype='float32')
 return embeddings
def get_embeddings_matrix(vocab, lemma_vocab, embeddings, keyedVector=False):
ps = PorterStemmer()
lc = LancasterStemmer()
sb = SnowballStemmer("english")
 n_words = len(vocab)
 if keyedVector:
emb_size = embeddings.vector_size
else:
emb_size = next(iter(embeddings.values())).shape[0]
if keyedVector:
emb_dict = {}
for word in vocab:
try:
emb_dict[word] = embeddings.get_vector(word)
 except:
continue
embeddings = emb_dict
embedding_matrix = np.zeros(( n_words+1, emb_size), dtype=np.float32)
 unknown_vec = np.zeros(( emb_size,), dtype=np.float32)- 1
unknown_words = 0
for word in tqdm(vocab):
emb_vec = embeddings.get(word)
 if emb_vec is not None:
embedding_matrix[vocab[word]] = emb_vec
continue
emb_vec = embeddings.get(lemma_vocab[word])
 if emb_vec is not None:
embedding_matrix[vocab[word]] = emb_vec
continue
emb_vec = embeddings.get(ps.stem(word))
if emb_vec is not None:
embedding_matrix[vocab[word]] = emb_vec
continue
emb_vec = embeddings.get(lc.stem(word))
if emb_vec is not None:
embedding_matrix[vocab[word]] = emb_vec
continue
emb_vec = embeddings.get(sb.stem(word))
if emb_vec is not None:
embedding_matrix[vocab[word]] = emb_vec
continue
embedding_matrix[vocab[word]] = unknown_vec
unknown_words += 1
print('% known words: {:.2%}'.format(1 - unknown_words/n_words))
return embedding_matrix<load_pretrained>
y = set1["Survived"]
X = set1[["Pclass", "Sex", "Age", "SibSp", "Parch", "Fare"]]
glove_file = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
glove_emb = get_embeddings(glove_file)
 glove_emb_matrix = get_embeddings_matrix(vocab, lemma_vocab, glove_emb)
 del glove_emb
gc.collect()<load_pretrained>
train_X, val_X, train_y, val_y = train_test_split(X, y, train_size=0.8, test_size=0.2, random_state=0)
 print(f"Training features shape: {train_X.shape}")
 print(f"Training labels shape: {train_y.shape}")
 print(f"Testing features shape: {val_X.shape}")
 print(f"Testing labels shape: {val_y.shape}" )
fasttext_file = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
fasttext_emb = get_embeddings(fasttext_file)
 fasttext_emb_matrix = get_embeddings_matrix(vocab, lemma_vocab, fasttext_emb)
 del fasttext_emb
gc.collect() 
<load_pretrained>
def get_mae2(max_leaf_nodes, train_X, val_X, train_y, val_y):
survival_model = RandomForestClassifier(max_leaf_nodes=max_leaf_nodes, random_state=0)
 survival_model.fit(train_X, train_y)
 val_predictions = survival_model.predict(val_X)
 return mean_absolute_error(val_y, val_predictions)
 for max_leaf_nodes in [5, 8, 10, 15, 20, 50, 100]:
mae = get_mae2(max_leaf_nodes, train_X, val_X, train_y, val_y)
 print(f"Max leaf nodes: {max_leaf_nodes} \t\t Mean Absolute Error: {mae}")
word2vec_file = '.. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'
word2vec_emb = KeyedVectors.load_word2vec_format(word2vec_file, binary=True)
 word2vec_emb_matrix = get_embeddings_matrix(vocab, lemma_vocab, word2vec_emb, keyedVector=True)
 del word2vec_emb
gc.collect()<load_pretrained>
survival_model = RandomForestClassifier(max_leaf_nodes=15, random_state=0)
 accuracy = survival_model.fit(X, y ).score(X, y)
 print(f"Accuracy value: {accuracy}" )
paragram_file = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
paragram_emb = get_embeddings(paragram_file)
 paragram_emb_matrix = get_embeddings_matrix(vocab, lemma_vocab, paragram_emb)
 del paragram_emb
gc.collect()<concatenate>
tdf = pd.read_csv("/kaggle/input/titanic/test.csv", index_col="PassengerId")
 rows, cols = tdf.shape
print(f"Original Test DataFrame has {rows} rows and {cols} columns")
 tdf.head()
emb_matrix = np.concatenate(( glove_emb_matrix,
paragram_emb_matrix), axis=1)
 del glove_emb_matrix, fasttext_emb_matrix, word2vec_emb_matrix, paragram_emb_matrix
gc.collect()<concatenate>
label_encoder = LabelEncoder()
tdf["Sex"] = label_encoder.fit_transform(tdf["Sex"])
 tdf["Family_Size"] = tdf["SibSp"] + tdf["Parch"] + 1
tdf["Fare_Per_Person"] = tdf["Fare"] / tdf["Family_Size"]
tdf.tail()
train_feat = np.concatenate(( train_stat, train_tokens), axis=1)
 test_feat = np.concatenate(( test_stat, test_tokens), axis=1 )<split>
tdf[tdf["Fare"].isnull() ]
x_train, x_val, label_train, label_val = train_test_split(train_feat, labels, test_size=0.1, random_state=0)
 train_data = TensorDataset(torch.from_numpy(x_train), torch.from_numpy(label_train))
valid_data = TensorDataset(torch.from_numpy(x_val), torch.from_numpy(label_val))
test_data = TensorDataset(torch.from_numpy(test_feat))
batch_size = 64
train_loader = DataLoader(train_data, shuffle=True, batch_size=batch_size)
 valid_loader = DataLoader(valid_data, shuffle=True, batch_size=batch_size)
 test_loader = DataLoader(test_data, shuffle=False, batch_size=batch_size )<train_model>
test_set1 = tdf.copy()
filtr =(~test_set1["Fare"].isnull())&(~test_set1["Age"].isnull())&(test_set1["Pclass"] == 3)
 test_set1 = test_set1[filtr]
rows, cols = test_set1.shape
print(f"test_set1 DataFrame has {rows} rows and {cols} columns")
 test_set1.head()
train_on_gpu=torch.cuda.is_available()
if train_on_gpu:
print('Training on GPU.')
 else:
print('No GPU available, training on CPU.' )<choose_model_class>
y = test_set1["Fare"]
X = test_set1[["Pclass", "Sex", "Age", "SibSp", "Parch"]]
fare_model = DecisionTreeRegressor(random_state=1)
 r_squared = fare_model.fit(X, y ).score(X, y)
 print(f"R-Squared value: {r_squared}" )
def init_emb_layer(self, embedding_matrix):

 embedding_matrix = torch.tensor(embedding_matrix, dtype=torch.float32)
 num_emb, emb_size = embedding_matrix.size()
emb_layer = nn.Embedding.from_pretrained(embedding_matrix)
 return emb_layer
class SelfAttention(nn.Module):

 def __init__(self, attention_size, batch_first=False, non_linearity="tanh"):
super(SelfAttention, self ).__init__()
self.batch_first = batch_first
self.attention_weights = nn.Parameter(torch.FloatTensor(attention_size))
self.softmax = nn.Softmax(dim=-1)
 if non_linearity == "relu":
self.non_linearity = nn.ReLU()
else:
self.non_linearity = nn.Tanh()
nn.init.uniform(self.attention_weights.data, -0.005, 0.005)
 def get_mask(self, attentions, lengths):

 max_len = max(lengths.data)
 mask = torch.autograd.Variable(torch.ones(attentions.size())).detach()
if attentions.data.is_cuda:
mask = mask.cuda()
for i, l in enumerate(lengths.data):
if l < max_len:
mask[i, :-l] = 0
return mask
def forward(self, inputs, lengths):
scores = self.non_linearity(inputs.matmul(self.attention_weights))
scores = self.softmax(scores)
 mask = self.get_mask(scores, lengths)
 masked_scores = scores * mask
_sums = masked_scores.sum(-1, keepdim=True)
 scores = masked_scores.div(_sums)
 weighted = torch.mul(inputs, scores.unsqueeze(-1 ).expand_as(inputs))
representations = weighted.sum(1 ).squeeze()
return representations
class Quora_model(nn.Module):
def __init__(self, hidden_layer_dim, embedding_matrix, hidden_dim, gru_layers, stat_layers, drop_prob=0.5):

 super(Quora_model, self ).__init__()
self.hidden_layer_dim = hidden_layer_dim
self.gru_layers = gru_layers
self.emb_dim = embedding_matrix.shape[1]
self.hidden_dim = hidden_dim
self.stat_layers = stat_layers
stat_in_dim = n_stats + 1
modules = []
for out_dim in self.stat_layers:
modules.append(nn.Linear(stat_in_dim, out_dim))
modules.append(nn.ReLU())
 stat_in_dim = out_dim
self.stat_dense = nn.Sequential(*modules)
 self.embedding = init_emb_layer(self, embedding_matrix)
 self.gru = nn.GRU(self.emb_dim, self.hidden_dim, self.gru_layers,
batch_first=True, bidirectional=True, dropout = drop_prob)
 self.attention = SelfAttention(self.hidden_dim*2, batch_first=True)
 self.final_dense = nn.Sequential(
 nn.Dropout(p=drop_prob),
nn.Linear(self.hidden_dim*2 + out_dim, self.hidden_layer_dim),
nn.ReLU() ,
nn.Dropout(p=drop_prob),
nn.Linear(self.hidden_layer_dim, 1),
nn.Sigmoid() 
)
 def forward(self, x, hidden):
batch_size, _ = x.size()
if hidden.size(1)!= batch_size:
hidden = hidden[:, :batch_size, :].contiguous()
lengths = x[:,n_stats].cpu().numpy().astype(int)
 seq_len = max(lengths)
 x_text = x[:, -seq_len:]
x_stat = x[:, :n_stats+1].type(torch.FloatTensor)
 if train_on_gpu:
x_stat = x_stat.cuda()
x_text = self.embedding(x_text)
 out_gru, _ = self.gru(x_text, hidden)
 out_att = self.attention(out_gru, lengths)
 out_stat = self.stat_dense(x_stat)
 out = torch.cat(( out_att, out_stat), dim=1)
 out = self.final_dense(out)
 return out
def init_hidden(self, batch_size):

 weight = next(self.parameters() ).data
if train_on_gpu:
hidden = weight.new(self.gru_layers*2, batch_size, self.hidden_dim ).zero_().cuda()
else:
hidden = weight.new(self.gru_layers*2, batch_size, self.hidden_dim ).zero_()
return hidden<choose_model_class>
tdf.loc[1044, ["Fare"]] = fare_model.predict([[3, 1, 60.5, 0, 0]])
 tdf.loc[1044, ["Fare_Per_Person"]] = fare_model.predict([[3, 1, 60.5, 0, 0]])
 tdf.loc[1044]
hidden_dim = 256
gru_layers = 1
dropout = 0.1
stat_layers_dim = [16, 8]
hidden_layer_dim = 64
model = Quora_model(hidden_layer_dim, emb_matrix, hidden_dim, gru_layers, stat_layers_dim, dropout)
 model<choose_model_class>
test_set1 = tdf.copy()
test_set1 = test_set1.loc[~tdf["Age"].isnull() ]
rows, cols = test_set1.shape
print(f"test_set1 DataFrame has {rows} rows and {cols} columns")
 test_set1.head()
epochs = 4
print_every = 1000
early_stop = 20
clip = 5
lr=0.001
criterion = nn.BCELoss()
optimizer = torch.optim.Adam(model.parameters() , lr=lr )<train_model>
y = test_set1["Age"]
X = test_set1[["Pclass", "Sex", "SibSp", "Parch", "Fare"]]
def train_model(model, train_loader, valid_loader, batch_size, epochs,
optimizer, criterion, clip, print_every, early_stop):
if(train_on_gpu):
model.cuda()
counter = 0
model.train()
breaker = False
for e in range(epochs):
for inputs, labels in train_loader:
counter += 1
if(train_on_gpu):
inputs, labels = inputs.cuda() , labels.cuda()
h = model.init_hidden(batch_size)
 model.zero_grad()
output = model(inputs, h)
 loss = criterion(output.squeeze() , labels.float())
 loss.backward()
nn.utils.clip_grad_norm_(model.parameters() , clip)
 optimizer.step()
if counter % print_every == 0:
with torch.no_grad() :
val_losses = []
all_val_labels = []
all_val_preds = []
all_val_probs = []
model.eval()
for inputs, labels in valid_loader:
all_val_labels += list(labels)
 if(train_on_gpu):
inputs, labels = inputs.cuda() , labels.cuda()
val_h = model.init_hidden(batch_size)
 output = model(inputs, val_h)
 val_loss = criterion(output.squeeze() , labels.float())
 val_losses.append(val_loss.item())
 preds = torch.round(output.squeeze())
 preds = np.squeeze(preds.cpu().numpy())
 all_val_preds += list(preds)
 output = np.squeeze(output.cpu().detach().numpy())
 all_val_probs += list(output)
 current_loss = np.mean(val_losses)
 print("Epoch: {}/{}...".format(e+1, epochs),
"Step: {}...".format(counter),
"Loss: {:.6f}...".format(loss.item()),
"Val Loss: {:.6f}...".format(current_loss),
"F1-score(threshold=0.5): {:.3%}".format(f1_score(all_val_labels, all_val_preds)))
 if counter == print_every or current_loss < best_loss:
best_loss = current_loss
best_val_labels = all_val_labels
best_probs = all_val_probs
torch.save(model.state_dict() , 'checkpoint.pth')
 counter_eval = 0
counter_eval += 1
if counter_eval == early_stop:
breaker = True
break
model.train()
if breaker:
break
state_dict = torch.load('checkpoint.pth')
 model.load_state_dict(state_dict)
 return best_probs, best_val_labels<train_model>
train_X, val_X, train_y, val_y = train_test_split(X, y, train_size=0.8, test_size=0.2, random_state=0)
 print(f"Training features shape: {train_X.shape}")
 print(f"Training labels shape: {train_y.shape}")
 print(f"Testing features shape: {val_X.shape}")
 print(f"Testing labels shape: {val_y.shape}" )
t0 = time.time()
all_val_probs, all_val_labels = train_model(model, train_loader, valid_loader, batch_size, epochs,
optimizer, criterion, clip, print_every, early_stop)
 tf = time.time()
print("
Execution time: {:.2f}min".format(( tf-t0)/60))<find_best_params>
def get_mae3(max_leaf_nodes, train_X, val_X, train_y, val_y):
age_model = DecisionTreeRegressor(max_leaf_nodes=max_leaf_nodes, random_state=0)
 age_model.fit(train_X, train_y)
 val_predictions = age_model.predict(val_X)
 return mean_absolute_error(val_y, val_predictions)
 for max_leaf_nodes in [5, 10, 15, 20, 50, 100]:
mae = get_mae3(max_leaf_nodes, train_X, val_X, train_y, val_y)
 print(f"Max leaf nodes: {max_leaf_nodes} \t\t Mean Absolute Error: {mae}")
best_score = 0
for thr in np.arange(0.0, 0.5, 0.005):
pred = np.array(all_val_probs > thr, dtype=int)
 score = f1_score(all_val_labels, pred)
 print("Threshold: {:.3f}...F1-score {:.3%}".format(thr, score))
if score > best_score:
best_score = score
best_thr = thr
print("
Best threshold: {:.3f}...F1-score {:.3%}".format(best_thr, best_score))<train_model>
age_model = DecisionTreeRegressor(random_state=1)
 age_model.fit(X, y )
model.eval()
with torch.no_grad() :
all_test_preds = []
for inputs in test_loader:
inputs = inputs[0]
if(train_on_gpu):
inputs = inputs.cuda()
test_h = model.init_hidden(batch_size)
 output = model(inputs, test_h)
 preds =(output.squeeze() > best_thr ).type(torch.IntTensor)
 preds = np.squeeze(preds.cpu().numpy())
 all_test_preds += list(preds.astype(int))<save_to_csv>
test_set2 = tdf.copy()
test_set2 = test_set2.loc[tdf["Age"].isnull() ]
rows, cols = test_set2.shape
print(f"test_set2 DataFrame has {rows} rows and {cols} columns")
 test_set2.head()
sub = pd.DataFrame({
'qid': test_df.qid,
'prediction': all_test_preds
})
 sub = sub[['qid', 'prediction']]
sub.to_csv('submission.csv', index=False, sep=',' )<import_modules>
X = test_set2[["Pclass", "Sex", "SibSp", "Parch", "Fare"]]
test_set2["Age"] = age_model.predict(X)
 test_set2.head()
tqdm.pandas()
gc.collect()<define_variables>
test_set1 = test_set1.append(test_set2)
 rows, cols = test_set1.shape
print(f"test_set1 DataFrame has {rows} rows and {cols} columns")
 test_set1.head()
max_features= 200000
max_senten_len = 40
max_senten_num = 3
embed_size = 300
VALIDATION_SPLIT = 0<import_modules>
X = test_set1[["Pclass", "Sex", "Age", "SibSp", "Parch", "Fare"]]
predictions = survival_model.predict(X)
 output = pd.DataFrame({"PassengerId": test_set1.index, "Survived": predictions})
 output.to_csv("my_submission.csv", index=False)
 print("my_submission.csv is ready to submit!" )