Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<normalization>
import numpy as np
import pandas as pd
import os
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
import torch.nn.functional as F
from sklearn.preprocessing import LabelEncoder
class DotProductAttentionScoring(nn.Module):
def __init__(self, scale):
super().__init__()
self.scale = np.sqrt(scale)
 def forward(self, query, keys):
b_q, n_q, d_q = query.size()
b_k, n_k, d_k = keys.size()
assert b_q == b_k
dot_products = torch.bmm(query, torch.transpose(keys, 1, 2)) / self.scale
return dot_products<categorify>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
class TimeInvariant(nn.Module):
def __init__(self, inner):
super().__init__()
self.inner = inner
def forward(self, data):
batch,time,dim = data.size()
result = self.inner(data.view(batch * time, dim))
result = result.view(batch, time, -1)
 return result 
<categorify>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
class PositionalEncoding(nn.Module):
"Implement the PE function."
def __init__(self, d_model, max_len=5000):
super(PositionalEncoding, self ).__init__()
pe = torch.zeros(max_len, d_model,dtype=torch.float32)
 position = torch.arange(0., max_len ).unsqueeze(1)
 div_term = torch.exp(torch.arange(0., d_model, 2)*
-(math.log(10000.0)/ d_model))
pe[:, 0::2] = torch.sin(position * div_term)
 pe[:, 1::2] = torch.cos(position * div_term)
 pe = pe.unsqueeze(0)
 self.register_buffer('pe', pe)
 def forward(self, x):
x = x + Variable(self.pe[:, :x.size(1)],
requires_grad=False)
 return x 
<choose_model_class>
all_df = pd.concat([train, test], sort=False )
class SelfAttentionNet(nn.Module):
def __init__(self, vocab, embeddings, num_trainable):
super().__init__()
self.pos_encoding = PositionalEncoding(50,120)
 self.num_trainable = num_trainable
self.embedding = nn.Embedding(num_embeddings=len(vocab.itos), embedding_dim=300,padding_idx=vocab.stoi['<pad>'])
 self.embedding.weight.data[num_trainable:].copy_(torch.from_numpy(embeddings))
self.drop_emb = nn.Dropout2d(p=0.5)
 self.pretrained_projection = nn.Sequential(
 nn.Linear(300,300))
key_transform = TimeInvariant(
 nn.Sequential(
 nn.Linear(350,300),
nn.ReLU() ,
nn.Linear(300,300)
))
query_transform = TimeInvariant(
 nn.Sequential(
 nn.Linear(350,300),
nn.ReLU() ,
nn.Linear(300,300)
))
value_transform = TimeInvariant(
 nn.Sequential(
 nn.Linear(350,300),
nn.ReLU() ,
nn.Linear(300,300)
))
attention_scoring = DotProductAttentionScoring(300.)
self.attention1 = GenericAttention(attention_scoring, query_transform, key_transform,value_transform)
 self.fc = nn.Sequential(
 nn.Dropout(p=0.2),
nn.Linear(300,500),
nn.BatchNorm1d(500),
nn.ReLU() ,
nn.Linear(500, 1)
)
 def zero_embedding_grad(self):
self.embedding.weight.grad[self.num_trainable:] = 0
def set_emb_dropout(self, p):
self.emb_dropout = p
def apply_spatial_dropout(self,dropout_layer, x):
x = torch.transpose(x,1,2)
 x = x.view(x.shape[0], x.shape[1], x.shape[2], 1)
 x = dropout_layer(x)
 x = torch.squeeze(x,dim=3)
 x = torch.transpose(x,1,2)
 return x
def get_attention_weights(self, x):
x = self.embed(x)
 x,weights = self.attention1(x, x, x)
 return weights
def mixed_embedding(self, x):
e = self.embedding(x)
 pretrained_mask = x < self.num_trainable
pretrained = e.view(-1,300)[pretrained_mask.view(-1)]
e.view(-1,300)[pretrained_mask.view(-1)] = self.pretrained_projection(pretrained)
 return e
def embed(self, x):
x = self.mixed_embedding(x)
 x = self.apply_spatial_dropout(self.drop_emb, x)
 pe = torch.zeros(x.size(0), x.size(1), 50,device=x.device)
 pe = self.pos_encoding(pe)
 x = torch.cat(( x,pe), dim=2)
 return x
def forward(self, x):
x = self.embed(x)
 x,_ = self.attention1(x, x, x)
 reduced,_ = torch.max(x, dim=1)
 return self.fc(reduced)
<choose_model_class>
def preprocess(df, cat_cols):
df = df.drop(['PassengerId', 'Name', 'Ticket', 'Cabin'], axis=1)
 for cat_col in cat_cols:
if cat_col in ['Embarked']:
df[cat_col] = LabelEncoder().fit_transform(df[cat_col].astype(str))
else:
df[cat_col] = LabelEncoder().fit_transform(df[cat_col])
 df = df.fillna(df.mean())
 return df
class Net(nn.Module):
def __init__(self, vocab, embeddings, num_trainable, normalize=False):
super().__init__()
self.num_trainable = num_trainable
self.embedding = nn.Embedding(num_embeddings=len(vocab.itos), embedding_dim=300,padding_idx=vocab.stoi['<pad>'])
 if normalize:
embeddings = embeddings / np.linalg.norm(embeddings,axis=1 ).reshape(-1,1)
 self.embedding.weight.data[num_trainable:].copy_(torch.from_numpy(embeddings))
self.drop_emb = nn.Dropout2d(p=0.5)
 self.rnn1 = nn.GRU(input_size=300, hidden_size=150,num_layers=1,bidirectional=True)
 self.drop_rnn = nn.Dropout(p=0.2)
 self.rnn2 = nn.GRU(input_size=300, hidden_size=150,num_layers=1,bidirectional=True)
 self.fc = nn.Sequential(
 nn.Dropout(p=0.3),
nn.Linear(300, 1)
)
 def embed(self, x):
return self.embedding(x)
 def zero_embedding_grad(self):
self.embedding.weight.grad[self.num_trainable:] = 0
def get_hidden(self, x):
x = self.embedding(x)
 x = torch.transpose(x,1,2)
 x = x.view(x.shape[0], x.shape[1], x.shape[2], 1)
 x = self.drop_emb(x)
 x = torch.squeeze(x,dim=3)
 x = torch.transpose(x,1,2)
 x = torch.transpose(x,0,1)
 x, hidden = self.rnn1(x)
 return x
def forward(self, x):
x = self.embedding(x)
 x = torch.transpose(x,1,2)
 x = x.view(x.shape[0], x.shape[1], x.shape[2], 1)
 x = self.drop_emb(x)
 x = torch.squeeze(x,dim=3)
 x = torch.transpose(x,1,2)
 x = torch.transpose(x,0,1)
 output, hidden = self.rnn1(x)
 output = self.drop_rnn(output)
 output = output + self.rnn2(output)[0]
output = output.transpose(0,1)
 x,_ = torch.max(output, dim=1)
 x = self.fc(x)
 return x<define_variables>
cat_cols = ['Pclass', 'Sex', 'SibSp', 'Parch', 'Embarked']
all_df = preprocess(all_df, cat_cols)
 all_df.head()
num_scratch = len(specials)+ len(trainable_words )<choose_model_class>
class TabularDataset(Dataset):
def __init__(self, df, categorical_columns, output_column=None):
super().__init__()
self.len = df.shape[0]
self.categorical_columns = categorical_columns
self.continous_columns = [col for col in df.columns if col not in self.categorical_columns + [output_column]]
if self.continous_columns:
self.cont_X = df[self.continous_columns].astype(np.float32 ).values
else:
self.cont_X = np.zeros(( self.len, 1))
if self.categorical_columns:
self.cat_X = df[self.categorical_columns].astype(np.int64 ).values
else:
self.cat_X = np.zeros(( self.len, 1))
if output_column != None:
self.has_label = True
self.label = df[output_column].astype(np.float32 ).values.reshape(-1, 1)
 else:
self.has_label = False
def __len__(self):
return self.len
def __getitem__(self, index):
if self.has_label:
return [self.label[index], self.cont_X[index], self.cat_X[index]]
else:
return [self.cont_X[index], self.cat_X[index]]
net = Net(vocab, np_vectors, num_scratch ).cuda()
best_model = BestModel('best_model', 'best_optimizer' )<choose_model_class>
train_ds = TabularDataset(train_df, cat_cols, 'Survived')
 train_dl = DataLoader(train_ds, 64, shuffle=True )
criterion = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([1.])).cuda()
optimizer = torch.optim.Adam(net.parameters() , lr=0.0003 )<prepare_x_and_y>
class TitanicNet(nn.Module):
def __init__(self, emb_dims, n_cont, lin_layer_sizes, output_size):
super().__init__()
self.emb_layers = nn.ModuleList([nn.Embedding(x, y)for x, y in emb_dims])
 self.n_embs = sum([y for x, y in emb_dims])
 self.n_cont = n_cont
first_lin_layer = nn.Linear(self.n_embs + self.n_cont, lin_layer_sizes[0])
 self.lin_layers = nn.ModuleList(
 [first_lin_layer] +
[nn.Linear(lin_layer_sizes[i], lin_layer_sizes[i + 1])for i in range(len(lin_layer_sizes)- 1)] 
)
 self.output_layer = nn.Linear(lin_layer_sizes[-1], output_size)
 nn.init.kaiming_normal_(self.output_layer.weight.data)
 self.first_bn_layer = nn.BatchNorm1d(self.n_cont)
 self.bn_layers = nn.ModuleList([nn.BatchNorm1d(size)for size in lin_layer_sizes])
 def forward(self, cont_data, cat_data):
if self.n_embs != 0:
x = [emb_layer(cat_data[:, i])for i, emb_layer in enumerate(self.emb_layers)]
x = torch.cat(x, 1)
 if self.n_cont != 0:
normalized_cont_data = self.first_bn_layer(cont_data)
 if self.n_embs != 0:
x = torch.cat([x, normalized_cont_data], 1)
 else:
x = cont_data
for lin_layer, bn_layer in zip(self.lin_layers, self.bn_layers):
x = torch.relu(lin_layer(x))
x = bn_layer(x)
 x = self.output_layer(x)
 x = torch.sigmoid(x)
 return x
def truncate_batch(batch):
x,y = batch
if x.shape[1] > 100:
x = x[:,:100]
return x,y<load_pretrained>
cat_dims = [int(all_df[col].nunique())for col in cat_cols]
cat_dims
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=256,
collate_fn=lambda samples: truncate_batch(train_dataset.collate(samples)) , shuffle=True)
 val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=256,collate_fn=lambda samples: truncate_batch(val_dataset.collate(samples)))
<define_variables>
emb_dims = [(x, min(50,(x + 1)// 2)) for x in cat_dims]
emb_dims
print(128 * 112 * 112 * 300 * 4 / 1024 / 1024 )<train_model>
torch.manual_seed(2 )
train_network(net, optimizer,criterion,train_loader,val_loader,16, 3,best_model,
after_gradient=lambda epoch, network: network.zero_embedding_grad())
 best_model.load(net, optimizer)
<train_model>
model = TitanicNet(emb_dims, n_cont=2, lin_layer_sizes=[50, 100, 50], output_size=1)
 optimizer = torch.optim.Adam(model.parameters() , lr=0.003)
 no_of_epochs = 10
criterion = nn.BCELoss()
for epoch in range(no_of_epochs):
epoch_loss = 0
epoch_accuracy = 0
i = 0
for y, cont_x, cat_x in train_dl:
preds = model(cont_x, cat_x)
 loss = criterion(preds, y)
 epoch_loss += loss
accuracy =(( preds > 0.5 ).float() == y ).float().mean()
epoch_accuracy += accuracy
optimizer.zero_grad()
loss.backward()
optimizer.step()
print("Epoch ", epoch, ", loss: ", epoch_loss.item() /len(train_dl), "accuracy: ", epoch_accuracy.item() /len(train_dl))
train_network(net, optimizer,criterion,train_loader,val_loader,10, 3,best_model,
after_gradient=lambda epoch, network: network.zero_embedding_grad())
 best_model.load(net, optimizer)
<load_pretrained>
test_df = all_df.tail(test.shape[0])
 test_ds = TabularDataset(test_df, cat_cols, 'Survived')
 test_dl = DataLoader(test_ds, len(test_ds))
best_model.load(net, optimizer )<create_dataframe>
with torch.no_grad() :
for _, cont_x, cat_x in test_dl:
preds = model(cont_x, cat_x)
 preds =(preds > 0.5 )
best_model_with_fixed_embeddings = best_model.copy('best_model_fixed', 'best_optimizer_fixed' )<load_pretrained>
output_df = pd.DataFrame({'PassengerId':test['PassengerId'],'Survived':preds.flatten().numpy() } )
<train_model><EOS> 
output_df.to_csv('titanic_preds.csv', index=False )