Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<choose_model_class>
%matplotlib inline
embedding_dim = 300
embedding_path = '.. /save/embedding_matrix.npy'
use_pretrained_embedding = True
hidden_size = 64
gru_len = hidden_size
Routings = 4
Num_capsule = 5
Dim_capsule = 5
dropout_p = 0.25
rate_drop_dense = 0.28
LR = 0.001
T_epsilon = 1e-7
num_classes = 30
class Embed_Layer(nn.Module):
def __init__(self, embedding_matrix=None, vocab_size=None, embedding_dim=300):
super(Embed_Layer, self ).__init__()
self.encoder = nn.Embedding(vocab_size + 1, embedding_dim)
 if use_pretrained_embedding:
self.encoder.weight.data.copy_(t.from_numpy(embedding_matrix))
def forward(self, x, dropout_p=0.25):
return nn.Dropout(p=dropout_p )(self.encoder(x))
class Caps_Layer(nn.Module):
def __init__(self, input_dim_capsule=gru_len * 2, num_capsule=Num_capsule, dim_capsule=Dim_capsule, \
routings=Routings, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Caps_Layer, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = self.squash
else:
self.activation = nn.ReLU(inplace=True)
 if self.share_weights:
self.W = nn.Parameter(
 nn.init.xavier_normal_(t.empty(1, input_dim_capsule, self.num_capsule * self.dim_capsule)))
 else:
self.W = nn.Parameter(
 t.randn(BATCH_SIZE, input_dim_capsule, self.num_capsule * self.dim_capsule))
def forward(self, x):
if self.share_weights:
u_hat_vecs = t.matmul(x, self.W)
 else:
print('add later')
 batch_size = x.size(0)
 input_num_capsule = x.size(1)
 u_hat_vecs = u_hat_vecs.view(( batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = u_hat_vecs.permute(0, 2, 1, 3)
 b = t.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = b.permute(0, 2, 1)
 c = F.softmax(b, dim=2)
 c = c.permute(0, 2, 1)
 b = b.permute(0, 2, 1)
 outputs = self.activation(t.einsum('bij,bijk->bik',(c, u_hat_vecs)))
 if i < self.routings - 1:
b = t.einsum('bik,bijk->bij',(outputs, u_hat_vecs))
return outputs
def squash(self, x, axis=-1):
s_squared_norm =(x ** 2 ).sum(axis, keepdim=True)
 scale = t.sqrt(s_squared_norm + T_epsilon)
 return x / scale 
<normalization>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv')
 all_data = [train,test]
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )<choose_model_class>
print(train[['Sex', 'Survived']].groupby(['Sex'], as_index=False ).mean())
class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self ).__init__()
fc_layer = 16
fc_layer1 = 16
nbr_filers_1cnn = 5
nbr_filers_2cnn = 10
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.1)
 self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)
 self.lstm_attention = Attention(hidden_size * 2, maxlen)
 self.gru_attention = Attention(hidden_size * 2, maxlen)
 self.CNN2D_1 = nn.Conv2d(
 in_channels = 1,
out_channels = nbr_filers_1cnn,
kernel_size =(12,10),
stride =(2,4),
padding =(3,5)
)
 self.BN_1 = nn.BatchNorm2d(nbr_filers_1cnn)
 self.MaxPool2D_1 = nn.MaxPool2d(kernel_size=(3,3), stride =(3,3))
self.CNN2D_2 = nn.Conv2d(
 in_channels = nbr_filers_1cnn,
out_channels = nbr_filers_2cnn,
kernel_size =(5,5),
stride =(2,2),
padding =(1,1), 
)
 self.BN_2 = nn.BatchNorm2d(nbr_filers_2cnn)
 self.MaxPool2D_2 = nn.MaxPool2d(kernel_size=(5,5))
self.linCNN = nn.Linear(10, 1)
 self.lincaps = nn.Linear(Num_capsule * Dim_capsule, 1)
 self.caps_layer = Caps_Layer()
self.bn = nn.BatchNorm1d(16, momentum=0.5)
 self.linear = nn.Linear(hidden_size*8+1, fc_layer1)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.fc = nn.Linear(fc_layer**2,fc_layer)
 self.out = nn.Linear(fc_layer, 1)
 def forward(self, x):
h_embedding = self.embedding(x[0])
 h_embedding = torch.squeeze(
 self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 x_h_gru = h_gru.unsqueeze(1)
 cnn_1 = self.CNN2D_1(x_h_gru)
 cnn_1 = self.BN_1(cnn_1)
 cnn_1 = self.relu(cnn_1)
 max_pool_1 = self.MaxPool2D_1(cnn_1)
 cnn_2 = self.CNN2D_2(max_pool_1)
 cnn_2 = self.BN_2(cnn_2)
 cnn_2 = self.relu(cnn_2)
 max_pool_2 = self.MaxPool2D_2(cnn_2)
 cnn_out = max_pool_2.reshape(max_pool_2.size(0),-1)
 cnnlin = self.relu(self.linCNN(cnn_out))
h_lstm_atten = self.lstm_attention(h_lstm)
 h_gru_atten = self.gru_attention(h_gru)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 f = torch.tensor(x[1], dtype=torch.float ).cuda()
conc = torch.cat(( h_lstm_atten, h_gru_atten,cnnlin, avg_pool, max_pool), 1)
 conc = self.relu(self.linear(conc))
conc = self.bn(conc)
 conc = self.dropout(conc)
 out = self.out(conc)
 return out<categorify>
print(train[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean())
class MyDataset(Dataset):
def __init__(self,dataset):
self.dataset = dataset
def __getitem__(self, index):
data, target = self.dataset[index]
return data, target, index
def __len__(self):
return len(self.dataset )<define_variables>
for dataset in all_data:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
print(train[['FamilySize', 'Survived']].groupby(['FamilySize'], as_index=False ).mean() )
def sigmoid(x):
return 1 /(1 + np.exp(-x))
train_preds = np.zeros(( len(x_train)))
 test_preds = np.zeros(( len(df_test)))
 seed_everything()
x_test_cuda = torch.tensor(x_test, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 avg_losses_f = []
avg_val_losses_f = []<data_type_conversions>
train = train.drop(['Parch', 'SibSp'], axis=1)
 test = test.drop(['Parch', 'SibSp'], axis=1)
 all_data = [train, test]
train.head()
seed_everything()
for i,(train_idx, valid_idx)in enumerate(splits):
x_train = np.array(x_train)
 y_train = np.array(y_train)
 features = np.array(features)
 x_train_fold = torch.tensor(x_train[train_idx.astype(int)], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(y_train[train_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
kfold_X_features = features[train_idx.astype(int)]
kfold_X_valid_features = features[valid_idx.astype(int)]
x_val_fold = torch.tensor(x_train[valid_idx.astype(int)], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(y_train[valid_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
model = NeuralNet()
model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum')
 step_size = 300
base_lr, max_lr = 0.001, 0.003
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
lr=max_lr)
 scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,
step_size=step_size, mode='exp_range',
gamma=0.99994)
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train = MyDataset(train)
 valid = MyDataset(valid)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(n_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for i,(x_batch, y_batch, index)in enumerate(train_loader):
f = kfold_X_features[index]
y_pred = model([x_batch,f])
 if scheduler:
scheduler.batch_step()
loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(( len(df_test)))
 avg_val_loss = 0.
 for i,(x_batch, y_batch, index)in enumerate(valid_loader):
f = kfold_X_valid_features[index]
y_pred = model([x_batch,f] ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))
avg_losses_f.append(avg_loss)
 avg_val_losses_f.append(avg_val_loss)
 for i,(x_batch,)in enumerate(test_loader):
f = test_features[i * batch_size:(i+1)* batch_size]
y_pred = model([x_batch,f] ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits)
 print('All \t loss={:.4f} \t val_loss={:.4f} \t '.format(np.average(avg_losses_f),np.average(avg_val_losses_f)))
<compute_test_metric>
for dataset in all_data:
dataset['Sex'] = dataset['Sex'].map({'female': 0, 'male': 1} ).astype(int)
 train.head()
def bestThresshold(y_train,train_preds):
tmp = [0,0,0]
delta = 0
for tmp[0] in tqdm(np.arange(0.1, 0.501, 0.01)) :
tmp[1] = f1_score(y_train, np.array(train_preds)>tmp[0])
 if tmp[1] > tmp[2]:
delta = tmp[0]
tmp[2] = tmp[1]
print('best threshold is {:.4f} with F1 score: {:.4f}'.format(delta, tmp[2]))
return delta
delta = bestThresshold(y_train,train_preds )<save_to_csv>
guess_ages = np.zeros(( 2,3))
guess_ages
submission = df_test[['qid']].copy()
submission['prediction'] =(test_preds > delta ).astype(int)
 submission.to_csv('submission.csv', index=False )<load_from_csv>
for dataset in all_data:
for i in range(0, 2):
for j in range(0, 3):
guess_data = dataset[(dataset['Sex'] == i)& \ 
(dataset['Pclass'] == j+1)]['Age'].dropna()
age_guess = guess_data.median()
guess_ages[i,j] = int(age_guess/0.5 + 0.5)* 0.5
for i in range(0, 2):
for j in range(0, 3):
dataset.loc[(dataset.Age.isnull())&(dataset.Sex == i)&(dataset.Pclass == j+1),\
'Age'] = guess_ages[i,j]
dataset['Age'] = dataset['Age'].astype(int)
 train.head()
text = torchtext.data.Field(lower=True, batch_first=True, tokenize=word_tokenize, fix_length=100)
 qid = torchtext.data.Field()
target = torchtext.data.Field(sequential=False, use_vocab=False, is_target=True)
 train_dataset = torchtext.data.TabularDataset(path='.. /input/train.csv', format='csv',
fields={'question_text':('text',text),
'target':('target',target)})
 train, val,test = train_dataset.split(split_ratio=[0.8,0.1,0.1],stratified=True,strata_field='target',random_state=random.getstate())
 submission_x = torchtext.data.TabularDataset(path='.. /input/test.csv', format='csv',
fields={'qid':('qid', qid),
'question_text':('text', text)})
 text.build_vocab(train_dataset, submission_x, min_freq=3)
 qid.build_vocab(submission_x)
 print('train dataset len:',len(train_dataset))
print('train len:',len(train))
print('val len:',len(val))
print('test len:',len(test))<feature_engineering>
train['Age_group'] = pd.cut(train['Age'], 5)
 train[['Age_group', 'Survived']].groupby(['Age_group'], as_index=False ).mean().sort_values(by='Age_group', ascending=True)
glove = torchtext.vocab.Vectors('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt')
 text.vocab.set_vectors(glove.stoi, glove.vectors, dim=300 )<train_model>
for dataset in all_data:
dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age']
train.head()
class EarlyStopping:

 def __init__(self, patience=7, verbose=False):

 self.patience = patience
self.verbose = verbose
self.counter = 0
self.best_score = None
self.early_stop = False
self.val_loss_min = np.Inf
def __call__(self, val_loss, model):
score = -val_loss
if self.best_score is None:
self.best_score = score
if self.verbose:
print(f'Validation loss decreased({self.val_loss_min:.6f} --> {val_loss:.6f} ).Saving model...')
 self.save_checkpoint(val_loss, model)
 elif score < self.best_score:
self.counter += 1
if self.verbose:
print(f'EarlyStopping counter: {self.counter} out of {self.patience}')
 if self.counter >= self.patience:
self.early_stop = True
else:
self.best_score = score
self.save_checkpoint(val_loss, model)
 self.counter = 0
def save_checkpoint(self, val_loss, model):

 if self.verbose:
print(f'Validation loss decreased({self.val_loss_min:.6f} --> {val_loss:.6f} ).Saving model...')
 torch.save(model.state_dict() , 'checkpoint.pt')
 self.val_loss_min = val_loss<set_options>
train = train.drop(['Age_group'], axis=1)
 all_data = [train, test]
train.head()
torch.cuda.init()
torch.cuda.empty_cache()
print('CUDA MEM:',torch.cuda.memory_allocated())
 print('cuda:', torch.cuda.is_available())
 print('cude index:',torch.cuda.current_device())
 batch_size = 512
print('batch_size:',batch_size)
 print('---')
 train_loader = torchtext.data.BucketIterator(dataset=train,
batch_size=batch_size,
shuffle=True,
sort=False)
 val_loader = torchtext.data.BucketIterator(dataset=val,
batch_size=batch_size,
shuffle=False,
sort=False)
 test_loader = torchtext.data.BucketIterator(dataset=test,
batch_size=batch_size,
shuffle=False,
sort=False)
 class SentimentLSTM(nn.Module):
def __init__(self,vocab_vectors,padding_idx,batch_size):
super(SentimentLSTM,self ).__init__()
print('Vocab vectors size:',vocab_vectors.shape)
 self.batch_size = batch_size
self.hidden_dim = 128
self.n_layers = 2
self.embedding = nn.Embedding.from_pretrained(vocab_vectors)
 self.embedding.weight.requires_grad = False
self.embedding.padding_idx = padding_idx
self.lstm = nn.LSTM(input_size=vocab_vectors.shape[1], hidden_size=self.hidden_dim, bidirectional=True,batch_first=True)
 self.linear1 = nn.Linear(self.n_layers*self.hidden_dim,self.hidden_dim)
 self.linear2 = nn.Linear(self.hidden_dim,1)
 self.dropout = nn.Dropout(0.2)
 def forward(self,x):
hidden =(torch.zeros(self.n_layers, x.shape[0], self.hidden_dim ).cuda() ,
torch.zeros(self.n_layers, x.shape[0], self.hidden_dim ).cuda())
 e = self.embedding(x)
 _, hidden = self.lstm(e, hidden)
 out = torch.cat(( hidden[0][-2,:,:], hidden[0][-1,:,:]), dim=1 ).cuda()
out = self.linear1(F.relu(out))
return self.linear2(self.dropout(out))
class SentimentBase(nn.Module):
def __init__(self):
super(SentimentBase,self ).__init__()
self.embedding = nn.Embedding(75966,300)
 self.linear1 = nn.Linear(300*100,128)
 self.linear2 = nn.Linear(128,1)
 def forward(self,x):
emb = self.embedding(x)
 pooled = emb.reshape(( emb.shape[0],emb.shape[1]*emb.shape[2]))
out = self.linear1(F.relu(pooled))
out = self.linear2(out)
 return out
class SentimentCNN(nn.Module):
def __init__(self,vocab_vectors,padding_idx,batch_size):
super(SentimentCNN,self ).__init__()
print('Vocab vectors size:',vocab_vectors.shape)
 self.batch_size = batch_size
self.hidden_dim = 128
self.embedding = nn.Embedding.from_pretrained(vocab_vectors)
 self.embedding.weight.requires_grad = False
self.embedding.padding_idx = padding_idx
self.cnns = nn.ModuleList([nn.Conv1d(in_channels=vocab_vectors.shape[1], out_channels=self.hidden_dim, kernel_size=k)for k in [3,4,5]])
 self.linear1 = nn.Linear(3*self.hidden_dim,self.hidden_dim)
 self.linear2 = nn.Linear(self.hidden_dim,1)
 self.dropout = nn.Dropout(0.2)
 @staticmethod
def conv_and_max_pool(x, conv):

 return F.relu(conv(x ).permute(0, 2, 1 ).max(1)[0])
 def forward(self,x):
e = self.embedding(x)
 e = e.permute(0,2,1)
 cnn_outs = []
for conv in self.cnns:
f =self.conv_and_max_pool(e,conv)
 cnn_outs.append(f)
 out = torch.cat(cnn_outs, dim=1 ).cuda()
out = self.linear1(F.relu(out))
return self.linear2(self.dropout(out))
class SentimentGRU(nn.Module):
def __init__(self,vocab_vectors,padding_idx,batch_size):
super(SentimentGRU,self ).__init__()
print('Vocab vectors size:',vocab_vectors.shape)
 self.batch_size = batch_size
self.hidden_dim = 128
self.n_layers = 2
self.embedding = nn.Embedding.from_pretrained(vocab_vectors)
 self.embedding.weight.requires_grad = False
self.embedding.padding_idx = padding_idx
self.gru = nn.GRU(input_size=vocab_vectors.shape[1], hidden_size=self.hidden_dim, bidirectional=True,batch_first=True)
 self.linear1 = nn.Linear(self.n_layers*self.hidden_dim,self.hidden_dim)
 self.linear2 = nn.Linear(self.hidden_dim,1)
 self.dropout = nn.Dropout(0.2)
 def forward(self,x):
hidden = torch.zeros(self.n_layers, x.shape[0], self.hidden_dim ).cuda()
e = self.embedding(x)
 _, hidden = self.gru(e, hidden)
 out = torch.cat(( hidden[-2,:,:], hidden[-1,:,:]), dim=1 ).cuda()
out = self.linear1(F.relu(out))
return self.linear2(self.dropout(out))
class Ensemble(nn.Module):
def __init__(self,vocab_vectors,padding_idx,batch_size):
super(Ensemble,self ).__init__()
self.lstm = SentimentLSTM(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size ).cuda()
self.gru = SentimentGRU(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size ).cuda()
self.cnn = SentimentCNN(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size ).cuda()
self.base = SentimentBase().cuda()
self.soft = nn.Softmax(dim=1)
 def forward(self,x):
o1 = self.lstm(x)
 o2 = self.gru(x)
 o3 = self.cnn(x)
 o4 = self.base(x)
 out = torch.cat([o1,o2,o3,o4],1)
 s_out = self.soft(out)
 return torch.sum(torch.mul(out,s_out),dim=1 ).reshape(x.shape[0],1)
 model = Ensemble(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size ).cuda()
print(model)
 print('-'*80)
 early_stopping = EarlyStopping(patience=2,verbose=True)
 loss_function = nn.BCEWithLogitsLoss().cuda()
optimizer = optim.Adam(model.parameters() ,lr=1e-3)
 def get_lr(optimizer):
for param_group in optimizer.param_groups:
return param_group['lr']
losses = []
val_losses=[]
epoch_acc=[]
epoch_val_acc=[]
lrs = []
for epoch in range(100):
epoch_losses=[]
epoch_val_losses = []
preds = []
val_preds=[]
targets = []
acc = []
model.train()
for batch,train_batch in enumerate(list(iter(train_loader)) ,1):
optimizer.zero_grad()
y_pred = model(train_batch.text.cuda() ).squeeze(1)
 y_numpy_pred =torch.sigmoid(y_pred ).cpu().detach().numpy()
preds += y_numpy_pred.tolist()
y_true = train_batch.target.float().cuda()
y_numpy_true = train_batch.target.cpu().detach().numpy()
targets += y_numpy_true.tolist()
loss = loss_function(y_pred,y_true)
 epoch_losses.append(loss.item())
 loss.backward()
optimizer.step()
lrs.append(get_lr(optimizer))
acc.append(accuracy_score(y_numpy_true,np.round(y_numpy_pred)))
 if batch % 100 == 0:
print('\rtraining(batch,loss,acc)| ',batch,' ===>',loss.item() ,' acc ',np.mean(acc),end='')
 losses.append(np.mean(epoch_losses))
targets = np.array(targets)
 preds = np.array(preds)
 search_result = threshold_search(targets, preds)
 train_f1 = search_result['f1']
epoch_acc.append(np.mean(acc))
targets = []
val_acc=[]
model.eval()
with torch.no_grad() :
for batch,val_batch in enumerate(list(val_loader),1):
y_pred = model(val_batch.text.cuda() ).squeeze(1)
 y_numpy_pred = torch.sigmoid(y_pred ).cpu().detach().numpy()
val_preds += y_numpy_pred.tolist()
y_true = val_batch.target.float().cuda()
y_numpy_true = val_batch.target.cpu().detach().numpy()
targets += y_numpy_true.tolist()
val_loss = loss_function(y_pred,y_true)
 epoch_val_losses.append(val_loss.item())
 val_acc.append(accuracy_score(y_numpy_true,np.round(y_numpy_pred)))
 if batch % 100 == 0:
print('\rvalidation(batch,acc)| ',batch,' ===>', np.mean(val_acc),end='')
 val_losses.append(np.mean(epoch_val_losses))
epoch_val_acc.append(np.mean(val_acc))
targets = np.array(targets)
 val_preds = np.array(val_preds)
 search_result = threshold_search(targets, val_preds)
 val_f1 = search_result['f1']
print('
EPOCH: ',epoch,'
has acc of ',epoch_acc[-1],' ,has loss of ',losses[-1], ' ,f1 of ',train_f1,'
val acc of ',epoch_val_acc[-1],' ,val loss of ',val_losses[-1],' ,val f1 of ',val_f1)
 print('-'*80)
 if early_stopping.early_stop:
print("Early stopping at ",epoch," epoch")
 break
else:
early_stopping(1.-val_f1, model)
 print('Training finished.... ' )<load_pretrained>
for dataset in all_data:
dataset['Embarked'] = dataset['Embarked'].fillna('S')
 print(train[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False ).mean())
print(os.listdir())
 model = Ensemble(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size ).cuda()
model.load_state_dict(torch.load('checkpoint.pt'))<init_hyperparams>
for dataset in all_data:
dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 train.head()
print('Threshold:',search_result['threshold'])
 submission_list = list(torchtext.data.BucketIterator(dataset=submission_x,
batch_size=batch_size,
sort=False,
train=False))
pred = []
with torch.no_grad() :
for submission_batch in submission_list:
model.eval()
x = submission_batch.text.cuda()
pred += torch.sigmoid(model(x ).squeeze(1)).cpu().data.numpy().tolist()
pred = np.array(pred)
 df_subm = pd.DataFrame()
df_subm['qid'] = [qid.vocab.itos[j] for i in submission_list for j in i.qid.view(-1 ).numpy() ]
df_subm['prediction'] =(pred > search_result['threshold'] ).astype(int)
 print(df_subm.head())
 df_subm.to_csv('submission.csv', index=False )<define_variables>
train = train.drop(['Fare'], axis=1)
 test = test.drop(['Fare'], axis=1)
 all_data = [train,test]
train.head()
list_train = [filepath for filepath in list_paths if "train/" in filepath]
shuffle(list_train)
 list_test = [filepath for filepath in list_paths if "test/" in filepath]
list_train = list_train
list_test = list_test
index = [os.path.basename(filepath)for filepath in list_test]<define_variables>
for dataset in all_data:
dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.', expand=False)
 pd.crosstab(train['Title'], train['Sex'])
list_classes = list(set([os.path.dirname(filepath ).split(os.sep)[-1] for filepath in list_paths if "train" in filepath]))<define_variables>
for dataset in all_data:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',\
	'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 train[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()
list_classes = ['Sony-NEX-7',
'Motorola-X',
'HTC-1-M7',
'Samsung-Galaxy-Note3',
'Motorola-Droid-Maxx',
'iPhone-4s',
'iPhone-6',
'LG-Nexus-5x',
'Samsung-Galaxy-S4',
'Motorola-Nexus-6']<categorify>
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
for dataset in all_data:
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0)
 train.head()
def get_class_from_path(filepath):
return os.path.dirname(filepath ).split(os.sep)[-1]
def read_and_resize(filepath):
im_array = np.array(Image.open(( filepath)) , dtype="uint8")
 pil_im = Image.fromarray(im_array)
 new_array = np.array(pil_im.resize(( 256, 256)))
 return new_array/255
def label_transform(labels):
labels = pd.get_dummies(pd.Series(labels))
label_index = labels.columns.values
return labels, label_index 
<prepare_x_and_y>
train = train.drop(['Name', 'PassengerId'], axis=1)
 test = test.drop(['Name'], axis=1)
 all_data = [train, test]
X_train = np.array([read_and_resize(filepath)for filepath in list_train])
 X_test = np.array([read_and_resize(filepath)for filepath in list_test] )<categorify>
X_train = train.drop("Survived", axis=1)
 Y_train = train["Survived"]
X_test = test.drop("PassengerId", axis=1 ).copy()
labels = [get_class_from_path(filepath)for filepath in list_train]
y, label_index = label_transform(labels)
 y = np.array(y )<choose_model_class>
random_forest= RandomForestClassifier(n_estimators=100,
max_features='auto',
criterion='entropy',
max_depth=10)
 random_forest.fit(X_train, Y_train)
 Y_prediction = random_forest.predict(X_test)
 random_forest.score(X_train, Y_train)
 acc_random_forest = round(random_forest.score(X_train, Y_train)* 100, 2)
 print(round(acc_random_forest,2,), "%")
input_shape =(256, 256, 3)
 nclass = len(label_index)
 def get_model() :
nclass = len(label_index)
 inp = Input(shape=input_shape)
 norm_inp = BatchNormalization()(inp)
 img_1 = Convolution2D(16, kernel_size=3, activation=activations.relu, padding="same" )(norm_inp)
 img_1 = Convolution2D(16, kernel_size=3, activation=activations.relu, padding="same" )(img_1)
 img_1 = MaxPooling2D(pool_size=(3, 3))(img_1)
 img_1 = Dropout(rate=0.2 )(img_1)
 img_1 = Convolution2D(32, kernel_size=3, activation=activations.relu, padding="same" )(img_1)
 img_1 = Convolution2D(32, kernel_size=3, activation=activations.relu, padding="same" )(img_1)
 img_1 = MaxPooling2D(pool_size=(3, 3))(img_1)
 img_1 = Dropout(rate=0.2 )(img_1)
 img_1 = Convolution2D(64, kernel_size=2, activation=activations.relu, padding="same" )(img_1)
 img_1 = Convolution2D(20, kernel_size=2, activation=activations.relu, padding="same" )(img_1)
 img_1 = GlobalMaxPool2D()(img_1)
 img_1 = Dropout(rate=0.2 )(img_1)
 dense_1 = Dense(20, activation=activations.relu )(img_1)
 dense_1 = Dense(nclass, activation=activations.softmax )(dense_1)
 model = models.Model(inputs=inp, outputs=dense_1)
 opt = optimizers.Adam()
model.compile(optimizer=opt, loss=losses.categorical_crossentropy, metrics=['acc'])
 model.summary()
return model<train_model>
clf = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0,
max_depth=1, random_state=0 ).fit(X_train, Y_train)
 y_prediction= clf.predict(X_test)
 clf.score(X_train, Y_train)
 acc_clf = round(clf.score(X_train, Y_train)* 100, 2)
 print(round(acc_clf,2,), "%")
model = get_model()
file_path="weights.best.hdf5"
checkpoint = ModelCheckpoint(file_path, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
 early = EarlyStopping(monitor="val_acc", mode="max", patience=1)
 callbacks_list = [checkpoint, early]
history = model.fit(X_train, y, validation_split=0.1, epochs=3, shuffle=True, verbose=2,
callbacks=callbacks_list)
 model.load_weights(file_path)
<predict_on_test>
model = LGBMClassifier().fit(X_train, Y_train)
 y_predict= model.predict(X_test)
 model.score(X_train, Y_train)
 acc_model = round(model.score(X_train, Y_train)* 100, 2)
 print(round(acc_model,2,), "%")
predicts = model.predict(X_test)
 predicts = np.argmax(predicts, axis=1)
 predicts = [label_index[p] for p in predicts] 
<save_to_csv>
logreg = LogisticRegression()
logreg.fit(X_train, Y_train)
 Y_pred = logreg.predict(X_test)
 acc_log = round(logreg.score(X_train, Y_train)* 100, 2)
 print(round(acc_log,2,), "%")
df = pd.DataFrame(columns=['fname', 'camera'])
 df['fname'] = index
df['camera'] = predicts
df.to_csv("sub.csv", index=False )<set_options>
decision_tree = DecisionTreeClassifier()
decision_tree.fit(X_train, Y_train)
 Y_pred = decision_tree.predict(X_test)
 acc_decision_tree = round(decision_tree.score(X_train, Y_train)* 100, 2)
 print(round(acc_decision_tree,2,), "%")
warnings.filterwarnings("ignore")
 pd.set_option("display.max_columns", 500)
 pd.set_option("display.max_rows", 500)
 register_matplotlib_converters()
sns.set()<set_options>
params_xgb = {'colsample_bylevel': 0.7, 'learning_rate': 0.03, 'max_depth': 3,
'n_estimators': 400, 'reg_lambda': 15, 'subsample': 0.5}
xgb = XGBClassifier(**params_xgb)
 y_preds = xgb.fit(X_train, Y_train ).predict(X_test)
 acc_xgb = round(xgb.score(X_train, Y_train)* 100, 2)
 print(round(acc_xgb,2,), "%")
def on_kaggle() :
return "KAGGLE_KERNEL_RUN_TYPE" in os.environ<install_modules>
results = pd.DataFrame({
'Model': ['LGBMClassifier', 'Logistic Regression',
'Random Forest', 'Boosting',
'Decision Tree','xgb'],
'Score': [ acc_model,acc_log,
acc_random_forest, acc_clf,
acc_decision_tree,acc_xgb]})
 result_df = results.sort_values(by='Score', ascending=False)
 result_df = result_df.set_index('Score')
 result_df.head(7)
if on_kaggle() :
os.system("pip install --quiet mlflow_extend" )<categorify>
rf = RandomForestClassifier(n_estimators=100)
 scores = cross_val_score(rf, X_train, Y_train, cv=10, scoring = "accuracy")
 print("Scores:", scores)
 print("Mean:", scores.mean())
 print("Standard Deviation:", scores.std())
def reduce_mem_usage(df, verbose=False):
start_mem = df.memory_usage().sum() / 1024 ** 2
int_columns = df.select_dtypes(include=["int"] ).columns
float_columns = df.select_dtypes(include=["float"] ).columns
for col in int_columns:
df[col] = pd.to_numeric(df[col], downcast="integer")
 for col in float_columns:
df[col] = pd.to_numeric(df[col], downcast="float")
 end_mem = df.memory_usage().sum() / 1024 ** 2
if verbose:
print(
 "Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)".format(
 end_mem, 100 *(start_mem - end_mem)/ start_mem 
)
)
 return df<load_from_csv>
params_xgb = {'colsample_bylevel': 0.7, 'learning_rate': 0.03, 'max_depth': 3,
'n_estimators': 400, 'reg_lambda': 15, 'subsample': 0.5}
xgb = XGBClassifier(**params_xgb)
 y_preds = xgb.fit(X_train, Y_train ).predict(X_test)
 print("Score: ",xgb.score, 4*100, "%")
def read_data() :
INPUT_DIR = "/kaggle/input" if on_kaggle() else "input"
INPUT_DIR = f"{INPUT_DIR}/m5-forecasting-accuracy"
print("Reading files...")
 calendar = pd.read_csv(f"{INPUT_DIR}/calendar.csv" ).pipe(reduce_mem_usage)
 prices = pd.read_csv(f"{INPUT_DIR}/sell_prices.csv" ).pipe(reduce_mem_usage)
 sales = pd.read_csv(f"{INPUT_DIR}/sales_train_validation.csv", ).pipe(
 reduce_mem_usage 
)
 submission = pd.read_csv(f"{INPUT_DIR}/sample_submission.csv" ).pipe(
 reduce_mem_usage 
)
 print("sales shape:", sales.shape)
 print("prices shape:", prices.shape)
 print("calendar shape:", calendar.shape)
 print("submission shape:", submission.shape)
 return sales, prices, calendar, submission<define_variables>
submission = pd.DataFrame({
"PassengerId": test['PassengerId'],
"Survived": y_preds
})
 submission.to_csv('submission.csv', index=False)