Kaggle competition: Titanic - Machine Learning from Disaster
def eval_model(model, data_iter, device, order_index=None):
model.eval()
predictions = []
with torch.no_grad() :
for batch_data in data_iter:
qid_batch, src_sents, src_seqs, src_lens, tgts = batch_data
src_seqs = src_seqs.to(device)
 out = model(src_seqs, src_lens, return_logits=False)
 predictions.append(out)
 predictions = torch.cat(predictions, dim=0)
 if order_index is not None:
predictions = predictions[order_index]
predictions = predictions.to('cpu' ).numpy().ravel()
return predictions 
<compute_train_metric>
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 train_data.head()
def cv(train_df, test_df, device=None, n_folds=10, shared_resources=None, share=True, **kwargs):
if device is None:
device = torch.device("cuda:{}".format(0)if torch.cuda.is_available() else "cpu")
 max_vocab_size = kwargs['max_vocab_size']
embed_size = kwargs['embed_size']
threshold = kwargs['threshold']
max_seq_len = kwargs['max_seq_len']
if shared_resources is None:
shared_resources = {}
if share:
if 'vocab' not in shared_resources:
counter = build_counter(chain(train_df['question_text'], test_df['question_text']))
vocab = build_vocab(counter, max_vocab_size=max_vocab_size)
 shared_resources['vocab'] = vocab
seqs = []
for sent in tqdm(train_df['question_text'], desc='tokenize'):
seq = tokens2ids(sent.split() [:max_seq_len], vocab['token2id'])
 seqs.append(seq)
 train_df['src_seqs'] = seqs
seqs = []
for sent in tqdm(test_df['question_text'], desc='tokenize'):
seq = tokens2ids(sent.split() [:max_seq_len], vocab['token2id'])
 seqs.append(seq)
 test_df['src_seqs'] = seqs
if 'embedding_matrix' not in shared_resources:
embedding_matrix = load_embedding(shared_resources['vocab'], max_vocab_size, embed_size)
 shared_resources['embedding_matrix'] = embedding_matrix
splits = list(
 StratifiedKFold(n_splits=n_folds, shuffle=True ).split(train_df['target'], train_df['target']))
scores = []
best_threshold = []
best_threshold_global = None
best_score = -1
predictions_train_reduced = []
targets_train = []
predictions_tes_reduced = np.zeros(( len(test_df), n_folds))
predictions_te = np.zeros(( len(test_df),))
for idx,(train_idx, valid_idx)in enumerate(splits):
grow_df = train_df.iloc[train_idx].reset_index(drop=True)
 dev_df = train_df.iloc[valid_idx].reset_index(drop=True)
 predictions_te_i, predictions_va, targets_va, best_threshold_i = main(grow_df, dev_df, test_df, device,
**kwargs,
idx=idx,
shared_resources=shared_resources,
return_reduced=True)
 predictions_tes_reduced[:, idx] = predictions_te_i
scores.append([f1_score(targets_va, predictions_va > threshold), roc_auc_score(targets_va, predictions_va)])
 best_threshold.append(best_threshold_i)
 predictions_te += predictions_te_i / n_folds
predictions_train_reduced.append(predictions_va)
 targets_train.append(targets_va)
 coeff =(np.corrcoef(predictions_tes_reduced, rowvar=False ).sum() - n_folds)/ n_folds /(n_folds - 1)
 predictions_train_reduced = np.concatenate(predictions_train_reduced)
 targets_train = np.concatenate(targets_train)
 for t in np.arange(0, 1, 0.01):
score = f1_score(targets_train, predictions_train_reduced > t)
 if score > best_score:
best_score = score
best_threshold_global = t
print('avg of best threshold {} macro-f1 best threshold {} best score {}'.format(best_threshold,
best_threshold_global, best_score))
return predictions_te, predictions_te, scores, best_threshold_global, coeff 
<split>
test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 test_data.head()
print(test_data.isnull().sum() )
def main(train_df, valid_df, test_df, device=None, epochs=3, fine_tuning_epochs=3, batch_size=512, learning_rate=0.001,
learning_rate_max_offset=0.001, dropout=0.1,
threshold=None,
max_vocab_size=95000, embed_size=300, max_seq_len=70, print_every_step=500, idx=0, shared_resources=None,
return_reduced=True):
if device is None:
device = torch.device("cuda:{}".format(0)if torch.cuda.is_available() else "cpu")
 if shared_resources is None:
shared_resources = {}
batch_time = AverageMeter()
data_time = AverageMeter()
mean_len = AverageMeter()
if 'vocab' not in shared_resources:
counter = build_counter(chain(train_df['question_text'], test_df['question_text']))
vocab = build_vocab(counter, max_vocab_size=max_vocab_size)
 else:
vocab = shared_resources['vocab']
if 'embedding_matrix' not in shared_resources:
embedding_matrix = load_embedding(vocab, max_vocab_size, embed_size)
 else:
embedding_matrix = shared_resources['embedding_matrix']
test_dataset = TextDataset(test_df, vocab=vocab, max_seq_len=max_seq_len)
 tb = BucketSampler(test_dataset, test_dataset.get_keys() , batch_size=batch_size,
shuffle_data=False)
 test_iter = DataLoader(dataset=test_dataset,
batch_size=batch_size,
sampler=tb,
num_workers=0,
collate_fn=collate_fn)
 train_dataset = TextDataset(train_df, vocab=vocab, max_seq_len=max_seq_len)
 valid_dataset = TextDataset(valid_df, vocab=vocab, max_seq_len=max_seq_len)
 vb = BucketSampler(valid_dataset, valid_dataset.get_keys() , batch_size=batch_size,
shuffle_data=False)
 valid_index_reverse = vb.get_reverse_indexes()
model = InsincereModel(device, hidden_dim=256, hidden_dim_fc=16, dropout=dropout,
embedding_matrixs=embedding_matrix,
vocab_size=len(vocab['token2id']),
embedding_dim=embed_size, max_seq_len=max_seq_len)
 if idx == 0:
print(model)
 print('total trainable {}'.format(count_parameters(model)))
 model = model.to(device)
 optimizer = optim.Adam([p for p in model.parameters() if p.requires_grad], lr=learning_rate)
 train_iter = DataLoader(dataset=train_dataset,
batch_size=batch_size,
sampler=BucketSampler(train_dataset, train_dataset.get_keys() , bucket_size=batch_size * 20,
batch_size=batch_size),
num_workers=0,
collate_fn=collate_fn)
 valid_iter = DataLoader(dataset=valid_dataset,
batch_size=batch_size,
sampler=vb,
collate_fn=collate_fn)
 loss_list = []
global_steps = 0
total_steps = epochs * len(train_iter)
 loss_fn = torch.nn.BCEWithLogitsLoss()
end = time.time()
predictions_tes = []
predictions_vas = []
n_fge = 0
clr = CyclicLR(optimizer, base_lr=learning_rate, max_lr=learning_rate + learning_rate_max_offset,
step_size=300, mode='exp_range')
 clr.on_train_begin()
fine_tuning_epochs = epochs - fine_tuning_epochs
predictions_te = None
for epoch in tqdm(range(epochs)) :
fine_tuning = epoch >= fine_tuning_epochs
start_fine_tuning = fine_tuning_epochs == epoch
if start_fine_tuning:
model.enable_learning_embedding()
optimizer = optim.Adam([p for p in model.parameters() if p.requires_grad], lr=learning_rate)
 global_steps = 0
total_steps =(epochs - fine_tuning_epochs)* len(train_iter)
 clr = CyclicLR(optimizer, base_lr=learning_rate, max_lr=learning_rate + learning_rate_max_offset,
step_size=int(len(train_iter)/ 8))
clr.on_train_begin()
predictions_te = np.zeros(( len(test_df),))
predictions_va = np.zeros(( len(valid_dataset.targets),))
for batch_data in train_iter:
data_time.update(time.time() - end)
 qids, src_sents, src_seqs, src_lens, tgts = batch_data
mean_len.update(sum(src_lens))
src_seqs = src_seqs.to(device)
 tgts = tgts.to(device)
 model.train()
optimizer.zero_grad()
out = model(src_seqs, src_lens, return_logits=True ).view(-1)
 loss = loss_fn(out, tgts)
 loss.backward()
optimizer.step()
loss_list.append(loss.detach().to('cpu' ).item())
 global_steps += 1
batch_time.update(time.time() - end)
 end = time.time()
if global_steps % print_every_step == 0:
curr_gpu_memory_usage = get_gpu_memory_usage(device_id=torch.cuda.current_device())
 print('Global step: {}/{} Total loss: {:.4f} Current GPU memory '
'usage: {} maxlen {} '.format(global_steps, total_steps, avg(loss_list), curr_gpu_memory_usage,
mean_len.avg))
loss_list = []
if fine_tuning and global_steps %(2 * clr.step_size)== 0:
predictions_te_tmp2 = eval_model(model, test_iter, device)
 predictions_va_tmp2 = eval_model(model, valid_iter, device, valid_index_reverse)
 report_perf(valid_dataset, predictions_va_tmp2, threshold, idx, epoch,
desc='val set mean')
 predictions_te = predictions_te * n_fge +(
 predictions_te_tmp2)
 predictions_va = predictions_va * n_fge +(
 predictions_va_tmp2)
 predictions_te /= n_fge + 1
predictions_va /= n_fge + 1
report_perf(valid_dataset, predictions_va, threshold, idx, epoch
, desc='val set(fge)')
 predictions_tes.append(predictions_te_tmp2.reshape([-1, 1]))
predictions_vas.append(predictions_va_tmp2.reshape([-1, 1]))
n_fge += 1
clr.on_batch_end()
if not fine_tuning:
predictions_va = eval_model(model, valid_iter, device, valid_index_reverse)
 report_perf(valid_dataset, predictions_va, threshold, idx, epoch)
 if predictions_te is not None:
predictions_te = predictions_te[tb.get_reverse_indexes() ]
else:
predictions_te = eval_model(model, test_iter, device, tb.get_reverse_indexes())
 best_score = -1
best_threshold = None
for t in np.arange(0, 1, 0.01):
score = f1_score(valid_dataset.targets, predictions_va > t)
 if score > best_score:
best_score = score
best_threshold = t
print('best threshold on validation set: {:.2f} score {:.4f}'.format(best_threshold, best_score))
if not return_reduced and len(predictions_vas)> 0:
predictions_te = np.concatenate(predictions_tes, axis=1)
 predictions_te = predictions_te[tb.get_reverse_indexes() , :]
predictions_va = np.concatenate(predictions_vas, axis=1)
 return predictions_te, predictions_va, valid_dataset.targets, best_threshold 
<load_from_csv>
y = train_data["Survived"]
features = ["Pclass", "Sex", "SibSp", "Parch", "Fare", "Age"]
X = pd.get_dummies(train_data[features])
 X_unknown = pd.get_dummies(test_data[features])
 my_imputer = SimpleImputer()
X = my_imputer.fit_transform(X)
 X_unknown = my_imputer.fit_transform(X_unknown)
 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=10 )
set_seed(233)
 epochs = 8
batch_size = 512
learning_rate = 0.001
learning_rate_max_offset = 0.002
fine_tuning_epochs = 2
threshold = 0.31
max_vocab_size = 120000
embed_size = 300
print_every_step = 500
max_seq_len = 70
share = True
dropout = 0.1
sub = pd.read_csv('.. /input/sample_submission.csv')
 train_df, test_df = load_data()
trn_idx = np.random.permutation(len(train_df))
train_df = train_df.iloc[trn_idx].reset_index(drop=True)
 n_folds = 5
n_repeats = 1
args = {'epochs': epochs, 'batch_size': batch_size, 'learning_rate': learning_rate, 'threshold': threshold,
'max_vocab_size': max_vocab_size,
'embed_size': embed_size, 'print_every_step': print_every_step, 'dropout': dropout,
'learning_rate_max_offset': learning_rate_max_offset,
'fine_tuning_epochs': fine_tuning_epochs, 'max_seq_len': max_seq_len}
predictions_te_all = np.zeros(( len(test_df),))
for _ in range(n_repeats):
if n_folds > 1:
_, predictions_te, _, threshold, coeffs = cv(train_df, test_df, n_folds=n_folds, share=share, **args)
 print('coeff between predictions {}'.format(coeffs))
else:
predictions_te, _, _, _ = main(train_df, test_df, test_df, **args)
 predictions_te_all += predictions_te / n_repeats
sub.prediction = predictions_te_all > threshold
sub.to_csv("submission.csv", index=False )<init_hyperparams>
optimal_alpha = 1
optimal_accuracy = 0
for i in range(20):
model = MLPClassifier(hidden_layer_sizes = [50, 50], alpha = 0.1*(i+1), activation='relu', solver='adam', random_state=1 ).fit(X_train, y_train)
 model_accuracy = model.score(X_test, y_test)
 if model_accuracy > optimal_accuracy:
optimal_accuracy = model_accuracy
optimal_alpha = 0.1*(i+1)
 print('The optimal alpha found was {}'.format(optimal_alpha))
print('The test accuracy was {}
'.format(optimal_accuracy))
embed_size = 600
max_features = None
maxlen = 57<import_modules>
optimal_estimators = 1
optimal_accuracy = 0
for i in range(20):
model = RandomForestClassifier(n_estimators=(i+1)*10, max_depth=5, random_state=1 ).fit(X_train, y_train)
 model_accuracy = model.score(X_test, y_test)
 if model_accuracy > optimal_accuracy:
optimal_accuracy = model_accuracy
optimal_estimators =(i+1)*10
print('The optimal number of estimators found was {}'.format(optimal_estimators))
print('The test accuracy was {}
'.format(optimal_accuracy))
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate, Lambda
from keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D, BatchNormalization
from keras.optimizers import Adam, Nadam
from keras.models import Model
from keras import backend as K
from keras.callbacks import Callback
from keras import initializers, regularizers, constraints, optimizers, layers
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from keras.engine.topology import Layer
import gc, re
from sklearn import metrics
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from sklearn.metrics import f1_score, roc_auc_score, confusion_matrix, auc, precision_recall_curve<define_variables>
model = RandomForestClassifier(n_estimators=110, max_depth=5, random_state=1 ).fit(X, y)
 predictions = model.predict(X_unknown)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False )