Kaggle competition: Titanic - Machine Learning from Disaster
filtered_vectors = filter_vectors(gensim_vectors, word_counts.keys() )<drop_column>
train = pd.read_csv('.. /input/titanic/train.csv')
 test_x = pd.read_csv('.. /input/titanic/test.csv')
 sub = pd.read_csv('.. /input/titanic/gender_submission.csv')
 df = pd.concat([train,test_x], sort = False)
 df.head()
del gensim_vectors<count_values>
CabinFill = df[df['Cabin'].notnull() ]
CabinNull = df[df['Cabin'].isnull() ]
CabinFill['Cabin'] = CabinFill['Cabin'].astype(str ).str[0]
df = pd.concat([CabinFill, CabinNull], sort = False ).sort_values(['PassengerId'])
 df['Sex'] = pd.Categorical(df.Sex ).codes
df['Embarked'] = pd.Categorical(df.Embarked ).codes
df['Cabin'] = pd.Categorical(df.Cabin ).codes
df['FamilySize'] = df['Parch'] + df['SibSp'] + 1
df['IsAlone'] = 0
df.loc[df['FamilySize'] == 1, 'IsAlone'] = 1
df.head(10 )
min_occurences = 14
filtered_counts = {w:c for w,c in word_counts.items() if c >= min_occurences}
print(len(filtered_counts))<import_modules>
df.isnull().sum()
class VocabLike:
def __init__(self, itos, stoi):
self.itos = itos
self.stoi = stoi<import_modules>
df['Title'] = df['Name'].map(lambda name: name.split(',')[1].split('.')[0].strip())
 df['Sur'] = df['Name'].map(lambda name: name.split(',')[0])
 df['Title'].value_counts()
from collections import defaultdict<feature_engineering>
titles_dummies = pd.get_dummies(df['Title'], prefix='Title')
 sur_dummies = pd.get_dummies(df['Sur'], prefix='Sur')
 df = pd.concat([df, titles_dummies, sur_dummies], axis=1)
 df.drop(['Title', 'Sur'], axis=1, inplace=True)
 df.head()
specials = ['<unk>', '<pad>', '<eos>']
filtered_counts.update({w:0 for w in specials})
 stoi = defaultdict(lambda:0)
 itos = [0] * len(filtered_counts)
 trainable_words = {w for w in filtered_counts.keys() if w not in specials and w not in filtered_vectors}
pretrained_words = {w for w in filtered_counts.keys() if w not in specials and w in filtered_vectors}
for i,w in enumerate(itertools.chain(specials, trainable_words, pretrained_words)) :
stoi[w] = i
itos[i] = w
vocab = VocabLike(itos, stoi )<feature_engineering>
logloss, accuracy, y_preds = [],[],[]
cv_train = np.zeros(( len(train),))
drop_cols = ['Name','PassengerId','Ticket','Survived','Fare','Cabin','Embarked',
'Age','SibSp','Parch']
train = df[:len(train)]
train_x = train.drop(drop_cols, axis=1)
 train_y = train['Survived']
test = df[len(train):]
test_x = test.drop(drop_cols, axis=1)
 params = {'objective': 'binary', 'seed': 71, 'verbose': 0, 'metrics': 'binary_logloss'}
num_round = 100
categorical_features = []
kf = StratifiedKFold(n_splits=4, shuffle=True, random_state=71)
 for tr_idx, va_idx in kf.split(train_x, train_y):
tr_x, va_x = train_x.iloc[tr_idx], train_x.iloc[va_idx]
tr_y, va_y = train_y.iloc[tr_idx], train_y.iloc[va_idx]
lgb_train = lgb.Dataset(tr_x, tr_y)
 lgb_eval = lgb.Dataset(va_x, va_y)
 model = lgb.train(params, lgb_train, num_boost_round=num_round,
categorical_feature=categorical_features,
valid_names=['train', 'valid'], valid_sets=[lgb_train, lgb_eval],
early_stopping_rounds=20)
 va_pred = model.predict(va_x, num_iteration=model.best_iteration)
 cv_train[va_idx] = va_pred
logloss.append(log_loss(va_y, va_pred))
va_pred =(va_pred > 0.5 ).astype(int)
 accuracy.append(accuracy_score(va_y, va_pred))
y_pred = model.predict(test_x, num_iteration=model.best_iteration)
 y_preds.append(y_pred)
 print(np.mean(logloss))
print(np.mean(accuracy))
def extract_vectors(vocab, vec_dict, offset, total, vec_size):
vectors = np.zeros(( total, vec_size), dtype=np.float32)
 for i in range(total):
word = vocab.itos[i + offset]
assert word in vec_dict
vectors[i] = vec_dict[word]
return vectors<feature_engineering>
y_pred_cv =(cv_train > 0.5 ).astype(int)
 accuracy_score(train_y, y_pred_cv )
np_vectors = extract_vectors(vocab, filtered_vectors,
len(specials)+ len(trainable_words),
len(pretrained_words),
300 )<statistical_test>
sub_y_lgb = sum(y_preds)/ len(y_preds)
 sub_y_lgb =(sub_y_lgb > 0.5 ).astype(int)
 sub['sub_y_lgb'] = sub_y_lgb
def nearest_neighbors(vocab, embeddings, word, topn, use_offset=False):
offset = len(specials)+ len(trainable_words)if use_offset else 0
assert word in vocab.stoi
word_index = vocab.stoi[word] - offset
sims = pairwise.cosine_similarity(embeddings[word_index].reshape(1,-1),embeddings ).ravel()
indices = np.argsort(sims)[::-1]
print(word)
 for i in range(topn):
sim_word = vocab.itos[indices[i] + offset]
print(sim_word, sims[indices[i]] )<compute_train_metric>
accuracy, y_preds_LR = [],[]
cv_train = np.zeros(( len(train),))
drop_cols = ['Name','PassengerId','Ticket','Survived','Fare','Cabin','Embarked',
'Age','SibSp','Parch']
train = df[:len(train)]
train_x = train.drop(drop_cols, axis=1)
 train_y = train['Survived']
test = df[len(train):]
test_x = test.drop(drop_cols, axis=1)
 kf = StratifiedKFold(n_splits=4, shuffle=True, random_state=71)
 for tr_idx, va_idx in kf.split(train_x, train_y):
tr_x, va_x = train_x.iloc[tr_idx], train_x.iloc[va_idx]
tr_y, va_y = train_y.iloc[tr_idx], train_y.iloc[va_idx]
model = LogisticRegression(penalty='l2', solver='sag', random_state=0)
 model.fit(tr_x, tr_y)
 va_pred = model.predict(va_x)
 cv_train[va_idx] = va_pred
va_pred =(va_pred > 0.5 ).astype(int)
 accuracy.append(accuracy_score(va_y, va_pred))
y_pred = model.predict(test_x)
 y_preds_LR.append(y_pred)
 print(np.mean(accuracy))
nearest_neighbors(vocab, np_vectors, 'we'll', 10, True )<set_options>
sub_y_LR = sum(y_preds_LR)/ len(y_preds_LR)
 sub_y_LR =(sub_y_LR > 0.5 ).astype(int)
 sub['sub_y_LR'] = sub_y_LR
gc.collect()<categorify>
drop_cols = ['Name','PassengerId','Ticket','Survived']
train = df[:len(train)]
X = train.drop(drop_cols, axis=1)
 y = train['Survived']
X_train, X_test, y_train, y_test = train_test_split(
 X, y, test_size=0.2, random_state=0)
 model = RandomForestClassifier()
model.fit(X_train, y_train)
 y_pred = model.predict(X_test)
 accuracy = accuracy_score(y_test, y_pred)
 fti = model.feature_importances_
print(accuracy )
def to_word_indices(tokens, vocab, start_index, end_index):
return [[start_index] + [vocab.stoi[w] for w in sent] + [end_index] for sent in tokens]<define_variables>
drop_cols = ['Name','PassengerId','Ticket','Survived']
test = df[len(train):]
test_x = test.drop(drop_cols, axis=1)
 sub_y_RF = model.predict(test_x)
 sub['sub_y_RF'] = sub_y_RF
class TokenToIdDataset(torch.utils.data.Dataset):
def __init__(self, tokens, labels, vocab, max_size=-1, min_size=10, precompute=False, precomputed=False):
self._start_index = vocab.stoi['<sos>']
self._end_index = vocab.stoi['<eos>']
self._pad_index = vocab.stoi['<pad>']
if precompute and not precomputed:
tokens = to_word_indices(tokens, vocab, self._start_index, self._end_index)
 self._precomputed = precompute or precomputed
self._tokens = tokens
self._labels = labels
print(len(self._labels))
self._vocab = vocab
self._len = len(tokens)
 self._max_size = max_size
self._min_size = min_size
self._cache = []
self._batch_size = -1
def __len__(self):
return self._len
def compute_cache(self, batch_size):
start = 0
batch_num = 0
cache = []
while start < len(self):
batch_end = min(start + batch_size, len(self))
cache.append(self.collate([self[j] for j in range(start, batch_end)]))
start = batch_end
self._cache = cache
self._batch_size = batch_size
def get_cache(self, batch_size):
if not self._cache or self._batch_size != batch_size:
self.compute_cache(batch_size)
 return self._cache
def collate(self, samples):
if self._max_size == -1:
size = max(self._min_size, max(len(s[0])for s in samples))
else:
size = self._max_size
labels_tensor = torch.tensor([s[1] for s in samples], dtype=torch.float32)
 texts_tensor = torch.zeros(len(samples), size, dtype=torch.long)
 texts_tensor += self._pad_index
for i, pair in enumerate(samples):
text, _ = pair
text = text if len(text)<= size else text[:size]
texts_tensor[i, size - len(text):] = torch.tensor(text)
 return texts_tensor, labels_tensor
def __getitem__(self, index):
text = self._tokens[index]
if self._precomputed:
indices = text
else:
indices = [self._start_index]
indices.extend(self._vocab.stoi[tok] for tok in text)
 indices.append(self._end_index)
 return indices, self._labels[index] 
<create_dataframe>
drop_cols = ['Name','PassengerId','Ticket','Survived','Fare','Cabin','Embarked',
'Age','SibSp','Parch']
x_NN = df.drop(drop_cols, axis=1)
 X_dummies_train = x_NN.iloc[0:890]
X_dummies_test = x_NN.iloc[891:]
Y = df.iloc[0:890]["Survived"]
min_length=10
train_dataset = TokenToIdDataset(train_tokens,train_labels.values, vocab,min_size=min_length, precompute=True)
 val_dataset = TokenToIdDataset(val_tokens, val_labels.values, vocab,min_size=min_length, precompute=True )<set_options>
def create_neural_net(in_shape, lyrs=[4], act='relu', opt='Adam', dr=0.0):
seed(37556)
 tf.random.set_seed(37556)
 model = Sequential()
model.add(Dense(lyrs[0], input_dim=in_shape, activation=act))
for i in range(1,len(lyrs)) :
model.add(Dense(lyrs[i], activation=act))
model.add(Dropout(dr))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer=opt,
metrics=['accuracy'])
 return model
gc.collect()<import_modules>
single_net = create_neural_net(X_dummies_train.shape[1], lyrs =[4])
 single_net.summary()
import tqdm
from tqdm import tqdm_notebook<load_pretrained>
training = single_net.fit(X_dummies_train, Y, epochs=100, batch_size=32,
validation_split=0.25, verbose=0)
 val_acc = np.mean(training.history['val_accuracy'])
 print("
%s: %.2f%%" %('val_acc', val_acc*100))
class BestModel:
def __init__(self, model_path, optimizer_path, best_loss=10000):
self.best_loss = best_loss
self.model_path = model_path
self.optimizer_path = optimizer_path
def update(self, loss, model, optimizer=None):
self.best_loss = loss
torch.save(model.state_dict() , self.model_path)
 if optimizer:
torch.save(optimizer.state_dict() , self.optimizer_path)
 def load(self, model, optimizer=None):
model.load_state_dict(torch.load(self.model_path))
if optimizer:
optimizer.load_state_dict(torch.load(self.optimizer_path))
model.eval()
def copy(self, new_model_path, new_optimizer_path):
copyfile(self.model_path, new_model_path)
 copyfile(self.optimizer_path, new_optimizer_path)
 return BestModel(new_model_path, new_optimizer_path, self.best_loss )<train_model>
nn = KerasClassifier(build_fn=create_neural_net,
in_shape = X_dummies_train.shape[1],
lyrs=[12, 8, 4], epochs=50, dr=0.1, batch_size=1,
verbose=0)
 nn.fit(X_dummies_train, Y)
 sub['sub_y_NN'] = nn.predict(X_dummies_test ).astype('int32' )
<categorify><EOS> 
model_cols = ['sub_y_lgb','sub_y_RF','sub_y_NN','sub_y_LR']
sub['Survived'] = np.sum(sub[model_cols], axis=1)
 sub['Survived'] =(sub['Survived'] >= 3 ).astype(int)
 sub.drop(model_cols, axis=1, inplace=True)
 sub.to_csv('sub_title_ensembled.csv', index=False)
 sub.head()