Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<choose_model_class>
%matplotlib inline
warnings.filterwarnings('ignore' )
filter_sizes = [1,2,3,5]
num_filters = 36
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Reshape(( maxlen, embed_size, 1))(x)
 maxpool_pool = []
for i in range(len(filter_sizes)) :
conv = Conv2D(num_filters, kernel_size=(filter_sizes[i], embed_size),
kernel_initializer='he_normal', activation='elu' )(x)
 maxpool_pool.append(MaxPool2D(pool_size=(maxlen - filter_sizes[i] + 1, 1))(conv))
z = Concatenate(axis=1 )(maxpool_pool)
 z = Flatten()(z)
 z = Dropout(0.1 )(z)
 outp = Dense(1, activation="sigmoid" )(z)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )<train_model>
test = pd.read_csv(".. /input/titanic/test.csv")
 train = pd.read_csv(".. /input/titanic/train.csv" )
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
print('Train')
 print(train.isnull().sum())
 print('==========================')
 print('Test')
 print(test.isnull().sum() )
pred_cnn_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_cnn_val_y>thresh ).astype(int))))<predict_on_test>
train = train.fillna({'Age': -0.1})
 test = test.fillna({'Age': -0.1})
 train['Sex'] = LabelEncoder().fit_transform(train['Sex'])
 test['Sex'] = LabelEncoder().fit_transform(test['Sex'])
 train.loc[~train.Cabin.isnull() , 'Cabin'] = 1
train.loc[train.Cabin.isnull() , 'Cabin'] = 0
test.loc[~test.Cabin.isnull() , 'Cabin'] = 1
test.loc[test.Cabin.isnull() , 'Cabin'] = 0
common_embarked = train.groupby(['Embarked'])['Embarked'].value_counts().idxmax() [0]
train = train.fillna({'Embarked': common_embarked})
 test = test.fillna({'Embarked': common_embarked})
 test.loc[test.Fare.isnull() , 'Fare'] = 0
pred_cnn_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
train['Title'] = train.Name.str.split(',', n=1, expand=True)[1].str.split('.',n=1, expand=True)[0]
train['Title'] = train.Title.str.strip()
test['Title'] = test.Name.str.split(',', n=1, expand=True)[1].str.split('.', n=1, expand=True)[0]
test['Title'] = test.Title.str.strip()
train.head()
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<set_options>
train.loc[train.Title == 'Ms', 'Title'] = 'Miss'
test.loc[test.Title == 'Ms', 'Title'] = 'Miss'
train.loc[~train.Title.isin(['Mr', 'Miss', 'Mrs', 'Master']), 'Title'] = 'Other'
test.loc[~test.Title.isin(['Mr', 'Miss', 'Mrs', 'Master']), 'Title'] = 'Other'
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<statistical_test>
train['TicketPrefix'] = train.Ticket.str.split(' ' ).apply(lambda x: x[0] if len(x)> 1 else 'No')
 test['TicketPrefix'] = test.Ticket.str.split(' ' ).apply(lambda x: x[0] if len(x)> 1 else 'No')
 train.head()
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'] )<train_model>
train.groupby(['TicketPrefix'])['TicketPrefix'].count()
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))<predict_on_test>
train.loc[train.TicketPrefix.str.startswith('A'), 'TicketPrefix'] = 'A'
train.loc[train.TicketPrefix.str.startswith('C'), 'TicketPrefix'] = 'C'
train.loc[train.TicketPrefix.str.startswith('F'), 'TicketPrefix'] = 'F'
train.loc[train.TicketPrefix.str.startswith('P'), 'TicketPrefix'] = 'P'
train.loc[train.TicketPrefix.str.startswith('S'), 'TicketPrefix'] = 'S'
train.loc[train.TicketPrefix.str.startswith('W'), 'TicketPrefix'] = 'W'
test.loc[test.TicketPrefix.str.startswith('A'), 'TicketPrefix'] = 'A'
test.loc[test.TicketPrefix.str.startswith('C'), 'TicketPrefix'] = 'C'
test.loc[test.TicketPrefix.str.startswith('F'), 'TicketPrefix'] = 'F'
test.loc[test.TicketPrefix.str.startswith('P'), 'TicketPrefix'] = 'P'
test.loc[test.TicketPrefix.str.startswith('S'), 'TicketPrefix'] = 'S'
test.loc[test.TicketPrefix.str.startswith('W'), 'TicketPrefix'] = 'W'
train.groupby(['TicketPrefix'])['TicketPrefix'].count()
pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))))<predict_on_test>
train['Alone'] =(( train.Parch + train.SibSp)== 0 ).astype(int)
 test['Alone'] =(( test.Parch + test.SibSp)== 0 ).astype(int)
 train.head()
pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
train = train.drop(['Name', 'SibSp', 'Parch', 'Embarked'], axis=1)
 test = test.drop(['Name', 'SibSp', 'Parch', 'Embarked'], axis=1)
 train.head()
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<statistical_test>
def encode_ticket(t):
e = {
'No': 0,
'A': 1,
'P': 2,
'S': 3,
'C': 4,
'W': 5,
'F': 6
}
return e.get(t, -1)
 train['Ticket'] = train.TicketPrefix.apply(encode_ticket)
 test['Ticket'] = test.TicketPrefix.apply(encode_ticket)
 train.head()
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'] )<train_model>
train.Title = LabelEncoder().fit_transform(train.Title)
 test.Title = LabelEncoder().fit_transform(test.Title)
 train.head()
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))<predict_on_test>
train.drop(['TicketPrefix'], axis=1, inplace=True)
 test.drop(['TicketPrefix'], axis=1, inplace=True)
 train.head()
pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_fasttext_val_y>thresh ).astype(int))))<predict_on_test>
data = pd.concat([train, test])
 data.drop(['Survived'], axis=1, inplace=True)
 data.head()
pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
data.drop(['PassengerId'], axis=1, inplace=True )
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<statistical_test>
predictors = data[data.Age > 0]
predictors.drop(['Age'], axis=1, inplace=True)
 targets = np.array(data[data.Age > 0].Age)
 predictors.shape, targets.shape
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'] )<train_model>
predictors = StandardScaler().fit_transform(predictors )
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))<predict_on_test>
mlp = MLPRegressor(hidden_layer_sizes=(150, 100))
mlp.fit(predictors, targets)
 y_pred = mlp.predict(predictors)
 mean_squared_error(y_pred, targets), mlp
pred_paragram_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_paragram_val_y>thresh ).astype(int))))<predict_on_test>
train.loc[train.Age < 0, 'Age'] = mlp.predict(StandardScaler().fit_transform(train[train['Age'] < 0][['Alone', 'Cabin', 'Fare', 'Pclass', 'Sex','Ticket', 'Title']]))
pred_paragram_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
test.loc[test.Age < 0, 'Age'] = mlp.predict(StandardScaler().fit_transform(test[test['Age'] < 0][['Alone', 'Cabin', 'Fare', 'Pclass', 'Sex','Ticket', 'Title']]))
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<find_best_params>
test.loc[test.Age < 0, 'Age'] = 0.1
train.loc[train.Age < 0, 'Age'] = 0.1
train.head()
pred_val_y =(4 * pred_glove_val_y + pred_fasttext_val_y + 3 * pred_paragram_val_y + 2 * pred_cnn_val_y)/ 10.0
thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(val_y,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh )<save_to_csv>
predictors = train.drop(['PassengerId', 'Survived'], axis=1)
 targets = train[['Survived']]
predictors = StandardScaler().fit_transform(predictors )
pred_test_y =(4 * pred_glove_test_y + pred_fasttext_test_y + 3 * pred_paragram_test_y + 2 * pred_cnn_test_y)/ 10.0
pred_test_y =(pred_test_y > best_thresh ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<import_modules>
x_train, x_test, y_train, y_test = train_test_split(predictors, targets, test_size = 0.05, random_state = 0 )
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Add, Concatenate, Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D, Convolution2D, Reshape
from keras.layers import Bidirectional, GlobalMaxPool1D, MaxPool1D, GlobalMaxPool2D, Flatten, ZeroPadding1D
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers<load_from_csv>
mlp = MLPRegressor(batch_size=50, hidden_layer_sizes=(140))
mlp.fit(x_train, y_train)
 y_pred = mlp.predict(x_train ).round()
test_pred = mlp.predict(x_test ).round()
score = accuracy_score(y_train, y_pred)
 test_score = accuracy_score(y_test, test_pred)
 mlp, score, test_score
<split><EOS> 
ids = test['PassengerId']
predictions = np.abs(mlp.predict(StandardScaler().fit_transform(test.drop(['PassengerId'], axis=1)) ).round() ).astype(int)
 output = pd.DataFrame({ 'PassengerId' : ids, 'Survived': predictions })
 output.to_csv('submission.csv', index=False )