Kaggle competition: Titanic - Machine Learning from Disaster
embeddings_index = {}
f = open('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt')
 for line in tqdm(f):
values = line.split(" ")
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
f.close()
print('Found %s word vectors.' % len(embeddings_index))<define_variables>
gender_submission = pd.read_csv(".. /input/titanic/gender_submission.csv")
 test = pd.read_csv(".. /input/titanic/test.csv")
 train = pd.read_csv(".. /input/titanic/train.csv" )
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]<feature_engineering>
test2 = pd.read_csv(".. /input/titanic/test.csv" )
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<set_options>
warnings.simplefilter('ignore')
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
print('Train columns with null values:
', train.isnull().sum())
 print("-" * 10)
 print('Test columns with null values:
', test.isnull().sum())
 print("-" * 10 )
model=Sequential()
model.add(Embedding(max_features, embed_size, weights=[embedding_matrix],input_length=maxlen,trainable = False))
model.add(Bidirectional(CuDNNLSTM(128, return_sequences=True)))
 model.add(Bidirectional(CuDNNLSTM(64, return_sequences=True)))
 model.add(Attention(maxlen))
model.add(Dense(64, activation="relu"))
model.add(Dropout(0.2))
model.add(Dense(1, activation="sigmoid"))
model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])
 print(model.summary() )<train_model>
dropping = ['PassengerId', 'Ticket','Cabin','Fare']
train.drop(dropping,axis=1, inplace=True)
 test.drop(dropping,axis=1, inplace=True)
 train.head()
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))<compute_test_metric>
data = [train,test]
for dataset in data:
dataset['Embarked'].fillna(dataset['Embarked'].mode() [0],inplace=True)
 Embarked = np.zeros(len(dataset))
Embarked[dataset['Embarked']== 'C'] = 1
Embarked[dataset['Embarked']== 'Q'] = 2
Embarked[dataset['Embarked']== 'S'] = 3
dataset['Embarked'] = Embarked
model.evaluate(x=val_X, y=val_y, batch_size=1024, verbose=1 )<predict_on_test>
train['Age'].fillna(train['Age'].median() , inplace = True)
 test['Age'].fillna(test['Age'].median() , inplace = True )
test_y_prediction = model.predict(test_X, batch_size = 1024, verbose = 1 )<predict_on_test>
print('Train columns with null values:
', train.isnull().sum())
 print("-" * 10)
 print('Test columns with null values:
', test.isnull().sum())
 print("-" * 10 )
pred_val_y = model.predict([val_X], batch_size=1024, verbose=1 )<compute_test_metric>
data = [train,test]
for dataset in data:
dataset['Age'] = dataset['Age']
dataset.loc[ dataset['Age'] <= 11, 'Age'] = 0
dataset.loc[(dataset['Age'] > 11)&(dataset['Age'] <= 18), 'Age'] = 1
dataset.loc[(dataset['Age'] > 18)&(dataset['Age'] <= 22), 'Age'] = 2
dataset.loc[(dataset['Age'] > 22)&(dataset['Age'] <= 27), 'Age'] = 3
dataset.loc[(dataset['Age'] > 27)&(dataset['Age'] <= 33), 'Age'] = 4
dataset.loc[(dataset['Age'] > 33)&(dataset['Age'] <= 40), 'Age'] = 5
dataset.loc[(dataset['Age'] > 40)&(dataset['Age'] <= 66), 'Age'] = 6
dataset.loc[ dataset['Age'] > 66, 'Age'] = 6
train['Age'].value_counts()
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_val_y>thresh ).astype(int))))<save_to_csv>
data = [train,test]
titles = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
for dataset in data:
dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.', expand=False)
 dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr',\
'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 dataset['Title'] = dataset['Title'].map(titles)
 dataset['Title'] = dataset['Title'].fillna(0)
 train = train.drop(['Name'], axis=1)
 test = test.drop(['Name'], axis=1 )
pred_test_y =(test_y_prediction>0.40 ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<import_modules>
data = [train,test]
for dataset in data:
sex = np.zeros(len(dataset))
sex[dataset['Sex']== 'male'] = 1
sex[dataset['Sex']== 'female'] = 0
dataset['Sex'] = sex
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers<load_from_csv>
data = [train,test]
for dataset in data:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )<split>
data = [train,test]
for dataset in data:
dataset['IsAlone'] = 0
dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1
train_df, val_df = train_test_split(train_df, test_size=0.08, random_state=2018)
 embed_size = 300
max_features = 95000
maxlen = 70
train_X = train_df["question_text"].fillna("_
val_X = val_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values<randomize_order>
train = train.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
 test = test.drop(['Parch', 'SibSp', 'FamilySize'], axis=1 )
np.random.seed(2018)
 trn_idx = np.random.permutation(len(train_X))
val_idx = np.random.permutation(len(val_X))
train_X = train_X[trn_idx]
val_X = val_X[val_idx]
train_y = train_y[trn_idx]
val_y = val_y[val_idx]<train_model>
train_y=train['Survived']
train_ft=train.drop('Survived',axis=1)
 kf = StratifiedKFold(n_splits=10)
 print(train_ft.head())
 print(train_y.head() )
<set_options>
svc = SVC(C = 45, gamma = 0.03)
 svc.fit(train_ft, train_y)
 acc_SVM = cross_val_score(svc,train_ft,train_y,cv=kf)
 print(acc_SVM.mean() )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<statistical_test>
predictions = svc.predict(test)
 print(predictions )
<train_model><EOS> 
submission = pd.DataFrame({ 'PassengerId': test2['PassengerId'],
'Survived': predictions })
 submission.to_csv('submission.csv', index = False )