Kaggle competition: Titanic - Machine Learning from Disaster
from collections import defaultdict
from glob import glob
from random import choice, sample
import cv2
import numpy as np
import pandas as pd
from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau,EarlyStopping
from keras.layers import Input, Dense, Flatten, GlobalMaxPool2D, GlobalAvgPool2D, Concatenate, Multiply, Dropout, Subtract, Add, Conv2D, Flatten
from keras.models import Model
from keras.layers import BatchNormalization
from keras.preprocessing import image
from keras.optimizers import Adam
import h5py
from keras.layers import LeakyReLU
from keras import regularizers
import gc
import psutil<define_variables>
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
train_file_path = ".. /input/train_relationships.csv"
train_folders_path = ".. /input/train/"
val_famillies = "F09"<define_variables>
df_train=pd.read_csv('.. /input/titanic/train.csv')
 df_test=pd.read_csv('.. /input/titanic/test.csv' )
all_images = glob(train_folders_path + "*/*/*.jpg")
 train_images = [x for x in all_images if val_famillies not in x]
val_images = [x for x in all_images if val_famillies in x]<define_variables>
PassengerId=df_test['PassengerId']
train_person_to_images_map = defaultdict(list)
 ppl = [x.split("/")[-3] + "/" + x.split("/")[-2] for x in all_images]
for x in train_images:
train_person_to_images_map[x.split("/")[-3] + "/" + x.split("/")[-2]].append(x)
 val_person_to_images_map = defaultdict(list)
 for x in val_images:
val_person_to_images_map[x.split("/")[-3] + "/" + x.split("/")[-2]].append(x )<load_from_csv>
df_train.drop(['PassengerId','Name','Ticket'],axis=1,inplace=True)
 df_test.drop(['PassengerId','Name','Ticket'],axis=1,inplace=True )
relationships = pd.read_csv(train_file_path)
 relationships = list(zip(relationships.p1.values, relationships.p2.values))
relationships = [x for x in relationships if x[0] in ppl and x[1] in ppl]<define_variables>
df_train.isnull().sum() /len(df_train)*100
train = [x for x in relationships if val_famillies not in x[0]]
val = [x for x in relationships if val_famillies in x[0]]<data_type_conversions>
df_train.drop(['Cabin'],axis=1,inplace=True)
 df_test.drop(['Cabin'],axis=1,inplace=True )
def read_img(path):
img = image.load_img(path, target_size=(197, 197))
img = np.array(img ).astype(np.float)
 return preprocess_input(img, version=2 )<define_variables>
df_train.dropna(subset=['Embarked'],inplace=True)
 df_train['Embarked'].isnull().sum()
def gen(list_tuples, person_to_images_map, batch_size=16):
ppl = list(person_to_images_map.keys())
 while True:
batch_tuples = sample(list_tuples, batch_size // 2)
 labels = [1] * len(batch_tuples)
 while len(batch_tuples)< batch_size:
p1 = choice(ppl)
 p2 = choice(ppl)
 if p1 != p2 and(p1, p2)not in list_tuples and(p2, p1)not in list_tuples:
batch_tuples.append(( p1, p2))
labels.append(0)
 for x in batch_tuples:
if not len(person_to_images_map[x[0]]):
print(x[0])
 X1 = [choice(person_to_images_map[x[0]])for x in batch_tuples]
X1 = np.array([read_img(x)for x in X1])
 X2 = [choice(person_to_images_map[x[1]])for x in batch_tuples]
X2 = np.array([read_img(x)for x in X2])
 yield [X1, X2], labels 
<define_search_model>
age_train_series=df_train.groupby(['Pclass','Sex'])['Age'].transform('median' )
def baseline_model() :
input_1 = Input(shape=(197, 197, 3))
input_2 = Input(shape=(197, 197, 3))
base_model = VGGFace(model='resnet50', include_top=False)
 for x in base_model.layers[:-3]:
x.trainable = True
x1 = base_model(input_1)
 x2 = base_model(input_2)
 x1 = Concatenate(axis=-1 )([GlobalAvgPool2D()(x1), GlobalAvgPool2D()(x1)])
 x2 = Concatenate(axis=-1 )([GlobalAvgPool2D()(x2), GlobalAvgPool2D()(x2)])
 x3 = Subtract()([x1, x2])
 x3 = Multiply()([x3, x3])
 x1_ = Multiply()([x1, x1])
 x2_ = Multiply()([x2, x2])
 x4 = Subtract()([x1_, x2_])
 x = Concatenate(axis=-1 )([x4, x3])
 x = Dense(256, activation="relu",kernel_regularizer=regularizers.l2(0.01))(x)
 x = BatchNormalization()(x)
 x = Dense(256, activation="relu",kernel_regularizer=regularizers.l2(0.01))(x)
 x = Dropout(0.01 )(x)
 x = Dense(128, activation="relu",kernel_regularizer=regularizers.l2(0.01))(x)
 x = BatchNormalization()(x)
 x = Dense(128, activation="relu",kernel_regularizer=regularizers.l2(0.01))(x)
 x = Dropout(0.01 )(x)
 x = Dense(100, activation="relu",kernel_regularizer=regularizers.l2(0.01))(x)
 out = Dense(1, activation="sigmoid" )(x)
 model = Model([input_1, input_2], out)
 model.compile(loss="binary_crossentropy", metrics=['acc'], optimizer=Adam(0.00001))
model.summary()
return model 
<install_modules>
age_test_series=df_test.groupby(['Pclass','Sex'])['Age'].transform('median' )
!pip install git+https://github.com/rcmalli/keras-vggface.git 
<set_options>
df_train['Age']=df_train['Age'].fillna(age_train_series )
print("available RAM:", psutil.virtual_memory())
 gc.collect()
print("available RAM:", psutil.virtual_memory() )<define_variables>
df_test['Age']=df_test['Age'].fillna(age_test_series )
test_path = ".. /input/test/"<save_to_csv>
df_test.isnull().sum()
def chunker(seq, size=32):
return(seq[pos:pos + size] for pos in range(0, len(seq), size))
submission = pd.read_csv('.. /input/sample_submission.csv')
 predictions = []
for batch in tqdm(chunker(submission.img_pair.values)) :
X1 = [x.split("-")[0] for x in batch]
X1 = np.array([read_img(test_path + x)for x in X1])
 X2 = [x.split("-")[1] for x in batch]
X2 = np.array([read_img(test_path + x)for x in X2])
 pred = curr_model.predict([X1, X2] ).ravel().tolist()
predictions += pred
submission['is_related'] = predictions
submission.to_csv("vgg_face.csv", index=False )<import_modules>
Fare_test_series=df_test.groupby(['Pclass'])['Fare'].transform('median')
 df_test['Fare']=df_test['Fare'].fillna(Fare_test_series )
import numpy as np
import pandas as pd
import os
from collections import defaultdict
from glob import glob
from random import choice, sample
from keras.preprocessing import image
import cv2
from tqdm import tqdm_notebook
import numpy as np
import pandas as pd
from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping
from keras.layers import Input, Dense, GlobalMaxPool2D, GlobalAvgPool2D, Concatenate, Multiply, Dropout, Subtract
from keras.models import Model
from keras.optimizers import Adam<install_modules>
df_train=pd.get_dummies(df_train,drop_first=True)
!pip install git+https://github.com/rcmalli/keras-vggface.git<import_modules>
df_test=pd.get_dummies(df_test,drop_first=True )
from keras_vggface.utils import preprocess_input
from keras_vggface.vggface import VGGFace<define_variables>
X=df_train.drop('Survived',axis=1)
 y=df_train['Survived']
train_file_path = ".. /input/train_relationships.csv"
train_folders_path = ".. /input/train/"<define_variables>
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.7, random_state=42 )
val_famillies_list = ["F07", "F08", "F09"] 
<define_variables>
from sklearn.tree import DecisionTreeClassifier
%%time
all_images = glob(train_folders_path + "*/*/*.jpg" )<define_variables>
dt=DecisionTreeClassifier(random_state=42 )
def get_train_val(family_name):
val_famillies = family_name
train_images = [x for x in all_images if val_famillies not in x]
val_images = [x for x in all_images if val_famillies in x]
train_person_to_images_map = defaultdict(list)
 ppl = [x.split("/")[-3] + "/" + x.split("/")[-2] for x in all_images]
for x in train_images:
train_person_to_images_map[x.split("/")[-3] + "/" + x.split("/")[-2]].append(x)
 val_person_to_images_map = defaultdict(list)
 for x in val_images:
val_person_to_images_map[x.split("/")[-3] + "/" + x.split("/")[-2]].append(x)
 relationships = pd.read_csv(train_file_path)
 relationships = list(zip(relationships.p1.values, relationships.p2.values))
relationships = [x for x in relationships if x[0] in ppl and x[1] in ppl]
train = [x for x in relationships if val_famillies not in x[0]]
val = [x for x in relationships if val_famillies in x[0]]
return train, val, train_person_to_images_map, val_person_to_images_map<categorify>
from sklearn.model_selection import GridSearchCV
def read_img(path):
img = image.load_img(path, target_size=(197, 197))
img = np.array(img ).astype(np.float)
 return preprocess_input(img, version=2)
 def gen(list_tuples, person_to_images_map, batch_size=16):
ppl = list(person_to_images_map.keys())
 while True:
batch_tuples = sample(list_tuples, batch_size // 2)
 labels = [1] * len(batch_tuples)
 while len(batch_tuples)< batch_size:
p1 = choice(ppl)
 p2 = choice(ppl)
 if p1 != p2 and(p1, p2)not in list_tuples and(p2, p1)not in list_tuples:
batch_tuples.append(( p1, p2))
labels.append(0)
 for x in batch_tuples:
if not len(person_to_images_map[x[0]]):
print(x[0])
 X1 = [choice(person_to_images_map[x[0]])for x in batch_tuples]
X1 = np.array([read_img(x)for x in X1])
 X2 = [choice(person_to_images_map[x[1]])for x in batch_tuples]
X2 = np.array([read_img(x)for x in X2])
 yield [X1, X2], labels
def focal_loss(gamma=2., alpha=.25):
def focal_loss_fixed(y_true, y_pred):
pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
return -K.sum(alpha * K.pow(1.- pt_1, gamma)* K.log(K.epsilon() +pt_1)) -K.sum(( 1-alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0 + K.epsilon()))
return focal_loss_fixed
def baseline_model() :
input_1 = Input(shape=(197, 197, 3))
input_2 = Input(shape=(197, 197, 3))
base_model = VGGFace(model='resnet50', include_top=False)
 for x in base_model.layers[:-3]:
x.trainable = True
for x in base_model.layers[-3:]:
x.trainable=False
x1 = base_model(input_1)
 x2 = base_model(input_2)
 x1 = Concatenate(axis=-1 )([GlobalMaxPool2D()(x1), GlobalAvgPool2D()(x1)])
 x2 = Concatenate(axis=-1 )([GlobalMaxPool2D()(x2), GlobalAvgPool2D()(x2)])
 x3 = Subtract()([x1, x2])
 x3 = Multiply()([x3, x3])
 x1_ = Multiply()([x1, x1])
 x2_ = Multiply()([x2, x2])
 x4 = Subtract()([x1_, x2_])
 x = Concatenate(axis=-1 )([x4, x3])
 x = Dense(100, activation="relu" )(x)
 x = Dropout(0.01 )(x)
 out = Dense(1, activation="sigmoid" )(x)
 model = Model([input_1, input_2], out)
 model.compile(loss=[focal_loss(alpha=.25, gamma=2)],
metrics=['acc'],
optimizer=Adam(0.00003))
model.summary()
return model<define_variables>
from sklearn.model_selection import GridSearchCV
n_val_famillies_list = len(val_famillies_list )<train_on_grid>
params = {
'max_depth': [3,5,8,12,15],
'min_samples_leaf': [5,8,12,15,20],
'criterion': ["gini", "entropy"]
}
for i in tqdm_notebook(range(n_val_famillies_list)) :
train, val, train_person_to_images_map, val_person_to_images_map = get_train_val(val_famillies_list[i])
 file_path = f"vgg_face_{i}.h5"
checkpoint = ModelCheckpoint(file_path, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
 reduce_on_plateau = ReduceLROnPlateau(monitor="val_acc", mode="max", factor=0.5, patience=10, verbose=1)
 es = EarlyStopping(monitor="val_acc", min_delta = 0.001, patience=30, verbose=1)
 callbacks_list = [checkpoint, reduce_on_plateau, es]
history = model.fit_generator(gen(train, train_person_to_images_map, batch_size=32),
use_multiprocessing=True,
validation_data=gen(val, val_person_to_images_map, batch_size=32),
epochs=300, verbose=1,
workers=4, callbacks=callbacks_list,
steps_per_epoch=400, validation_steps=200 )<load_from_csv>
grid_search = GridSearchCV(estimator=dt,
param_grid=params,
cv=4, n_jobs=-1, verbose=1, scoring = "accuracy" )
test_path = ".. /input/test/"
submission = pd.read_csv('.. /input/sample_submission.csv')
 def chunker(seq, size=32):
return(seq[pos:pos + size] for pos in range(0, len(seq), size))<predict_on_test>
grid_search.fit(X_train, y_train )
preds_for_sub = np.zeros(submission.shape[0])
 for i in tqdm_notebook(range(n_val_famillies_list)) :
file_path = f"vgg_face_{i}.h5"
model.load_weights(file_path)
 predictions = []
for batch in tqdm_notebook(chunker(submission.img_pair.values)) :
X1 = [x.split("-")[0] for x in batch]
X1 = np.array([read_img(test_path + x)for x in X1])
 X2 = [x.split("-")[1] for x in batch]
X2 = np.array([read_img(test_path + x)for x in X2])
 pred = model.predict([X1, X2] ).ravel().tolist()
predictions += pred
preds_for_sub += np.array(predictions)/ n_val_famillies_list<save_to_csv>
grid_search.cv_results_
submission['is_related'] = preds_for_sub
submission.to_csv("vgg_face.csv", index=False )<load_from_csv>
score_df = pd.DataFrame(grid_search.cv_results_)
 score_df.head()
sub1 = pd.read_csv('.. /input/smiles/vgg_face.csv')
 sub2 = pd.read_csv('.. /input/smiles/vgg_face(1 ).csv')
 sub3 = pd.read_csv('.. /input/smiles/submission(1 ).csv')
 temp=pd.read_csv('.. /input/smiles/submission(1 ).csv')
<save_to_csv>
score_df.nlargest(5,"mean_test_score" )
temp['is_related'] = 0.60*sub1['is_related'] + 0.22*sub2['is_related'] + 0.18*sub3['is_related']
temp.to_csv('submission4.csv', index=False )<define_variables>
grid_search.best_estimator_
!pip install git+https://github.com/rcmalli/keras-vggface.git
train_file_path = ".. /input/train_relationships.csv"
train_folders_path = ".. /input/train/"
val_famillies_list = ["F07", "F08", "F09"]
all_images = glob(train_folders_path + "*/*/*.jpg")
 relationships = pd.read_csv(train_file_path)
 def get_train_val(family_name, relationships=relationships):
val_famillies = family_name
train_images = [x for x in all_images if val_famillies not in x]
val_images = [x for x in all_images if val_famillies in x]
train_person_to_images_map = defaultdict(list)
 ppl = [x.split("/")[-3] + "/" + x.split("/")[-2] for x in all_images]
for x in train_images:
train_person_to_images_map[x.split("/")[-3] + "/" + x.split("/")[-2]].append(x)
 val_person_to_images_map = defaultdict(list)
 for x in val_images:
val_person_to_images_map[x.split("/")[-3] + "/" + x.split("/")[-2]].append(x)
 relationships = list(zip(relationships.p1.values, relationships.p2.values))
relationships = [x for x in relationships if x[0] in ppl and x[1] in ppl]
train = [x for x in relationships if val_famillies not in x[0]]
val = [x for x in relationships if val_famillies in x[0]]
return train, val, train_person_to_images_map, val_person_to_images_map
def read_img(path):
img = image.load_img(path, target_size=(197, 197))
img = np.array(img ).astype(np.float)
 return preprocess_input(img, version=2)
 def gen(list_tuples, person_to_images_map, batch_size=16):
ppl = list(person_to_images_map.keys())
 while True:
batch_tuples = sample(list_tuples, batch_size // 2)
 labels = [1] * len(batch_tuples)
 while len(batch_tuples)< batch_size:
p1 = choice(ppl)
 p2 = choice(ppl)
 if p1 != p2 and(p1, p2)not in list_tuples and(p2, p1)not in list_tuples:
batch_tuples.append(( p1, p2))
labels.append(0)
 for x in batch_tuples:
if not len(person_to_images_map[x[0]]):
print(x[0])
 X1 = [choice(person_to_images_map[x[0]])for x in batch_tuples]
X1 = np.array([read_img(x)for x in X1])
 X2 = [choice(person_to_images_map[x[1]])for x in batch_tuples]
X2 = np.array([read_img(x)for x in X2])
 yield [X1, X2], labels
def baseline_model() :
input_1 = Input(shape=(197, 197, 3))
input_2 = Input(shape=(197, 197, 3))
base_model = VGGFace(model='resnet50', include_top=False)
 for x in base_model.layers[:-3]:
x.trainable = True
for x in base_model.layers[-3:]:
x.trainable=False
x1 = base_model(input_1)
 x2 = base_model(input_2)
 x1 = Concatenate(axis=-1 )([GlobalMaxPool2D()(x1), GlobalAvgPool2D()(x1)])
 x2 = Concatenate(axis=-1 )([GlobalMaxPool2D()(x2), GlobalAvgPool2D()(x2)])
 x3 = Subtract()([x1, x2])
 x3 = Multiply()([x3, x3])
 x1_ = Multiply()([x1, x1])
 x2_ = Multiply()([x2, x2])
 x4 = Subtract()([x1_, x2_])
 x = Concatenate(axis=-1 )([x4, x3])
 x = Dense(100, activation="relu" )(x)
 x = Dropout(0.3 )(x)
 x = Dense(25, activation="relu" )(x)
 x = Dropout(0.3 )(x)
 out = Dense(1, activation="sigmoid" )(x)
 model = Model([input_1, input_2], out)
 model.compile(loss="binary_crossentropy", metrics=['acc'], optimizer=Adam(0.00001))
model.summary()
return model
model1 = baseline_model()
n_val_famillies_list = len(val_famillies_list )<train_on_grid>
dt_best=grid_search.best_estimator_
val_acc_list = []
def train_model1() :
for i in tqdm_notebook(range(n_val_famillies_list)) :
train, val, train_person_to_images_map, val_person_to_images_map = get_train_val(val_famillies_list[i])
 file_path = f"vgg_face_{i}.h5"
checkpoint = ModelCheckpoint(file_path, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
 reduce_on_plateau = ReduceLROnPlateau(monitor="val_acc", mode="max", factor=0.1, patience=10, verbose=1)
 es = EarlyStopping(monitor="val_acc", min_delta = 0.001, patience=20, verbose=1)
 callbacks_list = [checkpoint, reduce_on_plateau, es]
history = model1.fit_generator(gen(train, train_person_to_images_map, batch_size=16),
use_multiprocessing=True,
validation_data=gen(val, val_person_to_images_map, batch_size=16),
epochs=200, verbose=0,
workers=4, callbacks=callbacks_list,
steps_per_epoch=300, validation_steps=150)
 val_acc_list.append(np.max(history.history['val_acc']))
train_model1()<define_variables>
from sklearn.metrics import confusion_matrix, accuracy_score
model_rate_list = val_acc_list / np.sum(val_acc_list)
 model_rate_list<compute_test_metric>
accuracy_score(y_test,dt_best.predict(X_test))
model1_weight = np.matmul(val_acc_list, model_rate_list)
 model1_weight<load_from_csv>
predictions=dt_best.predict(df_test )
test_path = ".. /input/test/"
submission = pd.read_csv('.. /input/sample_submission.csv')
 def chunker(seq, size=32):
return(seq[pos:pos + size] for pos in range(0, len(seq), size))
def get_pred1() :
preds_for_sub = np.zeros(submission.shape[0])
 for i in tqdm_notebook(range(n_val_famillies_list)) :
file_path = f"vgg_face_{i}.h5"
model1.load_weights(file_path)
 predictions = []
for batch in tqdm_notebook(chunker(submission.img_pair.values)) :
X1 = [x.split("-")[0] for x in batch]
X1 = np.array([read_img(test_path + x)for x in X1])
 X2 = [x.split("-")[1] for x in batch]
X2 = np.array([read_img(test_path + x)for x in X2])
 pred = model1.predict([X1, X2] ).ravel().tolist()
predictions += pred
preds_for_sub += np.array(predictions)* model_rate_list[i]
return preds_for_sub
preds_model1 = get_pred1()<set_options>
titanic_3=pd.DataFrame({'PassengerId':PassengerId,'Survived':predictions})
 titanic_3.to_csv('My_3rd_submission',index=False )
<import_modules><EOS> 
pd.read_csv('My_3rd_submission' )