Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<define_variables>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
import keras
train_file_path = ".. /input/train_relationships.csv"
train_folders_path = ".. /input/train/"
val_famillies_list = ["F07", "F08", "F09"]<choose_model_class>
train = pd.read_csv("/kaggle/input/titanic/train.csv")
 train.head()
def baseline_model() :
input_1 = Input(shape=(197, 197, 3))
input_2 = Input(shape=(197, 197, 3))
base_model = VGGFace(model='resnet50', include_top=False)
 for layer in base_model.layers[:-3]:
layer.trainable = True
x1 = base_model(input_1)
 x2 = base_model(input_2)
 merged_add = Add()([x1, x2])
 merged_sub = Subtract()([x1,x2])
 merged_add = Conv2D(100 , [1,1] )(merged_add)
 merged_sub = Conv2D(100 , [1,1] )(merged_sub)
 merged = Concatenate(axis=-1 )([merged_add, merged_sub])
 merged = Flatten()(merged)
 merged = Dense(100, activation="relu" )(merged)
 merged = Dropout(0.3 )(merged)
 merged = Dense(25, activation="relu" )(merged)
 merged = Dropout(0.3 )(merged)
 out = Dense(1, activation="sigmoid" )(merged)
 model = Model([input_1, input_2], out)
 model.compile(loss="binary_crossentropy", metrics=['acc'], optimizer=Adam(0.00001))
model.summary()
return model
model2 = baseline_model()<train_on_grid>
test = pd.read_csv("/kaggle/input/titanic/test.csv")
 test.head()
val_acc_list = []
def train_model2() :
for i in tqdm_notebook(range(n_val_famillies_list)) :
train, val, train_person_to_images_map, val_person_to_images_map = get_train_val(val_famillies_list[i])
 file_path = f"vgg_face_{i}.h5"
checkpoint = ModelCheckpoint(file_path, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
 reduce_on_plateau = ReduceLROnPlateau(monitor="val_acc", mode="max", factor=0.1, patience=10, verbose=0)
 es = EarlyStopping(monitor="val_acc", min_delta = 0.001, patience=20, verbose=1)
 callbacks_list = [checkpoint, reduce_on_plateau, es]
history = model2.fit_generator(gen(train, train_person_to_images_map, batch_size=16),
use_multiprocessing=True,
validation_data=gen(val, val_person_to_images_map, batch_size=16),
epochs=200, verbose=1,
workers=4, callbacks=callbacks_list,
steps_per_epoch=300, validation_steps=150)
 val_acc_list.append(np.max(history.history['val_acc']))
train_model2()<define_variables>
original_train = train.copy()
original_test = test.copy()
original_train, original_test
model_rate_list = val_acc_list / np.sum(val_acc_list)
 model_rate_list<compute_test_metric>
train.isna().sum(axis=0), test.isna().sum(axis=0 )
model2_weight = np.matmul(val_acc_list, model_rate_list)
 model2_weight<load_from_csv>
train.drop(['PassengerId', 'Name', 'Ticket', 'Cabin'], axis=1, inplace=True)
 test.drop(['PassengerId', 'Name', 'Ticket', 'Cabin'], axis=1, inplace=True)
 train.info, test.info
test_path = ".. /input/test/"
submission = pd.read_csv('.. /input/sample_submission.csv')
 def chunker(seq, size=32):
return(seq[pos:pos + size] for pos in range(0, len(seq), size))
def get_pred2() :
preds_for_sub = np.zeros(submission.shape[0])
 for i in tqdm_notebook(range(n_val_famillies_list)) :
file_path = f"vgg_face_{i}.h5"
model2.load_weights(file_path)
 predictions = []
for batch in tqdm_notebook(chunker(submission.img_pair.values)) :
X1 = [x.split("-")[0] for x in batch]
X1 = np.array([read_img(test_path + x)for x in X1])
 X2 = [x.split("-")[1] for x in batch]
X2 = np.array([read_img(test_path + x)for x in X2])
 pred = model2.predict([X1, X2] ).ravel().tolist()
predictions += pred
preds_for_sub += np.array(predictions)* model_rate_list[i]
return preds_for_sub
preds_model2 = get_pred2()<set_options>
women = train.loc[train.Sex == 'female']["Survived"]
rate_women = sum(women)/len(women)
 print("% of women who survived:", rate_women )
gc.collect()<compute_test_metric>
men = train.loc[train.Sex == 'male']["Survived"]
rate_men = sum(men)/len(men)
 print("% of men who survived:", rate_men )
preds =(preds_model1 * model1_weight + preds_model2 * model2_weight)/(model1_weight + model2_weight )<save_to_csv>
embarked_s = train.loc[train.Embarked == 'S']["Survived"]
rate_embarked_s = sum(embarked_s)/len(embarked_s)
 print("% of embarked people from Southampton who survived:", rate_embarked_s )
submission['is_related'] = preds
submission.to_csv("vgg_face.csv", index=False )<install_modules>
embarked_c = train.loc[train.Embarked == 'C']["Survived"]
rate_embarked_c = sum(embarked_c)/len(embarked_c)
 print("% of embarked people from Cherbourg who survived:", rate_embarked_c )
!pip install git+https://github.com/rcmalli/keras-vggface.git<import_modules>
embarked_q = train.loc[train.Embarked == 'Q']["Survived"]
rate_embarked_q = sum(embarked_q)/len(embarked_q)
 print("% of embarked people from Queenstown who survived:", rate_embarked_q )
import h5py
from collections import defaultdict
from glob import glob
from random import choice, sample
import cv2
import numpy as np
import pandas as pd
from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau
from keras.layers import Input, Dense, Flatten, GlobalMaxPool2D, GlobalAvgPool2D, Concatenate, Multiply, Dropout, Subtract, Add, Conv2D
from keras.models import Model
from keras.optimizers import Adam
from keras_vggface.utils import preprocess_input
from keras_vggface.vggface import VGGFace<define_variables>
train.loc[:,'FamSize'] = train.loc[:,'SibSp'] + train.loc[:,'Parch']
test.loc[:,'FamSize'] = train.loc[:,'SibSp'] + train.loc[:,'Parch']
train = train.drop(['Parch', 'SibSp'], axis = 1)
 test = test.drop(['Parch', 'SibSp'], axis = 1 )
train_file_path = ".. /input/train_relationships.csv"
train_folders_path = ".. /input/train/"
val_famillies = "F09"<define_variables>
age_imputed = train.groupby(['Pclass', 'Sex'] ).Age.transform('mean')
 train.Age.fillna(age_imputed, inplace=True )
all_images = glob(train_folders_path + "*/*/*.jpg" )<define_variables>
age_imputed = test.groupby(['Pclass', 'Sex'] ).Age.transform('mean')
 test.Age.fillna(age_imputed, inplace=True )
train_images = [x for x in all_images if val_famillies not in x]
val_images = [x for x in all_images if val_famillies in x]<define_variables>
train['Embarked'].value_counts()
train_person_to_images_map = defaultdict(list )<define_variables>
train.Embarked.fillna('S', inplace=True )
ppl = [x.split("/")[-3] + "/" + x.split("/")[-2] for x in all_images]<load_from_csv>
test.Fare = test.Fare.fillna(test.Fare.mean() )
for x in train_images:
train_person_to_images_map[x.split("/")[-3] + "/" + x.split("/")[-2]].append(x)
 val_person_to_images_map = defaultdict(list)
 for x in val_images:
val_person_to_images_map[x.split("/")[-3] + "/" + x.split("/")[-2]].append(x)
 relationships = pd.read_csv(train_file_path)
 relationships = list(zip(relationships.p1.values, relationships.p2.values))
relationships = [x for x in relationships if x[0] in ppl and x[1] in ppl]
train = [x for x in relationships if val_famillies not in x[0]]
val = [x for x in relationships if val_famillies in x[0]]<categorify>
train.isna().sum(axis=0), test.isna().sum(axis=0 )
def read_img(path):
img = cv2.imread(path)
 img = np.array(img ).astype(np.float)
 return preprocess_input(img, version=2)
 def gen(list_tuples, person_to_images_map, batch_size=16):
ppl = list(person_to_images_map.keys())
 while True:
batch_tuples = sample(list_tuples, batch_size // 2)
 labels = [1] * len(batch_tuples)
 while len(batch_tuples)< batch_size:
p1 = choice(ppl)
 p2 = choice(ppl)
 if p1 != p2 and(p1, p2)not in list_tuples and(p2, p1)not in list_tuples:
batch_tuples.append(( p1, p2))
labels.append(0)
 for x in batch_tuples:
if not len(person_to_images_map[x[0]]):
print(x[0])
 X1 = [choice(person_to_images_map[x[0]])for x in batch_tuples]
X1 = np.array([read_img(x)for x in X1])
 X2 = [choice(person_to_images_map[x[1]])for x in batch_tuples]
X2 = np.array([read_img(x)for x in X2])
 yield [X1, X2], labels
def baseline_model() :
input_1 = Input(shape=(224, 224, 3))
input_2 = Input(shape=(224, 224, 3))
base_model = VGGFace(model='resnet50', include_top=False)
 for layer in base_model.layers[:-3]:
layer.trainable = True
x1 = base_model(input_1)
 x2 = base_model(input_2)
 merged_add = Add()([x1, x2])
 merged_sub = Subtract()([x1,x2])
 merged_add = Conv2D(100 , [1,1] )(merged_add)
 merged_sub = Conv2D(100 , [1,1] )(merged_sub)
 merged = Concatenate(axis=-1 )([merged_add, merged_sub])
 merged = Flatten()(merged)
 merged = Dense(100, activation="relu" )(merged)
 merged = Dropout(0.3 )(merged)
 merged = Dense(25, activation="relu" )(merged)
 merged = Dropout(0.3 )(merged)
 out = Dense(1, activation="sigmoid" )(merged)
 model = Model([input_1, input_2], out)
 model.compile(loss="binary_crossentropy", metrics=['acc'], optimizer=Adam(0.00001))
model.summary()
return model
file_path = "vgg_face.h5"
checkpoint = ModelCheckpoint(file_path, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
 reduce_on_plateau = ReduceLROnPlateau(monitor="val_acc", mode="max", factor=0.1, patience=20, verbose=1)
 callbacks_list = [checkpoint, reduce_on_plateau]
model = baseline_model()<train_model>
gender_mapping = {'male':1, 'female':0}
embarked_mapping = {'S':0, 'C':1, 'Q':2}
train['Sex'] = train['Sex'].map(gender_mapping)
 train['Embarked'] = train['Embarked'].map(embarked_mapping)
 test['Sex'] = test['Sex'].map(gender_mapping)
 test['Embarked'] = test['Embarked'].map(embarked_mapping )
model.fit_generator(gen(train, train_person_to_images_map, batch_size=16), use_multiprocessing=True,
validation_data=gen(val, val_person_to_images_map, batch_size=16), epochs=130, verbose=1,
workers = 4, callbacks=callbacks_list, steps_per_epoch=200, validation_steps=100 )<predict_on_test>
y_train = train.Survived.values
test_path = ".. /input/test/"
def chunker(seq, size=32):
return(seq[pos:pos + size] for pos in range(0, len(seq), size))
submission = pd.read_csv('.. /input/sample_submission.csv')
 predictions = []
for batch in tqdm(chunker(submission.img_pair.values)) :
X1 = [x.split("-")[0] for x in batch]
X1 = np.array([read_img(test_path + x)for x in X1])
 X2 = [x.split("-")[1] for x in batch]
X2 = np.array([read_img(test_path + x)for x in X2])
 pred = model.predict([X1, X2] ).ravel().tolist()
predictions += pred
submission['is_related'] = predictions
submission.to_csv("vgg_face.csv", index=False )<import_modules>
target = train['Survived'].values
import pandas as pd
import numpy as np
import scipy
from sklearn.linear_model import Ridge, LogisticRegression
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.preprocessing import LabelBinarizer
import gc<load_from_csv>
train = train.drop(['Survived'], axis=1 )
data = {
'train': pd.read_csv(".. /input/train.tsv", sep='\t'),
'test': pd.read_csv(".. /input/test.tsv", sep='\t'),
}<split>
lr_model = LogisticRegression(random_state=10, max_iter = 1000)
 logit_params = {
"C": [1, 3, 10, 20, 30, 40],
"solver": ["lbfgs", "liblinear"]
}
logit_gs = GridSearchCV(lr_model, logit_params, scoring="accuracy", cv = 5, n_jobs=4)
 logit_gs.fit(train, y_train )
y_train = np.log1p(data['train']['price'])
 X_train, X_valid, y_train, y_valid = train_test_split(data['train'], y_train, test_size=0.2, random_state=42 )<concatenate>
print(logit_gs.best_score_ )
n_train = X_train.shape[0]
n_valid = X_valid.shape[0]
n_test = data['test'].shape[0]
full_data = pd.concat([X_train, X_valid, data['test']], axis=0)
 del data['train']
gc.collect()<string_transform>
rf_model = RandomForestClassifier()
rf_params ={
'bootstrap': [True, False],
'max_depth': [10, None],
'max_features': ['auto', 'sqrt'],
'min_samples_leaf': [1, 2, 4],
'min_samples_split': [2, 5, 10],
'n_estimators': [5, 10, 15, 20, 25, 30]}
rf_gs = GridSearchCV(rf_model, rf_params, scoring='accuracy', cv=8, n_jobs=4)
 rf_gs.fit(train, y_train )
def split_cat(text):
try:
return text.split("/")
 except:
return("Unknown", "Unknown", "Unknown" )<feature_engineering>
print(rf_gs.best_score_ )
full_data['general_cat'], full_data['subcat_1'], full_data['subcat_2'] = zip(*full_data['category_name'].apply(lambda x: split_cat(x)))
 full_data["brand_name"] = full_data["brand_name"].fillna("unknown")
 full_data["item_description"] = full_data["item_description"].fillna("No description yet" )<count_unique_values>
svc_model = SVC()
test_parameters = {
"C": [1, 3, 10, 30, 100],
"kernel": ["linear", "poly", "rbf" , "sigmoid"],
}
svc_gs = GridSearchCV(svc_model, test_parameters, scoring="accuracy", cv=5, n_jobs=4)
 svc_gs.fit(train, y_train )
print("There are %d General categories." % full_data['general_cat'].nunique() )<count_unique_values>
print(svc_gs.best_score_ )
print("There are %d cat1 categories." % full_data['subcat_1'].nunique() )<count_unique_values>
lgb_model = LGBMClassifier()
test_parameters = {
"n_estimators": [int(x)for x in np.linspace(5, 30, 6)],
"reg_alpha": [0, 0.75, 1, 1.25],
"learning_rate": [0.5, 0.4, 0.35, 0.3, 0.25, 0.2],
"subsample": [0.5, 0.75, 1]
}
lgb_gs = GridSearchCV(lgb_model, test_parameters, scoring="accuracy", cv=8, n_jobs=4)
 lgb_gs.fit(train, y_train )
print("There are %d cat2 categories." % full_data['subcat_2'].nunique() )<data_type_conversions>
print(lgb_gs.best_score_ )
full_data['general_cat'] = full_data['general_cat'].astype('category')
 full_data['subcat_1'] = full_data['subcat_1'].astype('category')
 full_data['subcat_2'] = full_data['subcat_2'].astype('category')
 full_data['brand_name'] = full_data['brand_name'].astype('category')
 full_data['item_condition_id'] = full_data['item_condition_id'].astype('category' )<string_transform>
ensemble_model = VotingClassifier(estimators=[ 
("logit", logit_gs.best_estimator_), 
("rf", rf_gs.best_estimator_), 
("svc", svc_gs.best_estimator_), 
("lgb", lgb_gs.best_estimator_),
], voting = "hard" )
stopwords = {x: 1 for x in stopwords.words('english')}
non_alphanums = re.compile(u'[^A-Za-z0-9]+')
 def norm_text(text):
return u" ".join(
 [x for x in [y for y in non_alphanums.sub(' ', text ).lower().strip().split(" ")] if len(x)> 1 and x not in stopwords] )<choose_model_class>
ensemble_model.fit(train, y_train )
wb = wordbatch.WordBatch(norm_text, extractor=(WordBag, {"hash_ngrams": 2, "hash_ngrams_weights": [1.5, 1.0],
"hash_size": 2 ** 29, "norm": None, "tf": 'binary',
"idf": None,
}), procs=8)
 wb.dictionary_freeze= True
X_name = wb.fit_transform(full_data['name'])
 del(wb)
 X_name = X_name[:, np.array(np.clip(X_name.getnnz(axis=0)- 1, 0, 1), dtype=bool)]
wb = wordbatch.WordBatch(norm_text, extractor=(WordBag, {"hash_ngrams": 2, "hash_ngrams_weights": [1.0, 0.5],
"hash_size": 2 ** 29, "norm": "l2", "tf": 'binary',
"idf": None,
}), procs=8)
 wb.dictionary_freeze= True
X_description = wb.fit_transform(full_data['item_description'])
 del(wb)
 X_description = X_description[:, np.array(np.clip(X_name.getnnz(axis=0)- 1, 0, 1), dtype=bool)]<prepare_x_and_y>
ensemble_model.score(train, y_train )
lb = LabelBinarizer(sparse_output=True)
 X_brand = lb.fit_transform(full_data['brand_name'])
 X_cat = lb.fit_transform(full_data['general_cat'])
 X_subcat1 = lb.fit_transform(full_data['subcat_1'])
 X_subcat2 = lb.fit_transform(full_data['subcat_2'])
 X_dummies = csr_matrix(pd.get_dummies(full_data[['item_condition_id', 'shipping']],
sparse=True ).values)
 print(print(X_dummies.shape, X_description.shape, X_brand.shape, X_cat.shape,
X_subcat1.shape, X_subcat2.shape,X_name.shape))
X = hstack(( X_dummies, X_name, X_brand, X_cat, X_subcat1, X_subcat2, X_description)).tocsr()<train_model>
classifier = Sequential()
classifier.add(Dense(activation="relu", input_dim=6, units=11, kernel_initializer="uniform"))
classifier.add(Dense(activation="relu", units=11, kernel_initializer="uniform"))
classifier.add(Dropout(0.5))
classifier.add(Dense(activation="relu", units=11, kernel_initializer="uniform"))
classifier.add(Dropout(0.5))
classifier.add(Dense(activation="relu", units=5, kernel_initializer="uniform"))
classifier.add(Dense(activation="sigmoid", units=1, kernel_initializer="uniform"))
classifier.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'])
 classifier.summary()
ridge_model = Ridge(solver='auto', fit_intercept=True, alpha=0.4,
max_iter=200, normalize=False, tol=0.01, random_state = 42 )<train_model>
features = train[['Pclass', 'Sex', 'Age', 'Fare', 'Embarked', 'FamSize']].values
ridge_model.fit(X_train, y_train )<compute_test_metric>
history = classifier.fit(features, target, batch_size = 32, epochs=200, validation_split=0.1,verbose = 1,shuffle=True )
def rmsle(Y, Y_pred):
assert Y.shape == Y_pred.shape
return np.sqrt(np.mean(np.square(Y_pred - Y)) )<compute_test_metric>
predictions = ensemble_model.predict(test)
 predictions
<predict_on_test><EOS> 
submission = pd.DataFrame({'PassengerId': original_test.PassengerId,'Survived': predictions})
 submission.to_csv('my_submission.csv', index=False )