Kaggle competition: Titanic - Machine Learning from Disaster
log_loss(y_val, val_preds )<predict_on_test>
train = pd.read_csv(".. /input/titanic/train.csv")
 test = pd.read_csv(".. /input/titanic/test.csv")
 holdout_ids = test["PassengerId"]
%%time
val_shap_preds = model.predict(dval, pred_contribs=True )<predict_on_test>
male = train.loc[(train.Sex=='male')&(train.Age < 16)]
female = train.loc[(train.Sex=='female')&(train.Pclass==3)]
print(male['Survived'].value_counts())
 print(female['Survived'].value_counts() )
%%time
test_shap_preds = model.predict(dtest, pred_contribs=True)
<predict_on_test>
train['Title'] = 0
train['Title'] = train.Name.str.extract('([A-Za-z]+)\.')
 titles = {
"Mr": "man",
"Mme": "woman",
"Ms": "woman",
"Mrs": "woman",
"Master": "boy",
"Mlle": "woman",
"Miss": "woman",
"Capt": "man",
"Col": "man",
"Major": "man",
"Dr": "man",
"Rev": "man",
"Jonkheer": "man",
"Don": "man",
"Sir": "man",
"Countess": "woman",
"Dona": "woman",
"Lady": "woman"
}
train["Title"] = train["Title"].map(titles )
%%time
train_shap_preds = model.predict(dtrain, pred_contribs=True )<predict_on_test>
train['Surname'] = 0
train['Surname'] = train.Name.str.extract('([A-Za-z]+)\,')
 train.loc[(train.Title=='man'), 'Surname'] = 'DeadMan'
unique_surname =(train['Surname'].value_counts().index)[(train['Surname'].value_counts() == 1)]
passenger_lived = []
passenger_died = []
for surname in unique_surname:
single_df = train.loc[(train.Surname == surname)]
lived = single_df['Survived'].loc[(single_df.Survived == 1)].sum()
if(lived == 1):
passenger_lived.append(surname)
 else:
passenger_died.append(surname)
 non_unique_surname =(train['Surname'].value_counts().index)[ train['Surname'].value_counts() != 1 ].drop('DeadMan')
 families_lived = []
families_dead = []
for surname in non_unique_surname:
surname_df = train.loc[(train.Surname == surname)]
number_family = len(surname_df)
 number_lived = surname_df['Survived'].loc[(surname_df.Survived == 1)].sum()
if(number_lived/number_family >= 0.5):
families_lived.append(surname)
 else:
families_dead.append(surname)
 train['Surname'] = train['Surname'].replace(passenger_lived, 'LivedPassenger')
 train['Surname'] = train['Surname'].replace(passenger_died, 'DeadPassenger')
 train['Surname'] = train['Surname'].replace(families_lived, 'LivedFamily')
 train['Surname'] = train['Surname'].replace(families_dead, 'DeadFamily')
test_preds = model.predict(dtest )<save_to_csv>
train['Surname'].value_counts()
sample_submission['id'] = ids
sample_submission['label'] = test_preds
sample_submission.to_csv('submission.csv', index=False)
 sample_submission.head()<import_modules>
train['SurnameSurvival'] = 0
train.loc[(train.Surname == 'LivedPassenger'), 'SurnameSurvival'] = 1
train.loc[(train.Surname == 'LivedFamily'), 'SurnameSurvival'] = 1
import numpy as np
import pandas as pd
import zipfile
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import log_loss, roc_auc_score
from sklearn.linear_model import LogisticRegression
import os 
<load_from_csv>
family_lived = train.loc[(train.Surname == 'LivedFamily')]
print(family_lived['Survived'].value_counts() )
sample_submission = pd.read_csv('/kaggle/input/dogs-vs-cats-redux-kernels-edition/sample_submission.csv')
 sample_submission.tail()<define_variables>
prediction_train = train.SurnameSurvival
true_train = train.Survived
print(accuracy_score(true_train, prediction_train))
TRAIN_FOLDER = '.. /input/cats-and-dogs-embedded-data/train/train/'
TEST_FOLDER = '.. /input/cats-and-dogs-embedded-data/test/test/'
IMG_SIZE = 224<load_pretrained>
X = pd.read_csv(".. /input/titanic/train.csv" ).drop('Survived', axis=1)
 df = pd.concat([test,X],ignore_index=True,sort=False)
 test_index = test.index
train_image_list = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/train_image_list.npy')
 test_image_list = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/test_image_list.npy' )<define_variables>
df['Title'] = 0
df['Title'] = df.Name.str.extract('([A-Za-z]+)\.')
 titles = {
"Mr": "man",
"Mme": "woman",
"Ms": "woman",
"Mrs": "woman",
"Master": "boy",
"Mlle": "woman",
"Miss": "woman",
"Capt": "man",
"Col": "man",
"Major": "man",
"Dr": "man",
"Rev": "man",
"Jonkheer": "man",
"Don": "man",
"Sir": "man",
"Countess": "woman",
"Dona": "woman",
"Lady": "woman"
}
df["Title"] = df["Title"].map(titles )
ids = [int(x.split('.')[0])for x in test_image_list]
ids[:20]<load_pretrained>
df['Surname'] = 0
df['Surname'] = df.Name.str.extract('([A-Za-z]+)\,')
 df.loc[(df.Title=='man'), 'Surname'] = 'DeadMan'
for surname in passenger_lived:
df['Surname'] = df['Surname'].replace(surname, 'LivedPassenger')
 for surname in passenger_died:
df['Surname'] = df['Surname'].replace(surname, 'DeadPassenger')
 for surname in families_lived:
df['Surname'] = df['Surname'].replace(surname, 'LivedFamily')
 for surname in families_dead:
df['Surname'] = df['Surname'].replace(surname, 'DeadFamily')
target = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/target.npy' )<load_pretrained>
df['Surname'].value_counts()
train_EB5 = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_3/cats_and_dogs_3/train_EB5.npy')
 test_EB5 = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_3/cats_and_dogs_3/test_EB5.npy')
 train_EB6 = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_3/cats_and_dogs_3/train_EB6.npy')
 test_EB6 = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_3/cats_and_dogs_3/test_EB6.npy')
 train_EB7 = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_3/cats_and_dogs_3/train_EB7.npy')
 test_EB7 = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_3/cats_and_dogs_3/test_EB7.npy' )<split>
unique_surname =(df['Surname'].value_counts().index)[ df['Surname'].value_counts() == 1 ]
unique_surname
X_train_5, X_val_5, y_train, y_val = train_test_split(train_EB5, target, test_size=0.1, random_state=42)
 X_train_6, X_val_6, _, _ = train_test_split(train_EB6, target, test_size=0.1, random_state=42)
 X_train_7, X_val_7, _, _ = train_test_split(train_EB7, target, test_size=0.1, random_state=42 )<train_model>
single_passenger = df[df['Surname'].isin(unique_surname.to_list())]
single_woman = single_passenger['Surname'].loc[(single_passenger.Title=='woman')]
single_boy = single_passenger['Surname'].loc[(single_passenger.Title=='boy')]
df['Surname'] = df['Surname'].replace(single_woman, 'LivedPassenger')
 df['Surname'] = df['Surname'].replace(single_boy, 'DeadPassenger')
%%time
lr = LogisticRegression(C=0.012, max_iter=10000)
 lr.fit(X_train_5, y_train )<predict_on_test>
df['Surname'].value_counts()
val_preds_5 = lr.predict_proba(X_val_5)[:,1]<compute_test_metric>
new_families = df.loc[(df.Surname != 'DeadMan')&(df.Surname != 'LivedPassenger')&(df.Surname != 'DeadPassenger')&(df.Surname != 'LivedFamily')&(df.Surname != 'DeadFamily')]
new_families_list =(df['Surname'].value_counts().index)[ df['Surname'].value_counts() < 5 ]
new_families
roc_auc_score(y_val, val_preds_5 )<compute_test_metric>
df['Surname'] = df['Surname'].replace('Billiard', 'DeadFamily')
 df['Surname'] = df['Surname'].replace(new_families_list, 'LivedFamily')
 df['Surname'].value_counts()
log_loss(y_val, val_preds_5 )<predict_on_test>
df['SurnameSurvival'] = 0
df.loc[(df.Surname == 'LivedPassenger'), 'SurnameSurvival'] = 1
df.loc[(df.Surname == 'LivedFamily'), 'SurnameSurvival'] = 1
test_preds_5 = lr.predict_proba(test_EB5)[:,1]<train_model>
prediction = df['SurnameSurvival'].loc[test_index]
prediction
<predict_on_test><EOS> 
submission_df = {"PassengerId": holdout_ids,
"Survived": prediction}
submission = pd.DataFrame(submission_df)
 submission.to_csv("submission.csv",index=False )