Kaggle competition: Titanic - Machine Learning from Disaster
import numpy as np
import pandas as pd
import zipfile
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import log_loss, roc_auc_score
from sklearn.linear_model import LogisticRegression
import os 
<load_from_csv>
pd.set_option('display.max_columns', 500)
 sns.set_style('whitegrid')
 simplefilter(action='ignore', category=FutureWarning)
 simplefilter(action='ignore', category=DeprecationWarning)
 %matplotlib inline
sample_submission = pd.read_csv('/kaggle/input/dogs-vs-cats-redux-kernels-edition/sample_submission.csv')
 sample_submission.tail()<define_variables>
train_data = pd.read_csv('/kaggle/input/titanic/train.csv')
 test_data = pd.read_csv('/kaggle/input/titanic/test.csv')
 train_data.head()
TRAIN_FOLDER = '.. /input/cats-and-dogs-embedded-data/train/train/'
TEST_FOLDER = '.. /input/cats-and-dogs-embedded-data/test/test/'
IMG_SIZE = 224<load_pretrained>
submission = test_data['PassengerId']
train_data.drop(['PassengerId'], axis=1, inplace=True)
 test_data.drop(['PassengerId'], axis=1, inplace=True )
train_image_list = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/train_image_list.npy')
 test_image_list = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/test_image_list.npy' )<define_variables>
def get_stats(df):
lines = df.shape[0]
d_types = df.dtypes
count = df.count()
unique = df.apply(lambda x: x.unique().shape[0])
 nulls = df.isnull().sum()
null_ratio =(df.isnull().sum() /lines)*100
skew = df.skew()
names = ['data_types', 'count', 'unique', 'null', 'null_ratio']
temp_df = pd.concat([d_types, count, unique, nulls, null_ratio], axis=1)
 temp_df.columns = names
return temp_df
train_stats = get_stats(train_data)
 train_stats
ids = [int(x.split('.')[0])for x in test_image_list]
ids[:20]<load_pretrained>
test_stats = get_stats(test_data)
 test_stats
target = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/target.npy' )<load_pretrained>
train_data.drop(['Ticket'], axis=1, inplace=True)
 test_data.drop(['Ticket'], axis=1, inplace=True )
train_EB2_ns = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_4/cats_and_dogs_4/train_EB2_ns.npy')
 test_EB2_ns = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_4/cats_and_dogs_4/test_EB2_ns.npy' )<split>
test_data['Fare'].fillna(test_data['Fare'].median() , inplace=True)
 train_data['Embarked'].fillna(train_data['Embarked'].mode() [0], inplace=True )
X_train, X_val, y_train, y_val = train_test_split(train_EB2_ns, target, test_size=0.1, random_state=42 )<train_model>
train_data['Cabin'].fillna(0, inplace=True)
 test_data['Cabin'].fillna(0, inplace=True)
 train_data.loc[(train_data['Cabin'] != 0), 'Cabin'] = 1
test_data.loc[(test_data['Cabin'] != 0), 'Cabin'] = 1
print(train_data['Cabin'].value_counts())
 print(test_data['Cabin'].value_counts() )
%%time
lr = LogisticRegression(C=0.02, max_iter=10000)
 lr.fit(X_train, y_train )<predict_on_test>
train_data['Name'] = train_data['Name'].str.split(', ', expand=True)[1].str.split('.', expand=True)[0]
test_data['Name'] = test_data['Name'].str.split(', ', expand=True)[1].str.split('.', expand=True)[0]
train_title_mask =(train_data['Name'].value_counts() < 10)
 test_title_mask =(test_data['Name'].value_counts() < 10)
 train_data['Name'] = train_data['Name'].apply(lambda x: 'Misc' if train_title_mask.loc[x] == True else x)
 test_data['Name'] = test_data['Name'].apply(lambda x: 'Misc' if test_title_mask.loc[x] == True else x)
 print(train_data['Name'].value_counts())
 print(test_data['Name'].value_counts() )
val_preds = lr.predict_proba(X_val)[:,1]<compute_test_metric>
train_data['F_size'] = train_data['Parch'] + train_data['SibSp'] + 1
test_data['F_size'] = test_data['Parch'] + test_data['SibSp'] + 1
train_data['Alone'] = 0
test_data['Alone'] = 0
train_data['Alone'].loc[(train_data['F_size'] == 1)] = 1
test_data['Alone'].loc[(test_data['F_size'] == 1)] = 1
roc_auc_score(y_val, val_preds )<compute_test_metric>
train_data.drop(['Parch', 'SibSp'], axis=1, inplace=True)
 test_data.drop(['Parch', 'SibSp'], axis=1, inplace=True )
log_loss(y_val, val_preds )<predict_on_test>
temp = train_data.groupby('Pclass')[['Age']].median()
temp
for i in range(3):
train_data.loc[(train_data['Age'].isnull() &(train_data['Pclass'] ==(i + 1))), 'Age'] = temp.loc[(i + 1), 'Age']
test_data.loc[(test_data['Age'].isnull() &(test_data['Pclass'] ==(i + 1))), 'Age'] = temp.loc[(i + 1), 'Age']
test_preds = lr.predict_proba(test_EB2_ns)[:,1]<save_to_csv>
get_stats(test_data )
sample_submission['id'] = ids
sample_submission['label'] = test_preds
sample_submission.to_csv('submission.csv', index=False)
 sample_submission.head()<import_modules>
label = LabelEncoder()
train_data['Sex'] = label.fit_transform(train_data['Sex'])
 test_data['Sex'] = label.fit_transform(test_data['Sex'])
 train_data.head()
import numpy as np
import pandas as pd
import zipfile
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import log_loss, roc_auc_score
from sklearn.linear_model import LogisticRegression
import os 
<load_from_csv>
train_data[['Name', 'Embarked', 'F_size']] = train_data[['Name', 'Embarked', 'F_size']].astype(str)
 test_data[['Name', 'Embarked', 'F_size']] = test_data[['Name', 'Embarked', 'F_size']].astype(str)
 print(train_data.shape)
 train_data = pd.get_dummies(train_data)
 test_data = pd.get_dummies(test_data)
 print(train_data.shape)
 print(test_data.shape )
sample_submission = pd.read_csv('/kaggle/input/dogs-vs-cats-redux-kernels-edition/sample_submission.csv')
 sample_submission.tail()<define_variables>
pow_scale = PowerTransformer(method='yeo-johnson' ).fit(train_data[['Fare']])
 train_data[['Fare']] = pow_scale.transform(train_data[['Fare']])
 test_data[['Fare']] = pow_scale.transform(test_data[['Fare']])
 std_scale = StandardScaler().fit(train_data[['Age']])
 train_data[['Age']] = std_scale.transform(train_data[['Age']])
 test_data[['Age']] = std_scale.transform(test_data[['Age']] )
TRAIN_FOLDER = '.. /input/cats-and-dogs-embedded-data/train/train/'
TEST_FOLDER = '.. /input/cats-and-dogs-embedded-data/test/test/'
IMG_SIZE = 224<load_pretrained>
def initial_score(mod, X, y, folds):
cv = KFold(n_splits=folds, shuffle=True)
 cv_estimate = cross_val_score(mod, X, y, cv=cv, scoring='accuracy', n_jobs=4)
 mean = np.mean(cv_estimate)
 std = np.std(cv_estimate)
 return mean, std
train_image_list = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/train_image_list.npy')
 test_image_list = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/test_image_list.npy' )<define_variables>
svc_mod = SVC(probability=True)
 ada_mod = AdaBoostClassifier()
rfc_mod = RandomForestClassifier()
gbc_mod = GradientBoostingClassifier()
mlp_mod = MLPClassifier()
ids = [int(x.split('.')[0])for x in test_image_list]
ids[:20]<load_pretrained>
names = ['SVC', 'AdaBoostClassifier', 'RandomForestClassifier', 'GradientBoostingClassifier', 'MLPClassifier']
mods = [svc_mod, ada_mod, rfc_mod, gbc_mod, mlp_mod]
results = pd.DataFrame(columns=['mean accuracy', 'std accuracy'], index=names)
 for name, mod in zip(names, mods):
mean, std = initial_score(mod, train_data, Label, 10)
 results.loc[name, 'mean accuracy'] = mean
results.loc[name, 'std accuracy'] = std
results
target = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/target.npy' )<load_pretrained>
cv = KFold(n_splits=10, shuffle=True)
 X_train, X_test, y_train, y_test = train_test_split(train_data, Label, test_size = 0.3 )
train_EB3_ns = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_4/cats_and_dogs_4/train_EB3_ns.npy')
 test_EB3_ns = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_4/cats_and_dogs_4/test_EB3_ns.npy' )<split>
svc_cf = SVC(probability=True)
 svc_cf.get_params()
X_train, X_val, y_train, y_val = train_test_split(train_EB3_ns, target, test_size=0.1, random_state=42 )<train_model>
param_grid = [{'C': [1, 10, 100, 1000], 'kernel': ['linear']},
{'C': [1, 10, 100, 1000], 'gamma': [1.0/50.0, 1.0/200.0, 1.0/500.0, 1.0/1000.0], 'kernel': ['rbf']}]
svc_class_grid = GridSearchCV(estimator=svc_cf,
param_grid=param_grid,
cv=cv,
scoring='accuracy',
return_train_score=True,
n_jobs=4,
verbose=1)
 svc_class_grid.fit(train_data, Label)
 svc_best_params = svc_class_grid.best_estimator_
print('GridsearchCV Best Score: {}'.format(svc_class_grid.best_score_))
print('
Tested HyperParameters Values')
 print('kernal: {}'.format(svc_class_grid.best_estimator_.kernel))
print('gamma: {}'.format(svc_class_grid.best_estimator_.gamma))
print('C: {}'.format(svc_class_grid.best_estimator_.C))
%%time
lr = LogisticRegression(C=0.02, max_iter=10000)
 lr.fit(X_train, y_train )<predict_on_test>
svm_class = clone(svc_best_params)
 svm_class.fit(X_train, y_train)
 probabilities = svm_class.predict_proba(X_test)
 def score_model(probs, threshold):
return np.array([1 if x >= threshold else 0 for x in probs[:,1]])
 def print_metrics(labels, probs, threshold):
scores = score_model(probs, threshold)
 mets = metrics.precision_recall_fscore_support(labels, scores)
 conf = metrics.confusion_matrix(labels, scores)
 print(' Confusion matrix')
 print(' Score positive Score negative')
 print('Actual positive {:6d} {:6d}'.format(conf[0,0], conf[0,1]))
print('Actual negative {:6d} {:6d}'.format(conf[1,0], conf[1,1]))
print('')
 print('Accuracy {:.4f}'.format(metrics.accuracy_score(labels, scores)))
 print('AUC {:.4f}'.format(metrics.roc_auc_score(labels, probs[:,1])))
 print('Macro precision {:.4f}'.format(float(( float(mets[0][0])+ float(mets[0][1])) /2.0)))
 print('Macro recall {:.4f}'.format(float(( float(mets[1][0])+ float(mets[1][1])) /2.0)))
 print(' ')
 print(' Positive Negative')
 print('Num case {:6d} {:6d}'.format(mets[3][0], mets[3][1]))
print('Precision {:.4f} {:.4f}'.format(mets[0][0], mets[0][1]))
print('Recall {:.4f} {:.4f}'.format(mets[1][0], mets[1][1]))
print('F1 {:.4f} {:.4f}'.format(mets[2][0], mets[2][1]))
def plot_auc(labels, probs):
fpr, tpr, threshold = metrics.roc_curve(labels, probs[:,1])
 auc = metrics.auc(fpr, tpr)
 plt.title('Receiver Operating Characteristic')
 plt.plot(fpr, tpr, color = 'blue', label = 'AUC = {:.4f}'.format(auc))
plt.legend(loc = 'lower right')
 plt.plot([0, 1], [0, 1],'r--')
 plt.xlim([0, 1])
 plt.ylim([0, 1])
 plt.ylabel('True Positive Rate')
 plt.xlabel('False Positive Rate')
 plt.show()
print_metrics(y_test, probabilities,.5)
 plot_auc(y_test, probabilities )
val_preds = lr.predict_proba(X_val)[:,1]<compute_test_metric>
ad_clf = AdaBoostClassifier(DecisionTreeClassifier())
 ad_clf.get_params()
roc_auc_score(y_val, val_preds )<compute_test_metric>
param_grid = {'base_estimator__max_depth' :[1, 2, 3, 5],
'base_estimator__min_samples_split':[2, 3, 5],
'base_estimator__min_samples_leaf':[2, 3, 5, 10],
'n_estimators' :[50, 100, 250, 500, 750],
'learning_rate' :[0.0001, 0.001, 0.01, 0.1, 1, 5]}
ad_clf_grid = GridSearchCV(estimator=ad_clf,
param_grid=param_grid,
cv=cv,
scoring='accuracy',
return_train_score=True,
n_jobs=4,
verbose=1)
 ad_clf_grid.fit(train_data, Label)
 ada_best_params = ad_clf_grid.best_estimator_
print('GridsearchCV Best Score: ' + str(ad_clf_grid.best_score_))
print('
Tested HyperParameters Values')
 print('base_estimator__max_depth: {}'.format(ad_clf_grid.best_estimator_.base_estimator.max_depth))
print('base_estimator__min_samples_split: {}'.format(ad_clf_grid.best_estimator_.base_estimator.min_samples_split))
print('base_estimator__min_samples_leaf : {}'.format(ad_clf_grid.best_estimator_.base_estimator.min_samples_leaf))
print('n_estimators: {}'.format(ad_clf_grid.best_estimator_.n_estimators))
print('learning_rate: {}'.format(ad_clf_grid.best_estimator_.learning_rate))
log_loss(y_val, val_preds )<predict_on_test>
rf_clf = RandomForestClassifier()
rf_clf.get_params()
test_preds = lr.predict_proba(test_EB3_ns)[:,1]<save_to_csv>
param_grid = {'n_estimators': [10, 50, 100, 250, 500, 750],
'max_depth': [5, 8, 15, 25, 30],
'min_samples_split': [2, 5, 10, 15],
'min_samples_leaf':[2, 3, 5, 10, 15, 20]}
rf_clf_grid = GridSearchCV(estimator=rf_clf,
param_grid=param_grid,
cv=cv,
scoring='accuracy',
return_train_score=True,
n_jobs=4,
verbose=1)
 rf_clf_grid.fit(train_data, Label)
 rf_best_params = rf_clf_grid.best_estimator_
print('GridsearchCV Best Score: {}'.format(rf_clf_grid.best_score_))
print('
Tested HyperParameters Values')
 print('n_estimators: {}'.format(rf_clf_grid.best_estimator_.n_estimators))
print('max_depth: {}'.format(rf_clf_grid.best_estimator_.max_depth))
print('min_samples_split: {}'.format(rf_clf_grid.best_estimator_.min_samples_split))
print('min_samples_leaf: {}'.format(rf_clf_grid.best_estimator_.min_samples_leaf))
sample_submission['id'] = ids
sample_submission['label'] = test_preds
sample_submission.to_csv('submission.csv', index=False)
 sample_submission.head()<import_modules>
gb_clf = GradientBoostingClassifier()
gb_clf.get_params()
import numpy as np
import pandas as pd
import zipfile
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import log_loss, roc_auc_score
from sklearn.linear_model import LogisticRegression
import os 
<load_from_csv>
param_grid = {'n_estimators': [50, 100, 250, 500, 750, 1000],
'max_depth': [3, 5, 8, 15, 25],
'min_samples_split': [2, 5, 10],
'min_samples_leaf':[1, 2, 3, 5, 10],
'learning_rate' :[0.0001, 0.001, 0.01, 0.1, 1]}
gb_clf_grid = GridSearchCV(estimator=gb_clf,
param_grid=param_grid,
cv=KFold(n_splits=5, shuffle=True),
scoring='accuracy',
return_train_score=True,
n_jobs=4,
verbose=1)
 gb_clf_grid.fit(train_data, Label)
 gb_best_params = gb_clf_grid.best_estimator_
print('GridsearchCV Best Score: {}'.format(gb_clf_grid.best_score_))
print('
Tested HyperParameters Values')
 print('n_estimators: {}'.format(gb_clf_grid.best_estimator_.n_estimators))
print('max_depth: {}'.format(gb_clf_grid.best_estimator_.max_depth))
print('min_samples_split: {}'.format(gb_clf_grid.best_estimator_.min_samples_split))
print('min_samples_leaf: {}'.format(gb_clf_grid.best_estimator_.min_samples_leaf))
print('learning_rate: {}'.format(gb_clf_grid.best_estimator_.learning_rate))
sample_submission = pd.read_csv('/kaggle/input/dogs-vs-cats-redux-kernels-edition/sample_submission.csv')
 sample_submission.tail()<define_variables>
mlp_clf = MLPClassifier()
mlp_clf.get_params()
TRAIN_FOLDER = '.. /input/cats-and-dogs-embedded-data/train/train/'
TEST_FOLDER = '.. /input/cats-and-dogs-embedded-data/test/test/'
IMG_SIZE = 224<load_pretrained>
param_grid = {'hidden_layer_sizes' :[(50, 50, 10),(50, 100, 10),(100,)],
'solver' :['lbfgs', 'adam'],
'alpha' :[.000001,.00001,.0001,.001],
'early_stopping' :[True, False],
'beta_1' :[.8,.9,.99],
'beta_2' :[.8,.9,.99,.999],
'max_iter' :[100, 200, 500, 1000]}
mlp_clf_grid = GridSearchCV(estimator=mlp_clf,
param_grid=param_grid,
cv=KFold(n_splits=5, shuffle=True),
scoring='accuracy',
return_train_score=True,
n_jobs=4,
verbose=1)
 mlp_clf_grid.fit(train_data, Label)
 mlp_best_params = mlp_clf_grid.best_estimator_
print('GridSearchCV Best Score: {}'.format(mlp_clf_grid.best_score_))
print('
Tested HyperParameters Values')
 print('hidden_layer_sizes: {}'.format(mlp_clf_grid.best_estimator_.hidden_layer_sizes))
print('solver: {}'.format(mlp_clf_grid.best_estimator_.solver))
print('alpha: {}'.format(mlp_clf_grid.best_estimator_.alpha))
print('early_stopping: {}'.format(mlp_clf_grid.best_estimator_.early_stopping))
print('beta_1: {}'.format(mlp_clf_grid.best_estimator_.beta_1))
print('beta_2: {}'.format(mlp_clf_grid.best_estimator_.beta_2))
print('max_iter: {}'.format(mlp_clf_grid.best_estimator_.max_iter))
train_image_list = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/train_image_list.npy')
 test_image_list = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/test_image_list.npy' )<define_variables>
voting_cf = VotingClassifier(estimators=[('rfc', rf_best_params), 
('svc', svc_best_params), 
('ada', ada_best_params), 
('gbc', gb_best_params), 
('mlp', mlp_best_params)], voting='soft', n_jobs=4)
 voting_cf = voting_cf.fit(X_train, y_train)
 voting = voting_cf.predict(X_test )
ids = [int(x.split('.')[0])for x in test_image_list]
ids[:20]<load_pretrained>
def print_metrics_score(labels, scores):
mets = metrics.precision_recall_fscore_support(labels, scores)
 conf = metrics.confusion_matrix(labels, scores)
 print(' Confusion matrix')
 print(' Score positive Score negative')
 print('Actual positive {:6d} {:6d}'.format(conf[0,0], conf[0,1]))
print('Actual negative {:6d} {:6d}'.format(conf[1,0], conf[1,1]))
print('')
 print('Accuracy {:.4f}'.format(metrics.accuracy_score(labels, scores)))
 print('Macro precision {:.4f}'.format(float(( float(mets[0][0])+ float(mets[0][1])) /2.0)))
 print('Macro recall {:.4f}'.format(float(( float(mets[1][0])+ float(mets[1][1])) /2.0)))
 print(' ')
 print(' Positive Negative')
 print('Num case {:6d} {:6d}'.format(mets[3][0], mets[3][1]))
print('Precision {:.4f} {:.4f}'.format(mets[0][0], mets[0][1]))
print('Recall {:.4f} {:.4f}'.format(mets[1][0], mets[1][1]))
print('F1 {:.4f} {:.4f}'.format(mets[2][0], mets[2][1]))
print_metrics_score(y_test, voting )
target = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/target.npy' )<load_pretrained>
svc_best = clone(svc_best_params)
 svc_best.fit(train_data, Label)
 scores = score_model(svc_best.predict_proba(test_data),.5)
 sub = pd.DataFrame({'PassengerId':submission, 'Survived': scores})
 sub.to_csv('svc_prediction.csv', index=False )
train_EB7_ns = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_4/cats_and_dogs_4/train_EB7_ns.npy')
 test_EB7_ns = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_4/cats_and_dogs_4/test_EB7_ns.npy' )<split>
ada_best = clone(ada_best_params)
 ada_best.fit(train_data, Label)
 scores = score_model(ada_best.predict_proba(test_data),.5)
 sub = pd.DataFrame({'PassengerId':submission, 'Survived': scores})
 sub.to_csv('adaboost_prediction.csv', index=False )
X_train, X_val, y_train, y_val = train_test_split(train_EB7_ns, target, test_size=0.1, random_state=42 )<train_model>
rf_best = clone(rf_best_params)
 rf_best.fit(train_data, Label)
 scores = score_model(rf_best.predict_proba(test_data),.5)
 sub = pd.DataFrame({'PassengerId':submission, 'Survived': scores})
 sub.to_csv('random_forest_prediction.csv', index=False )
%%time
lr = LogisticRegression(C=0.026, max_iter=10000)
 lr.fit(X_train, y_train )<predict_on_test>
gb_best = clone(gb_best_params)
 gb_best.fit(train_data, Label)
 scores = score_model(gb_best.predict_proba(test_data),.5)
 sub = pd.DataFrame({'PassengerId':submission, 'Survived': scores})
 sub.to_csv('gradient_boost_prediction.csv', index=False )
val_preds = lr.predict_proba(X_val)[:,1]<compute_test_metric>
mlp_best = clone(mlp_best_params)
 mlp_best.fit(train_data, Label)
 scores = score_model(mlp_best.predict_proba(test_data),.5)
 sub = pd.DataFrame({'PassengerId':submission, 'Survived': scores})
 sub.to_csv('MPL_nural_prediction.csv', index=False )
<compute_test_metric><EOS> 
voting_final_cf = clone(voting_cf)
 voting_final_cf.fit(train_data, Label)
 sub = pd.DataFrame({'PassengerId':submission, 'Survived': voting_final_cf.predict(test_data)})
 sub.to_csv('4_est_vote_prediction.csv', index=False )