Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_pretrained>
pd.set_option('display.max_columns', 30)
 sns.set()
learn.freeze()
learn.lr_find(start_lr=1e-6, end_lr=1e1, wd=5e-3)
 learn.recorder.plot(suggestion=True )<train_model>
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )
learn.fit_one_cycle(2, max_lr=3e-2, div_factor=10, final_div=100, wd=5e-3)
 learn.save('stage-1')
 learn.recorder.plot_losses()<train_model>
print('Number of records:')
 print(len(df_train.index))
print('{:=<70}'.format(''))
print('Missing values in the training dataset:')
 print(df_train.isnull().sum())
 print('{:=<70}'.format(''))
learn.unfreeze() 
<train_model>
print('Number of records:')
 print(len(df_test.index))
print('{:=<70}'.format(''))
print('Missing values in the test dataset:')
 print(df_test.isnull().sum())
 print('{:=<70}'.format(''))
learn.fit_one_cycle(cyc_len=5, max_lr=slice(5e-5, 5e-4), pct_start=0, wd=1e-3)
 learn.save('stage-3')
<train_model>
df_train['Dataset'] = 'train'
df_test['Dataset'] = 'test'
df_test.insert(loc=1, column='Survived', value=np.nan)
 df_all = df_train.append(df_test )
tta_params = {'beta':0.12, 'scale':1.0}<load_from_csv>
print('Number of records:')
 print(len(df_all.index))
print('{:=<70}'.format(''))
print('Missing values in the test dataset:')
 print(df_all.isnull().sum())
 print('{:=<70}'.format(''))
sample_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 sample_df.head()<define_variables>
print('Percentage of passengers who survived(1)and died(0):')
 print(df_train['Survived'].value_counts(normalize=True))
learn.data.add_test(ImageList.from_df(
 sample_df, PATH,
folder='test_images',
suffix='.png' 
))<choose_model_class>
sex = df_train['Sex'].value_counts(normalize=True)
 print('{:=<70}'.format(''))
print('Proportion of passengers by gender:')
 print(sex)
 print('{:=<70}'.format(''))
f_passengers = sex['female']
m_passengers = sex['male']
preds,y = learn.TTA(ds_type=DatasetType.Test, **tta_params )<prepare_output>
proportions_col = ['category', '%passengers', '%survived']
proportions_cat = ['female', 'male', 'class 1', 'class 2', 'class 3']
proportions_survived = [f_survived, m_survived, c1_survived, c2_survived, c3_survived]
proportions_passengers =[f_passengers, m_passengers, c1_passengers, c2_passengers, c3_passengers]
df_proportions = pd.DataFrame(
 list(zip(proportions_cat,
proportions_passengers,
proportions_survived)
),
columns=proportions_col)
 df_proportions['variance'] = df_proportions['%passengers'] - df_proportions['%survived']
df_proportions_melted = df_proportions.melt(id_vars=['category', 'variance'],
var_name='ratio type',
value_name='ratio value')
 my_red = '
my_green = '
l_colours = [my_red if i >=0 else my_green for i in df_proportions['variance'].tolist() ]
d_colours = dict(zip(df_proportions['category'],l_colours))
fig, ax = plt.subplots(figsize=(6,8))
ax = sns.pointplot(x='ratio type',
y='ratio value',
hue='category',
data=df_proportions_melted,
palette=d_colours)
 for i in range(0, len(df_proportions)) :
lbl_offset = 1.1
if(df_proportions['%passengers'] - df_proportions['%survived'])[i] < 0:
ax.text(x=lbl_offset,
y=df_proportions['%survived'][i],
s=df_proportions['category'][i],
ha='left',
color=my_green)
 else:
ax.text(x=lbl_offset,
y=df_proportions['%survived'][i],
s=df_proportions['category'][i],
ha='left',
color=my_red)
 ax.set(title='Proportions of passengers in the total and survivors population',
ylabel='Percentage')
 ax.xaxis.label.set_visible(False)
 ax.legend_.set_visible(False)
 plt.show()
sample_df.diagnosis = preds.argmax(1)
 sample_df.head()<save_to_csv>
summary = pd.pivot_table(data=df_train,
index=['Survived'],
columns=['Sex', 'Pclass'],
values=['Name'],
aggfunc=('count'),
margins=True,
margins_name='Total')
 print('{:=<70}'.format(''))
print(summary.div(summary.iloc[-1]))
print('{:=<70}'.format(''))
sample_df.to_csv('submission.csv',index=False)
 _ = sample_df.hist()<count_values>
def get_chi2(crosstab, HasMargin=True, proba=0.95):

 var1 = crosstab.index.name
var2 = crosstab.columns.name
if HasMargin==True:
i = -1
else:
i = ""
stat, p, dof, expected = chi2_contingency(
 observed=crosstab.iloc[:i,:i],
correction=True)
 critical = chi2.ppf(proba, dof)
 if stat >= critical:
chi2_independence = 'The variables {} and {} are dependent(X2>eX2)'.format(var1, var2)
 else:
chi2_independence = 'The variables {} and {} are independent(X2<eX2)'.format(var1, var2)
 alpha = 1 - proba
if p <= alpha:
chi2_significance = 'The test is statistically significant(p<alpha)'
else:
chi2_significance = 'The test is not statistically significant(p>alpha)'
print('{:^70}'.format('Chi2 test of independence'))
print('{:=<70}'.format(''))
print('Tested variables')
 print(var1)
 print(var2)
 print('{:=<70}'.format(''))
print('{}'.format('Test of independence'))
print('Chi-square statistic(X2): {:.3f}'.format(stat))
print('Minimum expected chi-square statistic(eX2): {:.3f}'.format(critical))
print(chi2_independence)
 print('{:=<70}'.format(''))
print('Significance of the test')
 print('p-value(p): {:.3f}'.format(p))
print('Accepted significance level(alpha): {:.3f}'.format(alpha))
print(chi2_significance)
 print('{:=<70}'.format(''))
sample_df['diagnosis'].value_counts()<set_options>
get_chi2(summary_sex, proba=0.99 )
%reload_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
get_chi2(summary_class, proba=0.99 )
import fastai
from fastai import *
from fastai.vision import *
from fastai.callbacks import *
import cv2
import pandas as pd
import matplotlib.pyplot as plt<set_options>
df_train['FamilySize'] = df_train['SibSp'] + df_train['Parch'] + 1
df_all['FamilySize'] = df_all['SibSp'] + df_all['Parch'] + 1
print('Make sure cudnn is enabled:', torch.backends.cudnn.enabled )<set_options>
family_passengers = df_train['FamilySize'].value_counts(normalize=True, sort=False ).to_frame().reset_index()
family_passengers.columns = ['FamilySize', '%total']
print('{:=<70}'.format(''))
print(family_passengers)
 print('{:=<70}'.format(''))
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 1667
seed_everything(SEED )<feature_engineering>
df_family = pd.merge(left=family_passengers,
right=family_survived,
how='inner',
on='FamilySize')
 df_family_melted = df_family.melt(id_vars=['FamilySize'],
var_name='ratio type',
value_name='ratio value')
 ax = sns.catplot(data=df_family_melted,
x='ratio type',
y='ratio value',
col='FamilySize',
col_wrap=9,
kind='point',
height=3,
aspect=0.8)
 ax.set_axis_labels('', 'Percentage')
 plt.subplots_adjust(top=0.7)
 plt.suptitle('Proportions of passengers: total and survivors')
 plt.show()
base_image_dir = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 train_dir = os.path.join(base_image_dir,'train_images/')
 df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 df.head(10 )<define_variables>
get_chi2(summary_family )
src =(ImageList.from_df(df=df,path='./',cols='path')
.split_by_rand_pct(0.2, seed=42)
.label_from_df(cols='diagnosis',label_cls=FloatList)
)
 src<normalization>
df_median_age = pd.pivot_table(data=df_age,
index=['Sex','Pclass'],
values='Age',
aggfunc=np.median ).reset_index()
def fill_age(data):
median = df_median_age
age = data['Age']
sex = data['Sex']
pclass = data['Pclass']
if pd.isnull(age):
if sex == 'female' and pclass == 1:
return median[(median['Sex'] == 'female')&(median['Pclass'] == 1)]['Age'].values[0]
if sex == 'female' and pclass == 2:
return median[(median['Sex'] == 'female')&(median['Pclass'] == 2)]['Age'].values[0]
if sex == 'female' and pclass == 3:
return median[(median['Sex'] == 'female')&(median['Pclass'] == 3)]['Age'].values[0]
if sex == 'male' and pclass == 1:
return median[(median['Sex'] == 'male')&(median['Pclass'] == 1)]['Age'].values[0]
if sex == 'male' and pclass == 2:
return median[(median['Sex'] == 'male')&(median['Pclass'] == 2)]['Age'].values[0]
if sex == 'male' and pclass == 3:
return median[(median['Sex'] == 'male')&(median['Pclass'] == 3)]['Age'].values[0]
else:
return age
df_train['Age'] = df_train[['Age', 'Sex', 'Pclass']].apply(fill_age, axis=1)
 df_all['Age'] = df_all[['Age', 'Sex', 'Pclass']].apply(fill_age, axis=1)
 print('{:=<70}'.format(''))
print('Missing age values in the combined dataset:')
 print(df_all['Age'].isnull().sum())
 print('{:=<70}'.format(''))
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=0.10, max_zoom=1.3, max_warp=0.0, max_lighting=0.2)
 data =(
 src.transform(tfms,size=224)
.databunch() 
.normalize(imagenet_stats)
)
 data<compute_test_metric>
print(df_train['Embarked'].value_counts(normalize=True))
def quadratic_kappa(y_hat, y):
return torch.tensor(cohen_kappa_score(torch.round(y_hat), y, weights='quadratic'),device='cuda:0' )<compute_test_metric>
df_train['Embarked'].fillna(value='S', inplace=True)
 df_all['Embarked'].fillna(value='S', inplace=True )
kappa = KappaScore()
kappa.weights = "quadratic" 
<choose_model_class>
get_chi2(summary_embarked )
learn = cnn_learner(data, base_arch=models.resnet152 ,
metrics=[quadratic_kappa],model_dir='/kaggle',
pretrained=True,
callback_fns=[partial(EarlyStoppingCallback, monitor='quadratic_kappa',
min_delta=0.01, patience=3)] )<train_model>
df_all[df_all['Fare'].isnull() == True]
learn.fit_one_cycle(6, 3e-2)
<save_model>
median_fare = df_train['Fare'][df_train['Pclass']==3].median()
df_all['Fare'].fillna(value=median_fare, inplace=True )
learn.save('stage1' )<train_model>
d_ticket_count = dict(df_all['Ticket'].value_counts())
 df_all['TicketCount'] = df_all['Ticket'].map(d_ticket_count)
 df_all['AdjFare'] = df_all['Fare'] / df_all['TicketCount']
df_train['AdjFare'] = df_all[df_all['Dataset'] == 'train']['AdjFare']
learn.fit_one_cycle(10, max_lr=slice(1e-6,1e-3))<predict_on_test>
df_all['LastName'] = df_all['Name'].str.extract(pat= '^([^,]*),', expand=True)
valid_preds = learn.get_preds(ds_type=DatasetType.Valid )<import_modules>
df_all['TicketNum'] = df_all['Ticket'].str.replace(pat= '(\D)', repl= '')
import numpy as np
import pandas as pd
import os
import scipy as sp
from functools import partial
from sklearn import metrics
from collections import Counter
import json<compute_test_metric>
d_ticket_count = dict(df_all['Ticket'].value_counts())
 df_all['TicketCount'] = df_all['Ticket'].map(d_ticket_count )
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']<train_model>
df_all['SharedTicket'] = [1 if i > 1 else 0 for i in df_all['TicketCount']]
d_shared_name = dict(df_all['LastName'].value_counts())
 df_all['SharedName'] = [1 if i > 1 else 0 for i in df_all['LastName'].map(d_shared_name)]
df_all['SharedFeatures'] = df_all['SharedTicket'] + df_all['SharedName']
optR = OptimizedRounder()
optR.fit(valid_preds[0],valid_preds[1] )<load_from_csv>
shared_features = [
df_all['SharedFeatures'] == 0,
df_all['SharedFeatures'] == 1, 
(df_all['SharedTicket'] == 0)&(df_all['SharedName'] == 1)&(df_all['FamilySize'] == 1), 
(df_all['SharedTicket'] == 0)&(df_all['SharedName'] == 1)&(df_all['FamilySize'] > 1), 
(df_all['SharedTicket'] == 1)&(df_all['SharedName'] == 0)
 ]
is_single = [1, 0, 1, 0, 0]
df_all['IsSingle'] = np.select(shared_features, is_single )
sample_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 sample_df.head()<save_to_csv>
from sklearn.model_selection import train_test_split
from sklearn import preprocessing
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
learn.data.add_test(ImageList.from_df(sample_df,'.. /input/aptos2019-blindness-detection',folder='test_images',suffix='.png'))
preds,y = learn.get_preds(ds_type=DatasetType.Test)
 test_predictions = optR.predict(preds, coefficients)
 sample_df.diagnosis = test_predictions.astype(int)
 sample_df.head()
sample_df.to_csv('submission.csv',index=False )<set_options>
df_all['Sex'] = [1 if i == 'female' else 0 for i in df_all['Sex']]
l_predictors = ['Dataset', 'Survived', 'Pclass', 'Sex', 'Embarked', 'IsSingle', 'FamilySize']
df_final = df_all[l_predictors]
df_final = pd.get_dummies(data = df_final,
columns = ['Pclass', 'Embarked', 'FamilySize'] )
%reload_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
predictors = df_final[df_final['Dataset'] == 'train'].drop(['Dataset', 'Survived'], axis=1)
 targets = df_final[df_final['Dataset'] == 'train']['Survived']
X_train, X_test, y_train, y_test = train_test_split(predictors, targets, test_size=0.30, random_state=0 )
from fastai import *
from fastai.vision import *
import torch<define_variables>
rfc = RandomForestClassifier(n_estimators=100, random_state = 42)
 rfc.fit(X_train, y_train)
 rfc_pred = rfc.predict(X_test)
 print(accuracy_score(rfc_pred, y_test))
path = Path('.. /input/aptos2019-blindness-detection')
 path_train = path/'train_images'
path_test = path/'test_images'
path, path_train, path_test<load_from_csv>
param = {'n_estimators': [100, 500],
'criterion' :['gini'],
'max_features': ['auto'],
'max_depth': [3, 4, 5]}
grid = GridSearchCV(estimator=rfc, param_grid=param, refit=True, cv=3)
 grid.fit(X_train,y_train)
 grid_pred = grid.predict(X_test)
 print(accuracy_score(grid_pred, y_test))
labels = pd.read_csv(path/'train.csv')
 labels.head()<feature_engineering>
grid.best_params_
tfms = get_transforms(
 do_flip=True,
flip_vert=True,
max_warp=0.1,
max_rotate=360.,
max_zoom=1.1,
max_lighting=0.1,
p_lighting=0.5
)<define_search_space>
rfc_best = RandomForestClassifier(random_state = 42,
criterion = 'gini',
max_depth = 4,
max_features = 'auto',
n_estimators = 100)
 rfc_best.fit(X_train, y_train)
 rfc_best_pred = rfc_best.predict(X_test)
 print(accuracy_score(rfc_best_pred, y_test))
aptos19_stats =([0.42, 0.22, 0.075], [0.27, 0.15, 0.081] )<load_from_csv>
svc = SVC(random_state=42, gamma='scale')
 svc.fit(X_train, y_train)
 svc_pred = svc.predict(X_test)
 print(accuracy_score(svc_pred, y_test))
test_labels = pd.read_csv(path/'sample_submission.csv')
 test = ImageList.from_df(test_labels, path = path_test, suffix = '.png' )<split>
param = {'C': [1, 10, 100, 1000],
'gamma' :[1, 0.1, 0.001, 0.0001],
'kernel': ['linear', 'rbf']}
svm_grid = GridSearchCV(estimator=svc, param_grid=param, refit=True, cv=3, iid=False)
 svm_grid.fit(X_train,y_train)
 svm_grid_pred = svm_grid.predict(X_test)
 print(accuracy_score(svm_grid_pred, y_test))
src =(ImageList.from_df(labels, path = path_train, suffix = '.png')
.split_by_rand_pct(seed = 42)
.label_from_df(cols = 'diagnosis')
.add_test(test))<normalization>
svm_grid.best_params_
data =(
 src.transform(
 tfms,
size = 512,
resize_method=ResizeMethod.SQUISH,
padding_mode='zeros' 
)
.databunch(bs=8)
.normalize(aptos19_stats))<choose_model_class>
svm_best = SVC(random_state = 42,
C = 1,
gamma = 0.1,
kernel = 'rbf')
 svm_best.fit(X_train, y_train)
 svm_best_pred = svm_best.predict(X_test)
 print(accuracy_score(svm_best_pred, y_test))
kappa = KappaScore()
kappa.weights = "quadratic"<choose_model_class>
X_test_submit = df_final[df_final['Dataset'] == 'test'].drop(['Dataset', 'Survived'], axis=1)
 submit_pred = rfc_best.predict(X_test_submit)
 file_submit = pd.DataFrame({'PassengerId': df_test['PassengerId'].values,
'Survived': submit_pred.astype(np.int32)})
 file_submit.to_csv('titanic_submit_pred.csv', index=False )