Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<import_modules>
%matplotlib inline
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
X = pd.read_csv(".. /input/titanic/train.csv")
 X_test_full = pd.read_csv(".. /input/titanic/test.csv" )
import numpy as np
import pandas as pd
import os
import scipy as sp
from functools import partial
from sklearn import metrics
from collections import Counter
import json<compute_test_metric>
print('<<Training set>>
', X.isnull().sum())
 print('---------------------------')
 print('<<Test set>>
', X_test_full.isnull().sum() )
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']<train_model>
print('Percent of missing "Age" values on training dataset:', '%.2f%%' %(( X['Age'].isnull().sum() /X.shape[0])*100))
print('Percent of missing "Embarked" values on training dataset:', '%.2f%%' %(( X['Embarked'].isnull().sum() /X.shape[0])*100))
print('Percent of missing "Cabin" values on training dataset:', '%.2f%%' %(( X['Cabin'].isnull().sum() /X.shape[0])*100))
print('---------------------------')
 print('Percent of missing "Age" values on test dataset:', '%.2f%%' %(( X_test_full['Age'].isnull().sum() /X_test_full.shape[0])*100))
print('Percent of missing "Fare" values on test dataset:', '%.2f%%' %(( X_test_full['Fare'].isnull().sum() /X_test_full.shape[0])*100))
print('Percent of missing "Cabin" values on test dataset:', '%.2f%%' %(( X_test_full['Cabin'].isnull().sum() /X_test_full.shape[0])*100))
optR = OptimizedRounder()
optR.fit(valid_preds[0],valid_preds[1] )<load_from_csv>
women = X.loc[X.Sex == 'female']["Survived"]
rate_women = sum(women)/len(women)
 men = X.loc[X.Sex == 'male']["Survived"]
rate_men = sum(men)/len(men)
 print("Rate of women who survived:", "%.5f"% rate_women)
 print("Rate of men who survived:", "%.5f"% rate_men )
sample_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 sample_df.head()<predict_on_test>
first_class = X.loc[X.Pclass == 1]["Survived"]
first_rate = sum(first_class)/len(first_class)
 second_class = X.loc[X.Pclass == 2]["Survived"]
second_rate = sum(second_class)/len(second_class)
 third_class = X.loc[X.Pclass == 3]["Survived"]
third_rate = sum(third_class)/len(third_class)
 print("Rate of 1st class who survived:", "%.5f"% first_rate)
 print("Rate of 2nd class who survived:", "%.5f"% second_rate)
 print("Rate of 3rd class who survived:", "%.5f"% third_rate )
learn.data.add_test(ImageList.from_df(sample_df,'.. /input/aptos2019-blindness-detection',folder='test_images',suffix='.png')) 
<choose_model_class>
sibs = X.loc[X.SibSp == 1]["Survived"]
sibs_rate = sum(sibs)/len(sibs)
 parents = X.loc[X.Parch == 1]["Survived"]
parents_rate = sum(parents)/len(parents)
 print("Rate of survivors with siblings:", "%.5f"% sibs_rate)
 print("Rate of survivors with parents:", "%.5f"% parents_rate )
preds,y = learn.TTA(ds_type=DatasetType.Test)
<save_to_csv>
df = pd.DataFrame({'title':name[0], 'name':name[2], 'last_name' : name[1],
'survived': X['Survived'], 'sibsp': X['SibSp'], 'parch': X['Parch'],
'age': X['Age'], 'sex': X['Sex']})
 df_test = pd.DataFrame({'title':name_test[0], 'name':name_test[2], 'last_name' : name_test[1],
'age': X_test_full['Age'], 'sex':X_test_full['Sex']})
 df1 = df.copy()
df2 = df.copy()
for row in range(891):
if df1.loc[row,'sibsp']==0 and df1.loc[row,'parch']==0:
df1.drop([row], inplace=True)
 if df2.loc[row,'sibsp']!=0 or df2.loc[row,'parch']!=0:
df2.drop([row], inplace=True )
test_predictions = optR.predict(preds, coefficients)
 sample_df.diagnosis = test_predictions.astype(int)
 sample_df.head()
sample_df.to_csv('submission.csv',index=False )<set_options>
df1['last_name'].value_counts()
%matplotlib inline
warnings.filterwarnings('always')
 warnings.filterwarnings('ignore')
 print(os.listdir(".. /input"))<load_from_csv>
fam = df1["survived"]
fam_rate = sum(fam)/len(fam)
 lonely = df2["survived"]
lonely_rate = sum(lonely)/len(lonely)
 print("Rate of survivors with family onboard:", "%.5f"% fam_rate)
 print("Rate of survivors without family onboard:", "%.5f"% lonely_rate )
df_train = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 df_test = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 x_train = df_train['id_code']
y_train = df_train['diagnosis']<import_modules>
X.groupby('Pclass' ).Fare.mean()
import torch
import torch.utils.data
import torchvision<define_variables>
emb_s = X.loc[X.Embarked == 'S']["Survived"]
s_rate = sum(emb_s)/len(emb_s)
 emb_c = X.loc[X.Embarked == 'C']["Survived"]
c_rate = sum(emb_c)/len(emb_c)
 emb_q = X.loc[X.Embarked == 'Q']["Survived"]
q_rate = sum(emb_q)/len(emb_q)
 print("Rate of survivors that embarked from Southampton:", "%.5f"% s_rate)
 print("Rate of survivors that embarked from Cherbourg:","%.5f"% c_rate)
 print("Rate of survivors that embarked from Queenstown:","%.5f"% q_rate )
def get_label(diagnosis):
return ','.join([str(i)for i in range(diagnosis + 1)] )<feature_engineering>
X['Ticket'].value_counts()
df_train['label'] = df_train.diagnosis.apply(get_label )<set_options>
df.groupby('title' ).age.mean()
df_train.head(10)
 torch.cuda.manual_seed_all(13 )<normalization>
def replace_titles(x):
title = x['title']
if title in ['Don', 'Major', 'Capt', 'Jonkheer', 'Rev', 'Col', 'Sir']:
return 'Mr'
elif title in ['the Countess', 'Mme', 'Dona', 'Lady']:
return 'Mrs'
elif title in ['Mlle', 'Ms']:
return 'Miss'
elif title =='Dr':
if x['sex']=='male':
return 'Mr'
else:
return 'Mrs'
else:
return title
df['title']=df.apply(replace_titles, axis=1 )
tfms =([RandTransform(tfm=TfmCrop(crop_pad), kwargs={'row_pct':(0.4, 1), 'col_pct':(0.1, 0.9), 'padding_mode': 'reflection'}, p=1.0, resolved={}, do_run=True, is_random=True, use_on_y=True),
RandTransform(tfm=TfmPixel(rgb_randomize), kwargs={'channel':2, 'thresh':0.1}, p=0.75, resolved={}, do_run=True, is_random=True, use_on_y=True),
RandTransform(tfm=TfmPixel(rgb_randomize), kwargs={'channel':1, 'thresh':0.1}, p=0.75, resolved={}, do_run=True, is_random=True, use_on_y=True),
RandTransform(tfm=TfmAffine(dihedral_affine), kwargs={}, p=1.0, resolved={}, do_run=True, is_random=True, use_on_y=True),
RandTransform(tfm=TfmAffine(rotate), kwargs={'degrees':(-15.0, 15.0)}, p=0.75, resolved={}, do_run=True, is_random=True, use_on_y=True),
RandTransform(tfm=TfmAffine(zoom), kwargs={'scale':(1.0, 1.05), 'row_pct':(0, 1), 'col_pct':(0, 1)}, p=0.75, resolved={}, do_run=True, is_random=True, use_on_y=True),
RandTransform(tfm=TfmLighting(brightness), kwargs={'change':(0.4, 0.6)}, p=0.75, resolved={}, do_run=True, is_random=True, use_on_y=True),
RandTransform(tfm=TfmLighting(contrast), kwargs={'scale':(0.8, 1.25)}, p=0.75, resolved={}, do_run=True, is_random=True, use_on_y=True)],
[RandTransform(tfm=TfmCrop(crop_pad), kwargs={}, p=1.0, resolved={}, do_run=True, is_random=True, use_on_y=True)] )<load_pretrained>
X['Title'] = df['title']
data = ImageDataBunch.from_df('./',
df=df_train,
valid_pct=0.25,
folder=".. /input/aptos2019-blindness-detection/train_images",
suffix=".png",
ds_tfms=tfms,
size=224,
bs=156,
num_workers=32,
label_col='label', label_delim=',' ).normalize(imagenet_stats )<import_modules>
X_test_full.groupby('Pclass' ).Fare.mean()
print(f'Classes:
{data.classes}' )<predict_on_test>
X_test_full.loc[X_test_full.Fare.isnull() ]
def get_preds(arr):
mask = arr == 0
return np.clip(np.where(mask.any(1), mask.argmax(1), 5)- 1, 0, 4 )<define_variables>
X_test_full.Fare = X_test_full.Fare.fillna(12.46 )
last_output = torch.tensor([
[1.7226, 1.7226, 1.7226, 1.7226, 1.7226],
[0, 0, 0, 0, 1.7226],
[0.12841, -7.6266, -6.3899, -2.1333, -0.48995],
[0.68119, 1.7226, -1.9895, -0.097746, 0.53576]
])
 arr =(torch.sigmoid(last_output)> 0.5 ).numpy() ; arr<predict_on_test>
df_test.groupby('title' ).title.count()
assert(get_preds(arr)== np.array([4, 0, 0, 1])).all()<train_model>
df_test.loc[df_test.title.isin(['Col', 'Dona', 'Don', 'Dr', 'Ms', 'Rev'])]
class ConfusionMatrix(Callback):
"Computes the confusion matrix."
def on_train_begin(self, **kwargs):
self.n_classes = 0
def on_epoch_begin(self, **kwargs):
self.cm = None
def on_batch_end(self, last_output:Tensor, last_target:Tensor, **kwargs):
preds = torch.tensor(get_preds(( torch.sigmoid(last_output)> 0.5 ).cpu().numpy()))
targs = torch.tensor(get_preds(last_target.cpu().numpy()))
if self.n_classes == 0:
self.n_classes = last_output.shape[-1]
self.x = torch.arange(0, self.n_classes)
 cm =(( preds==self.x[:, None])&(targs==self.x[:, None, None])).sum(dim=2, dtype=torch.float32)
 if self.cm is None: self.cm = cm
else: self.cm += cm
def on_epoch_end(self, **kwargs):
self.metric = self.cm
@dataclass
class KappaScore(ConfusionMatrix):
"Compute the rate of agreement(Cohens Kappa )."
weights:Optional[str]=None
def on_epoch_end(self, last_metrics, **kwargs):
sum0 = self.cm.sum(dim=0)
 sum1 = self.cm.sum(dim=1)
 expected = torch.einsum('i,j->ij',(sum0, sum1)) / sum0.sum()
if self.weights is None:
w = torch.ones(( self.n_classes, self.n_classes))
w[self.x, self.x] = 0
elif self.weights == "linear" or self.weights == "quadratic":
w = torch.zeros(( self.n_classes, self.n_classes))
w += torch.arange(self.n_classes, dtype=torch.float)
 w = torch.abs(w - torch.t(w)) if self.weights == "linear" else(w - torch.t(w)) ** 2
else: raise ValueError('Unknown weights.Expected None, "linear", or "quadratic".')
 k = torch.sum(w * self.cm)/ torch.sum(w * expected)
 return add_metrics(last_metrics, 1-k )<init_hyperparams>
df_test.groupby('title' ).age.mean()
class Ranger(Optimizer):
def __init__(self, params, lr=1e-2, alpha=0.5, k=8, betas= (.9,0.999), eps=1e-8, weight_decay=0.1):
if not 0.0 <= alpha <= 1.0:
raise ValueError(f'Invalid slow update rate: {alpha}')
 if not 1 <= k:
raise ValueError(f'Invalid lookahead steps: {k}')
 if not lr > 0:
raise ValueError(f'Invalid Learning Rate: {lr}')
 if not eps > 0:
raise ValueError(f'Invalid eps: {eps}')
 defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)
 super().__init__(params,defaults)
 for group in self.param_groups:
group["step_counter"] = 0
self.alpha = alpha
self.k = k
self.radam_buffer = [[None,None,None] for ind in range(10)]
self.slow_weights = [[p.clone().detach() for p in group['params']]
for group in self.param_groups]
for w in it.chain(*self.slow_weights):
w.requires_grad = False
def __setstate__(self, state):
print("set state called")
 super(Ranger, self ).__setstate__(state)
 def step(self, closure=None):
loss = None
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data.float()
if grad.is_sparse:
raise RuntimeError('RAdam does not support sparse gradients')
 p_data_fp32 = p.data.float()
state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p_data_fp32)
 state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)
 else:
state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)
 state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
beta1, beta2 = group['betas']
exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 state['step'] += 1
buffered = self.radam_buffer[int(state['step'] % 10)]
if state['step'] == buffered[0]:
N_sma, step_size = buffered[1], buffered[2]
else:
buffered[0] = state['step']
beta2_t = beta2 ** state['step']
N_sma_max = 2 /(1 - beta2)- 1
N_sma = N_sma_max - 2 * state['step'] * beta2_t /(1 - beta2_t)
 buffered[1] = N_sma
if N_sma > 5:
step_size = group['lr'] * math.sqrt(( 1 - beta2_t)*(N_sma - 4)/(N_sma_max - 4)*(N_sma - 2)/ N_sma * N_sma_max /(N_sma_max - 2)) /(1 - beta1 ** state['step'])
 else:
step_size = group['lr'] /(1 - beta1 ** state['step'])
 buffered[2] = step_size
if group['weight_decay'] != 0:
p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)
 if N_sma > 5:
denom = exp_avg_sq.sqrt().add_(group['eps'])
 p_data_fp32.addcdiv_(-step_size, exp_avg, denom)
 else:
p_data_fp32.add_(-step_size, exp_avg)
 p.data.copy_(p_data_fp32)
 for group,slow_weights in zip(self.param_groups,self.slow_weights):
group['step_counter'] += 1
if group['step_counter'] % self.k != 0:
continue
for p,q in zip(group['params'],slow_weights):
if p.grad is None:
continue
q.data.add_(self.alpha,p.data - q.data)
 p.data.copy_(q.data)
 return loss
optar = partial(Ranger )<choose_model_class>
df_test['title']=df_test.apply(replace_titles, axis=1)
 print(df_test.groupby('title' ).title.count())
 print(df_test.groupby('title' ).age.mean())
 print(df_test.groupby('title' ).age.median() )
kappa = KappaScore(weights="quadratic")
 learn = cnn_learner(data, models.resnet50, metrics=[kappa, accuracy_thresh],
opt_func = optar,
callback_fns = [
partial(EarlyStoppingCallback, monitor='kappa_score', min_delta=0.001, patience=3),
partial(ReduceLROnPlateauCallback),
partial(SaveModelCallback, every = 'improvement', monitor='kappa_score', name='bestordinal')],
model_dir="/tmp" ).to_fp16().mixup(stack_y=False )<train_model>
X_test_full['Title'] = df_test['title']
lr = min_loss_lr
learn.fit_one_cycle(10, lr)
 torch.cuda.manual_seed_all(18 )<train_model>
def f(row):
if row['Age'] <= 16: val = 1
else: val = 0
return val
X['Minor'] = X.apply(f, axis=1)
 X_test_full['Minor'] = X_test_full.apply(f, axis=1)
 def f(row):
if row['Parch'] > 0 and row['Title'] == 'Mrs' and row['Age'] > 18: val = 1
else: val = 0
return val
X['Mother'] = X.apply(f, axis=1)
 X_test_full['Mother'] = X_test_full.apply(f, axis=1 )
learn.unfreeze()
learn.lr_find
lrs = learn.recorder.lrs
losses = learn.recorder.losses
mg =(np.gradient(np.array(losses)) ).argmin()
ml = np.argmin(losses[1:])
 min_grad_lr = lrs[mg]
print(min_grad_lr)
 min_loss_lr = lrs[ml]/10
print(min_loss_lr)
 lr2 = min_loss_lr
learn.unfreeze()
learn.fit_one_cycle(10, max_lr = lr2 )<load_pretrained>
X['Embarked'].fillna(X['Embarked'].value_counts().idxmax() , inplace=True)
 X['Deck'] = X['Cabin'].apply(lambda s: s[0] if pd.notnull(s)else 'U')
 X_test_full['Deck'] = X_test_full['Cabin'].apply(lambda s: s[0] if pd.notnull(s)else 'U')
 X['Deck'].unique()
X.drop('Cabin', axis=1, inplace=True)
 X_test_full.drop('Cabin', axis=1, inplace=True )
learn.load('bestordinal' )<train_model>
plt.figure(figsize=(10,5))
plt.title('Training dataset heatmap')
 X['Title'] = LabelEncoder().fit_transform(X['Title'])
 X['Embarked'] = X['Embarked'].astype('|S')
 X['Embarked'] = LabelEncoder().fit_transform(X['Embarked'])
 X['Deck'] = X['Deck'].astype('|S')
 X['Deck'] = LabelEncoder().fit_transform(X['Deck'])
 sns.heatmap(X.corr() , annot=True, cmap='coolwarm')
 plt.show()
learn.freeze()
learn.fit_one_cycle(15, max_lr=lr2/50,wd=1e-1 )<load_from_csv>
df['age'] = np.where(( df.age.isnull())&(df.title=="Master"),5,
np.where(( df.age.isnull())&(df.title=="Miss"),21,
np.where(( df.age.isnull())&(df.title=="Mr"),33,
np.where(( df.age.isnull())&(df.title=="Mrs"),36,
df.age))))
learn.load('bestordinal')
 sample_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 sample_df.head()<define_variables>
df.isnull().sum()
learn.data.add_test(ImageList.from_df(sample_df,'.. /input/aptos2019-blindness-detection',folder='test_images',suffix='.png'))<predict_on_test>
df[['title', 'survived']].groupby(['title'], as_index=False ).mean()
preds, y = learn.get_preds(DatasetType.Test )<count_values>
X['Age'] = df['age']
X.isnull().sum()
sample_df.diagnosis = get_preds(( preds > 0.5 ).cpu().numpy())
 sample_df.diagnosis.value_counts()<save_to_csv>
X.isnull().sum()
sample_df.to_csv('submission.csv',index=False )<import_modules>
plt.figure(figsize=(10,5))
plt.title('Test dataset heatmap')
 X_test_full['Title'] = LabelEncoder().fit_transform(X_test_full['Title'])
 X_test_full['Embarked'] = X_test_full['Embarked'].astype('|S')
 X_test_full['Embarked'] = LabelEncoder().fit_transform(X_test_full['Embarked'])
 X_test_full['Deck'] = X_test_full['Deck'].astype('|S')
 X_test_full['Deck'] = LabelEncoder().fit_transform(X_test_full['Deck'])
 sns.heatmap(X_test_full.corr() , annot=True, cmap='coolwarm')
 plt.show()
from fastai import *
from fastai.vision import *
import numpy as np
import scipy as sp
from sklearn import metrics
import cv2
import PIL<define_variables>
df_test['age'] = np.where(( df_test.age.isnull())&(df_test.title=="Master"),7,
np.where(( df_test.age.isnull())&(df_test.title=="Miss"),22,
np.where(( df_test.age.isnull())&(df_test.title=="Mr"),32,
np.where(( df_test.age.isnull())&(df_test.title=="Mrs"),39,
df_test.age))))
path = Path('/kaggle/input/aptos2019-blindness-detection' )<set_options>
df_test.isnull().sum()
path.ls()<load_from_csv>
X_test_full['Age'] = df_test['age'].copy()
X_test_full.isnull().sum()
df = pd.read_csv(path/'train.csv' )<count_values>
for df in([X, X_test_full]):
df['FamilySize'] = df.SibSp + df.Parch + 1
df['Fare'] = LabelEncoder().fit_transform(df['Fare'])
 df['Age'] = LabelEncoder().fit_transform(df['Age'])
 df.drop(['SibSp'], axis=1, inplace=True)
 df.drop(['Parch'], axis=1, inplace=True)
 df.drop(['Ticket'], axis=1, inplace=True)
 df.drop(['Name'], axis=1, inplace=True )
df.diagnosis.value_counts()<feature_engineering>
X['Deck_class'] = X['Pclass']*X['Deck']
X_test_full['Deck_class'] = X_test_full['Pclass']*X_test_full['Deck']
X['Emb_class'] = X['Pclass']*X['Embarked']
X_test_full['Emb_class'] = X_test_full['Pclass']*X_test_full['Embarked']
tfms = get_transforms(do_flip=True, flip_vert=True,max_warp=0., xtra_tfms =[crop_pad() ,symmetric_warp() ] )<normalization>
train_dummies = pd.get_dummies(X, columns=['Sex'])
 test_dummies = pd.get_dummies(X_test_full, columns=['Sex'])
 X = train_dummies.copy()
X_test_full = test_dummies.copy()
for df in([X, X_test_full]):
for col in(['Fare', 'Age', 'FamilySize', 'Deck', 'Title', 'Pclass', 'Deck_class', 'Emb_class']):
df[col] =(df[col]-df[col].mean())/np.std(df[col])
 df.drop(['Sex_male'], axis=1, inplace=True )
data =(
 src.transform(tfms,size=128)
.databunch() 
.normalize(imagenet_stats)
 )<compute_test_metric>
y = X.Survived
X.drop(['Survived'], axis=1, inplace=True)
 X_train, X_valid, y_train, y_valid = train_test_split(X, y, train_size=0.7, test_size=0.3, random_state=27 )
def quadratic_kappa(y_hat, y):
return torch.tensor(cohen_kappa_score(torch.round(y_hat), y, weights='quadratic'),device='cuda:0')
<choose_model_class>
from sklearn import metrics
from sklearn.metrics import confusion_matrix
from sklearn.metrics import roc_auc_score, accuracy_score, log_loss, auc
from sklearn.metrics import accuracy_score
import scikitplot as skplt
from sklearn.inspection import permutation_importance
import xgboost as xgb
from xgboost import XGBClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.naive_bayes import GaussianNB
from sklearn.model_selection import RandomizedSearchCV
from sklearn.model_selection import learning_curve
learn = cnn_learner(data,models.resnet152,metrics=[quadratic_kappa],model_dir='/kaggle',pretrained=True )<find_best_params>
Results = pd.DataFrame({'Model': [],'Accuracy': [], 'Recall':[],
'Precision':[], 'F1_score':[],
'Roc_Auc':[], 'Log_loss':[],
'Positive Samples':[],
})
 def model_evaluators(y_valid, preds, preds_proba, modelName):
tn, fp, fn, tp = confusion_matrix(y_valid, preds ).ravel()
acc_ =(tp + tn)/(tp + tn + fn + fp)
 sens_ = tp /(tp + fn)
 sp_ = tn /(tn + fp)
 PPV = tp /(tp + fp)
 NPV = tn /(tn + fn)
 F1 = 2*(PPV*sens_)/(PPV+sens_)
 fpr, tpr, thr = metrics.roc_curve(y_valid, preds_proba)
 rocauc = auc(fpr, tpr)
 log_loss_ = log_loss(y_valid, preds_proba)
 correct_guesses = accuracy_score(y_valid, preds, normalize=False)
 res = pd.DataFrame({'Model': [modelName],
'Accuracy': ["%.3f"% acc_],
'Recall':["%.3f"% sens_],
'Precision':["%.3f"% PPV],
'F1_score':["%.3f"% F1],
'Roc_Auc':["%.3f"% rocauc],
'Log_loss':["%.3f"% log_loss_],
'Positive Samples':[correct_guesses]
})
 return res
learn.lr_find() 
<train_model>
predictors = [i for i in X_train.columns if i not in ['PassengerId']]
def modelfit_eval(clf, X_train, y_train, X_valid, y_valid, predictors, clf_name):
model = clf
model.fit(X_train[predictors], y_train)
 preds = model.predict(X_valid[predictors])
 preds_proba = model.predict_proba(X_valid[predictors])[:, 1]
model_xgb = model
res = model_evaluators(y_valid, preds, preds_proba, clf_name)
 return model, res
model_xgb, res = modelfit_eval(XGBClassifier() , X_train, y_train, X_valid, y_valid, predictors, 'XGB Classifier')
 Results = Results.append(res, ignore_index=True)
 model_log, res = modelfit_eval(LogisticRegression(solver='liblinear'), X_train, y_train, X_valid, y_valid, predictors, 'Logistic Regression')
 Results = Results.append(res, ignore_index=True)
 model_dtree, res = modelfit_eval(DecisionTreeClassifier() , X_train, y_train, X_valid, y_valid, predictors, 'Decision Tree Classifier')
 Results = Results.append(res, ignore_index=True)
 model_rf, res = modelfit_eval(RandomForestClassifier() , X_train, y_train, X_valid, y_valid, predictors, 'Random Forest Classifier')
 Results = Results.append(res, ignore_index=True)
 model_svm, res = modelfit_eval(SVC(gamma=0.001, probability=True), X_train, y_train, X_valid, y_valid, predictors, 'SVC')
 Results = Results.append(res, ignore_index=True)
 model_nb, res = modelfit_eval(GaussianNB() , X_train, y_train, X_valid, y_valid, predictors, 'Gaussian Naive Bayes')
 Results = Results.append(res, ignore_index=True)
 Results
lr=1e-2
learn.fit_one_cycle(3,lr )<categorify>
def modelfit(model, X_train, y_train, predictors, useTrainCV=True, cv_folds=5, early_stopping_rounds=50):
if useTrainCV:
xgb_param = model.get_xgb_params()
xgtrain = xgb.DMatrix(X_train[predictors].values,
label=y_train.values)
 cvresult = xgb.cv(xgb_param,
xgtrain,
num_boost_round=model.get_params() ['n_estimators'],
nfold=cv_folds,
metrics='auc',
early_stopping_rounds=early_stopping_rounds)
 model.set_params(n_estimators=cvresult.shape[0])
 print("Best iteration:", cvresult.shape[0])
 print(cvresult.tail(1))
model.fit(X_train[predictors], y_train, eval_metric='auc')
 predictions = model.predict(X_train[predictors])
 predprob = model.predict_proba(X_train[predictors])[:,1]
print("
Model Report")
 print("Accuracy : %.4g" % metrics.accuracy_score(y_train.values, predictions))
print("AUC Score(Train): %f" % metrics.roc_auc_score(y_train, predprob))
feat_imp = pd.Series(model.get_booster().get_fscore() ).sort_values(ascending=False)
 feat_imp.plot(kind='bar', title='Feature Importances')
 plt.ylabel('Feature Importance Score' )
learn.data = data =(src.transform(tfms,size=256 ).databunch().normalize(imagenet_stats))<train_model>
xgb1 = XGBClassifier(learning_rate =0.1,
n_estimators=1000,
max_depth=4,
min_child_weight=0,
gamma=0,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27)
 modelfit(xgb1, X_train, y_train, predictors )
learn.fit_one_cycle(5,max_lr=slice(1e-4))<find_best_params>
param_test1 = {'max_depth':range(3,10,1),
'min_child_weight':range(0,6,1)}
gsearch1 = GridSearchCV(estimator = XGBClassifier(learning_rate =0.1,
n_estimators=3,
max_depth=4,
min_child_weight=0,
gamma=0,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27),
param_grid = param_test1,
scoring='roc_auc',
n_jobs=4,
cv=5)
 gsearch1.fit(X_train[predictors],y_train)
 def grid_scores(model):
print("Best parameters set found on development set:")
 print(model.best_params_)
 print("
Grid scores on development set:")
 means = model.cv_results_['mean_test_score']
stds = model.cv_results_['std_test_score']
for mean, std, params in zip(means, stds, model.cv_results_['params']):
print("%0.3f(+/-%0.03f)for %r"
%(mean, std * 2, params))
grid_scores(gsearch1 )
learn.lr_find()<train_model>
param_test3 = {'gamma':[i/10.0 for i in range(0,10)]}
gsearch3 = GridSearchCV(estimator = XGBClassifier(learning_rate =0.1,
n_estimators=3,
max_depth=4,
min_child_weight=1,
gamma=0,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27),
param_grid = param_test3,
scoring='roc_auc',
n_jobs=4,
cv=5)
 gsearch3.fit(X_train[predictors],y_train)
 grid_scores(gsearch3 )
learn.fit_one_cycle(5,max_lr=9e-04 )<predict_on_test>
xgb2 = XGBClassifier(learning_rate =0.1,
n_estimators=1000,
max_depth=4,
min_child_weight=1,
gamma=0.6,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27)
 modelfit(xgb2, X_train, y_train, predictors )
valid_preds = learn.get_preds(ds_type = DatasetType.Valid )<compute_test_metric>
param_test4 = {'subsample':[i/10.0 for i in range(6,10)],
'colsample_bytree':[i/10.0 for i in range(6,10)]}
gsearch4 = GridSearchCV(estimator = XGBClassifier(learning_rate =0.1,
n_estimators=8,
max_depth=4,
min_child_weight=1,
gamma=0.6,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27),
param_grid = param_test4,
scoring='roc_auc',
n_jobs=4,
cv=5)
 gsearch4.fit(X_train[predictors], y_train)
 grid_scores(gsearch4 )
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']<train_model>
param_test5 = {'subsample':[i/100.0 for i in range(85,100,5)],
'colsample_bytree':[i/100.0 for i in range(75,90,5)]}
gsearch5 = GridSearchCV(estimator = XGBClassifier(learning_rate =0.1,
n_estimators=8,
max_depth=4,
min_child_weight=1,
gamma=0.6,
subsample=0.9,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27),
param_grid = param_test5,
scoring='roc_auc',
n_jobs=4,
cv=5)
 gsearch5.fit(X_train[predictors],y_train)
 grid_scores(gsearch5 )
optR = OptimizedRounder()
optR.fit(valid_preds[0],valid_preds[1] )<load_from_csv>
param_test6 = {'reg_alpha':[1e-5, 0.0005, 1e-2, 0.1, 0.5, 1, 100]}
gsearch6 = GridSearchCV(estimator = XGBClassifier(learning_rate =0.1,
n_estimators=8,
max_depth=4,
min_child_weight=1,
gamma=0.6,
subsample=0.9,
colsample_bytree=0.75,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27),
param_grid = param_test6,
scoring='roc_auc',
n_jobs=4,
cv=5)
 gsearch6.fit(X_train[predictors],y_train)
 grid_scores(gsearch6 )
sample_df = pd.read_csv(path/'sample_submission.csv' )<define_variables>
param_test7 = {'reg_alpha':[1e-07, 1e-06, 1e-05, 1e-04, 1e-03, 1e-02]}
gsearch7 = GridSearchCV(estimator = XGBClassifier(learning_rate =0.1,
n_estimators=8,
max_depth=4,
min_child_weight=1,
gamma=0.6,
subsample=0.9,
colsample_bytree=0.75,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27),
param_grid = param_test7,
scoring='roc_auc',
n_jobs=4,
cv=5)
 gsearch7.fit(X_train[predictors],y_train)
 grid_scores(gsearch7 )
learn.data.add_test(ImageList.from_df(sample_df,path,folder='test_images',suffix='.png'))<predict_on_test>
xgb3 = XGBClassifier(learning_rate =0.1,
n_estimators=8,
max_depth=4,
min_child_weight=1,
gamma=0.6,
subsample=0.9,
colsample_bytree=0.75,
reg_alpha=1e-07,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27)
 modelfit(xgb3, X_train, y_train, predictors )
preds,y = learn.get_preds(DatasetType.Test )<predict_on_test>
xgb4 = XGBClassifier(learning_rate =0.001,
n_estimators=200,
max_depth=4,
min_child_weight=1,
gamma=0.6,
subsample=0.9,
colsample_bytree=0.75,
reg_alpha=1e-07,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27)
 modelfit(xgb4, X_train, y_train, predictors )
test_predictions = optR.predict(preds,coefficients )<data_type_conversions>
model_xgb = xgb4
model_xgb.fit(X_train[predictors], y_train)
 preds = model_xgb.predict(X_valid[predictors])
 preds_proba = model_xgb.predict_proba(X_valid[predictors])[:, 1]
res = model_evaluators(y_valid, preds, preds_proba, 'Hyperparametarized XGB')
 Results = Results.append(res, ignore_index=True)
 Results
sample_df.diagnosis = test_predictions.astype(int)
 sample_df.head()<save_to_csv>
rf_grid_param = {'n_estimators': [100, 200, 300, 400],
'max_features': ['auto', 'sqrt', 'log2'],
'max_depth': [4, 5, 7, None],
'min_samples_split': [2, 3, 5, 7],
'min_samples_leaf': [1, 3, 5, 7]}
dt_grid_param = {'criterion': ['gini', 'entropy'],
'max_features': ['auto', 'sqrt', 'log2'],
'max_depth': [4, 5, 7, None],
'min_samples_split': [2, 3, 5, 7],
'min_samples_leaf': [1, 3, 5, 7]}
log_grid_param = {'penalty': ['l1', 'l2'],
'C': np.logspace(-4, 4, 20),
'solver': ['liblinear', 'saga']}
nb_grid_param = {'var_smoothing': np.logspace(0,-9, num=100)}
model_rf = RandomizedSearchCV(estimator=model_rf,
param_distributions=rf_grid_param,
n_iter=300,
cv =5,
scoring='roc_auc',
verbose=2,
random_state=27,
n_jobs = -1)
 model_dtree = RandomizedSearchCV(estimator=model_dtree,
param_distributions=dt_grid_param,
n_iter=300,
cv =5,
scoring='roc_auc',
verbose=1,
random_state=27,
n_jobs = -1)
 model_log = RandomizedSearchCV(estimator=model_log,
param_distributions =log_grid_param,
n_iter=300,
cv = 5,
scoring='roc_auc',
verbose=1,
random_state=27,
n_jobs=-1)
 model_nb = RandomizedSearchCV(estimator=model_nb,
param_distributions=nb_grid_param,
n_iter=300,
cv = 5,
scoring='roc_auc',
verbose=1,
random_state=27,
n_jobs=-1)
 preds_list = []
for model in [model_rf, model_dtree, model_log, model_nb]:
model.fit(X_train[predictors], y_train)
 model = model.best_estimator_
preds = model.predict(X_valid[predictors])
 preds_proba = model.predict_proba(X_valid[predictors])[:, 1]
clf_name = str(model ).split('(')[0]
res = model_evaluators(y_valid, preds, preds_proba, 'Hyperparametarized '+clf_name)
 Results = Results.append(res, ignore_index=True)
 preds_list.append(preds )
sample_df.to_csv('submission.csv',index = False )<define_variables>
preds_proba = model_rf.best_estimator_.predict_proba(X_valid[predictors])[:, 1]
X_valid['preds_proba'] = preds_proba
X_valid['preds'] = preds_list[0]
X_valid['true_preds'] = y_valid
wrong_preds = X_valid.loc[(( X_valid.preds == 1)&(X_valid.true_preds == 0)) | 
(( X_valid.preds == 0)&(X_valid.true_preds == 1)) ].sort_values(by=['preds'],
ascending=False)
 X_valid.drop('preds_proba', axis=1, inplace=True)
 X_valid.drop('preds', axis=1, inplace=True)
 X_valid.drop('true_preds', axis=1, inplace=True )
train_csv = '.. /input/aptos2019-blindness-detection/train.csv'
image_dataset = '.. /input/aptos2019-blindness-detection/train_images'
sample_csv = '.. /input/aptos2019-blindness-detection/sample_submission.csv'
test_dataset = '.. /input/aptos2019-blindness-detection/test_images'
resnet_weights = '.. /input/resnet50-image-weights/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5'
xception_weights = '.. /input/xception-weights/xception_weights_tf_dim_ordering_tf_kernels_notop.h5' 
<compute_test_metric>
print('Total Error examples:', wrong_preds.shape[0])
 print('-------------------------')
 print(wrong_preds['preds'].value_counts())
 print('-------------------------')
 print(wrong_preds['Sex_female'].value_counts())
 print('-------------------------')
 print(wrong_preds['Embarked'].value_counts())
 print('-------------------------')
 print(wrong_preds['Pclass'].value_counts())
 print('-------------------------')
 print(wrong_preds['Title'].value_counts())
 print('-------------------------')
 print(wrong_preds['Deck'].value_counts())
 print('-------------------------')
 print(wrong_preds['FamilySize'].value_counts() )
def cohens_kappa(y_true, y_pred):
y_true_classes = tf.argmax(y_true, 1)
 y_pred_classes = tf.argmax(y_pred, 1)
 return tf.contrib.metrics.cohen_kappa(y_true_classes, y_pred_classes, 5)[1]<import_modules>
print('Unknown Decks and Lonely Travellers:',
wrong_preds.loc[(( wrong_preds.Deck < 0.522068)&(wrong_preds.Deck > 0.522066)) & 
(( wrong_preds.FamilySize < -0.560974)&(wrong_preds.FamilySize > -0.560976)) ].shape[0])
 print('Unknown Decks Lonely Travellers embarked from Southampton:',
wrong_preds.loc[(( wrong_preds.Deck < 0.522068)&(wrong_preds.Deck > 0.522066)) & 
(wrong_preds.Embarked == 2)& 
(( wrong_preds.FamilySize < -0.560974)&(wrong_preds.FamilySize > -0.560976)) ].shape[0])
 print('Mr.title and Lonely Traveller:',
wrong_preds.loc[(( wrong_preds.Title < 0.214012)&(wrong_preds.Title > 0.214010)) & 
(( wrong_preds.FamilySize < -0.560974)&(wrong_preds.FamilySize > -0.560976)) ].shape[0])
 print('Unknown Deck and Embarked from S:',
wrong_preds.loc[(( wrong_preds.Deck < 0.522068)&(wrong_preds.Deck > 0.522066)) & 
(wrong_preds.Embarked == 2)].shape[0] )
img = Input(shape=(224,224,3))
base_model = Xception(include_top = False,
weights=xception_weights,
input_tensor=img,
pooling='avg',
input_shape = None)
 final_layer = base_model.layers[-1].output
final_layer = Dropout(0.2 )(final_layer)
 dense_layer1 = Dense(512, activation='relu' )(final_layer)
 dense_layer1 = Dropout(0.3 )(dense_layer1)
 dense_layer2 = Dense(128,activation= 'relu' )(dense_layer1)
 dense_layer2 = Dropout(0.3 )(dense_layer2)
 output_layer = Dense(5,activation='softmax' )(dense_layer2)
 model = Model(img,output_layer)
 for layer in base_model.layers:
layer.trainable = True
model.compile(optimizer = 'adam',loss = 'categorical_crossentropy',metrics = ['accuracy'])
 model.summary()<choose_model_class>
def f(row):
if(row['Title'] > 0.214010 and row['Title'] < 0.214012)and(row['FamilySize'] > -0.560976 and row['FamilySize'] < -0.560974): val = 1
else: val = 0
return val
X['Mr_lonely'] = X.apply(f, axis=1)
 X_test_full['Mr_lonely'] = X_test_full.apply(f, axis=1)
 def f(row):
if(row['Deck'] > 0.522066 and row['Deck'] < 0.522068)and row['Embarked'] == 2: val = 1
else: val = 0
return val
X['Unknown_s'] = X.apply(f, axis=1)
 X_test_full['Unknown_s'] = X_test_full.apply(f, axis=1 )
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
min_delta=0.0004,
patience=5,
factor=0.5,
min_lr=1e-6,
mode='auto',
verbose=1 )<choose_model_class>
X_train, X_valid, y_train, y_valid = train_test_split(X, y, train_size=0.7, test_size=0.3, random_state=27)
 rf_grid_param = {'n_estimators': [200, 300, 400, 700],
'max_features': ['auto', 'sqrt', 'log2'],
'max_depth': [4, 5, 7, None],
'min_samples_split': [2, 3, 5, 7],
'min_samples_leaf': [1, 3, 5, 7]}
model_rf, _ = modelfit_eval(RandomForestClassifier() , X_train, y_train, X_valid, y_valid, predictors, 'Random Forest Classifier')
 model_rf = RandomizedSearchCV(estimator=model_rf,
param_distributions=rf_grid_param,
n_iter=300,
cv =5,
scoring='roc_auc',
verbose=2,
random_state=27,
n_jobs = -1)
 model_rf.fit(X_train[predictors], y_train)
 model = model_rf.best_estimator_
preds = model.predict(X_valid[predictors])
 preds_proba = model.predict_proba(X_valid[predictors])[:, 1]
res = model_evaluators(y_valid, preds, preds_proba, 'Random Forest after Error Analysis')
 Results = Results.append(res, ignore_index=True )
early_stop = EarlyStopping(monitor='val_loss',
min_delta=0.0001,
patience=10,
verbose=1,
mode='auto' )<feature_engineering>
preds_test = model.predict(X_test_full[predictors])
 output = pd.DataFrame({'PassengerId': X_test_full.PassengerId, 'Survived': preds_test})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )