Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<import_modules>
%%markdown
Titanic competition in Kaggle *https://www.kaggle.com/c/titanic/*
import random
import numpy as np
import pandas as pd
import torch
import PIL.Image as pil
import matplotlib.pyplot as plt
from fastai.vision import *
from efficientnet_pytorch import EfficientNet
from sklearn.model_selection import StratifiedKFold
import os<set_options>
%matplotlib inline
warnings.filterwarnings("ignore", category=UserWarning, module="torch.nn.functional" )<install_modules>
input_path = '/kaggle/input/titanic/'
train_set = pd.read_csv(input_path+'train.csv')
 test_set = pd.read_csv(input_path+'test.csv')
 dataset = [train_set, test_set]
!pip install torch==1.4.0 torchvision==0.5.0<set_options>
%%markdown
%reload_ext autoreload
%autoreload 2
%matplotlib inline<set_options>
def missing_values_df(df):

 missing_values = df.isnull().sum().sort_values(ascending = False)
 missing_values = missing_values[missing_values>0]
ratio = missing_values/len(df)*100
output_df= pd.concat([missing_values, ratio], axis=1, keys=['Total missing values', 'Percentage'])
 return output_df
print('Missing values in the columns of training dataset with percentage')
 display(missing_values_df(train_set))
print('
------------------------------------------------
')
 print('Missing values in the columns of test dataset with percentage')
 display(missing_values_df(test_set))
seed = 42
def random_seed(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
random_seed(seed )<define_variables>
%%markdown
1)fill the *NaN* values with mean(*Age feature*)our more frequent values(*Embarked feature*)
 2)Add *titles* of passengers from names then delete names
3)Encode categorical features into integers
4)make 4 bins of age to categorize it
5)Normalize our datasets(training_set and test_set)
 6)Split our training set into training and validation sets(80%/20%)
path = '/kaggle/input/siim-isic-melanoma-classification'
path<define_variables>
for i in range(len(dataset)) :
freq_port = dataset[i]['Embarked'].dropna().mode() [0]
dataset[i]['Embarked'] = dataset[i]['Embarked'].fillna(freq_port)
 dataset[i] = dataset[i].fillna(dataset[i].mean() )
img_path = '/kaggle/input/melanoma-merged-external-data-512x512-jpeg'
img_path<load_from_csv>
print("Titels of passengers by sex")
 dataset[0]['Title'] = dataset[0].Name.str.extract('([A-Za-z]+)\.', expand=False)
 display(pd.crosstab(dataset[0]['Sex'], dataset[0]['Title']))
train_df = pd.read_csv(img_path + '/folds_13062020.csv')
 train_df.head()<load_from_csv>
for i, data in enumerate(dataset):
dataset[i]['Title'] = data.Name.str.extract('([A-Za-z]+)\.', expand=False)
 dataset[i]['Title'] = data['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev',
'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset[i]['Title'] = data['Title'].replace(['Mlle', 'Ms'], 'Miss')
 dataset[i]['Title'] = data['Title'].replace('Mme', 'Mrs')
 print(dataset[i]['Title'].unique() )
test_df = pd.read_csv(path + '/test.csv')
 test_df.head()<load_from_csv>
encoder = LabelEncoder()
categoricalFeatures = dataset[0].select_dtypes(include=['object'] ).columns
for i, data in enumerate(dataset):
data[categoricalFeatures]=data[categoricalFeatures].astype(str)
 encoded = data[categoricalFeatures].apply(encoder.fit_transform)
 for j in categoricalFeatures:
dataset[i][j]=encoded[j]
dataset[0].head()
sample_df = pd.read_csv(path + '/sample_submission.csv')
 sample_df.head()<feature_engineering>
bins = [0,18,60,80]
labels = [1,2,3]
for i, data in enumerate(dataset):
dataset[i] = dataset[i].drop(['PassengerId', 'Name', 'Ticket', 'Cabin'], axis=1)
 dataset[i]['Age']=pd.cut(dataset[i]['Age'],bins=bins ,labels=labels)
 dataset[i]['Age']=dataset[i]['Age'].astype('int64')
 print('training dataset:')
 display(dataset[0].head())
 print('testing dataset:')
 display(dataset[1].head() )
tfms = get_transforms(flip_vert=True, max_rotate=15, max_zoom=1.2, max_lighting=0.3, max_warp=0, p_affine=0, p_lighting=0.8 )<compute_train_metric>
X=dataset[0].iloc[:, 1:]
Y=dataset[0].iloc[:, 0]
x_test=dataset[1].iloc[:, 0:]
normalized_data = X
normalized_data=normalized_data.append(x_test)
 normalized_x_train = normalized_data.values
normalized_x_train /= np.max(np.abs(normalized_x_train),axis=0)
 X = pd.DataFrame(normalized_x_train[:891,:],
columns=['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked', 'Title'])
 print(X.head())
 print(len(X))
x_test = pd.DataFrame(normalized_x_train[891:,:],
columns=['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked', 'Title'])
 display(x_test.head())
 print(len(x_test))
display(X.columns )
class FocalLoss(nn.Module):
def __init__(self, gamma=2., reduction='mean'):
super().__init__()
self.gamma = gamma
self.reduction = reduction
def forward(self, inputs, targets):
CE_loss = nn.CrossEntropyLoss(reduction='none' )(inputs, targets)
 pt = torch.exp(-CE_loss)
 F_loss =(( 1 - pt)**self.gamma)* CE_loss
if self.reduction == 'sum':
return F_loss.sum()
elif self.reduction == 'mean':
return F_loss.mean()<init_hyperparams>
X_train, X_val, y_train, y_val = train_test_split(X, Y, test_size = 0.20)
 print("Training set shape: "+str(X_train.shape))
print("Validation set shape: "+str(X_val.shape))
submission_ver = '0002'
arch = [EfficientNet.from_pretrained('efficientnet-b0', num_classes=2)]
fc_size = 1280
lin_size = 1000
n_folds = 5
size = [256]
bs = 32
stage_1_epochs = 3
lr1 = [1e-1]
lr_eff_1 = [1e-3]
is_stage_2 = False
stage_2_epochs = 4
lr2 = [slice(1e-7, 1e-4)]
lr_eff_2 = [slice(1e-4, 1e-3)]
custom_loss = True
loss_func = FocalLoss()
w_decay = 0.01
use_fp16 = True
num_wkrs = os.cpu_count()
use_tta = True
submit_after_train = False
oversampling_flag = True
oversampling_size = 5
preds_to_int = False
smooth_preds = False
smooth_alpha = 0.01<count_unique_values>
%%markdown
1)Logistic Regression
2)Decision Tree
3)Random Forest
4)XGBoost
num_classes = len(np.unique(train_df['target']))
num_classes<feature_engineering>
accuracies_list = list()
accuracies = namedtuple('accuracies',('Model', 'accuracy'))
test_df['image_name'] = '512x512-test/512x512-test/' + test_df['image_name'] + '.jpg'<create_dataframe>
%%markdown
test_data = ImageList.from_df(test_df, img_path)
 test_data<prepare_output>
logreg = LogisticRegression()
logreg.fit(X_train, y_train)
 Y_pred = logreg.predict(X_val)
 acc_log = round(logreg.score(X_train, y_train)* 100, 2)
 print('accuracy: {}'.format(acc_log))
accuracies_list.append(accuracies('Logistic Regression', acc_log))
labels_df = train_df[['image_id', 'target']].copy()
labels_df.head()<categorify>
%%markdown
def k_fold(df, num_fld, seed = seed):
for fold in range(num_fld):
df.loc[df.fold == fold, f'is_valid_{fold}'] = True
df.loc[df.fold != fold, f'is_valid_{fold}'] = False<concatenate>
decisiontree = DecisionTreeClassifier()
decisiontree.fit(X_train, y_train)
 y_pred = decisiontree.predict(X_val)
 acc_decisiontree = round(accuracy_score(y_pred, y_val)* 100, 2)
 print('accuracy: {}'.format(acc_decisiontree))
accuracies_list.append(accuracies('Decision Tree', acc_decisiontree))
k_fold(train_df, n_folds, seed )<filter>
%%markdown
def oversample(fld, df, os_size, num_fld=5):
train_df_fld = df.loc[df['fold'] != fld]
valid_df_fld = df.loc[df['fold'] == fld]
train_df_md = train_df_fld.loc[train_df_fld['target'] == 1]
if os_size == 'auto':
os_size = int(np.floor(train_df_fld.loc[train_df_fld['target'] == 0]['target'].value_counts() [0]/train_df_fld.loc[train_df_fld['target'] == 1]['target'].value_counts() [1]))
train_df_md = train_df_md.append([train_df_md] *(os_size - 1))
full_df_fld = pd.concat([train_df_fld, train_df_md, valid_df_fld] ).sample(frac=1.0, random_state=seed ).reset_index(drop=True)
 return full_df_fld
for x in range(n_folds):
fold_df = train_df.copy()
if oversampling_flag:
fold_df = oversample(x, train_df, oversampling_size)
 globals() ['src_%s' %x] =(ImageList.from_df(fold_df, img_path + '/512x512-dataset-melanoma', folder='512x512-dataset-melanoma', suffix='.jpg' ).split_from_df(col =(3 + x)))
<categorify>
clf = RandomForestClassifier(max_depth=10, max_leaf_nodes =20,random_state=0)
 clf.fit(X_train,y_train)
 y_pred=clf.predict(X_val)
 acc_random_forest = round(accuracy_score(y_pred, y_val)* 100, 2)
 print('accuracy: {}'.format(acc_random_forest))
accuracies_list.append(accuracies('Random Forest', acc_random_forest))
def get_data(fold, size, bs, padding_mode='reflection'):
return(globals() ['src_%s' %fold].label_from_df(cols='target')
.add_test(test_data)
.transform(tfms, size=size, padding_mode=padding_mode)
.databunch(bs=bs, num_workers = num_wkrs ).normalize(imagenet_stats))<categorify>
%%markdown
def preds_smoothing(encodings , alpha):
K = encodings.shape[1]
y_ls =(1 - alpha)* encodings + alpha / K
return y_ls<compute_test_metric>
xgb = xgboost.XGBClassifier(random_state=5,learning_rate=0.01)
 xgb.fit(X_train, y_train)
 y_pred = xgb.predict(X_val)
 acc_xgb = round(accuracy_score(y_pred, y_val)* 100, 2)
 print('accuracy: {}'.format(acc_xgb))
accuracies_list.append(accuracies('XGBoost', acc_xgb))
def print_metrics(val_preds, val_labels):
targs, preds = LongTensor([]), Tensor([])
 val_preds = F.softmax(val_preds, dim=1)[:,-1]
preds = torch.cat(( preds, val_preds.cpu()))
targs = torch.cat(( targs, val_labels.cpu().long()))
print('AUCROC = ' + str(auc_roc_score(preds, targs ).item()))<set_options>
%%markdown
1)Declare consts
2)Training Set && Testing Set preparation for pytorch
3)Define our DL model class
4)Instantiate our model, loss and optimizer
5)Define fit function
6)Training process
7)Define Predict Function
8)Preprare for submission
gc.collect()<feature_engineering>
BATCH_SIZE = 1
LEARNING_RATE = 0.001
EPOCHS = 800
INPUT_NODES = 8
for model in arch:
if hasattr(model, '__name__'):
model_name = model.__name__
else:
model_name = "EfficientNet"
globals() [model_name + '___val_preds'] = []
globals() [model_name + '___val_labels'] = []
globals() [model_name + '___test_preds'] = []
print(f'/////////////////////////////////////////////////////')
 print(f'//////////////// MODEL: {model_name} ////////////////')
 print(f'/////////////////////////////////////////////////////
')
 for fld in range(n_folds):
print(f'
//////////////// FOLD {fld} ////////////////
')
 for sz in size:
print(f'-------- Size: {sz} --------
')
 print("Preparing data & applying settings...
 ")
 data = get_data(fld, sz, bs)
 gc.collect()
if sz == size[0]:
if model_name != "EfficientNet":
learn = cnn_learner(data, model, metrics=[AUROC() ], bn_final=True)
 else:
model._fc = nn.Sequential(nn.Linear(fc_size, lin_size, bias=True),
nn.ReLU() ,
nn.Dropout(p=0.5),
nn.Linear(lin_size, num_classes, bias = True))
learn = Learner(data, model, metrics=[AUROC() ])
 learn = learn.split([learn.model._conv_stem,learn.model._blocks,learn.model._conv_head])
 else:
learn.data = data
if custom_loss:
learn.loss_func = loss_func
if use_fp16:
learn = learn.to_fp16()
if model_name != "EfficientNet":
lr = lr1[size.index(sz)]
else:
lr = lr_eff_1[size.index(sz)]
print("Data is ready.Learning - Stage 1...")
 learn.fit_one_cycle(stage_1_epochs, slice(lr), wd=w_decay)
 if is_stage_2:
print("Stage 1 complete.Stage 2...")
 if model_name != "EfficientNet":
lr = lr2[size.index(sz)]
else:
lr = lr_eff_2[size.index(sz)]
learn.unfreeze()
learn.fit_one_cycle(stage_2_epochs, lr, wd=w_decay)
 print(f"Final learning is over for size {sz}
")
 val_preds, val_labels = learn.get_preds()
print_metrics(val_preds, val_labels)
 globals() [model_name + '___val_preds'].append(val_preds)
 globals() [model_name + '___val_labels'].append(val_labels)
 if use_tta == False:
print(f'
Saving test results for fold {fld}...')
 test_preds, _ = learn.get_preds(DatasetType.Test)
 globals() [model_name + '___test_preds'].append(test_preds[:, 1])
 else:
print(f'
Saving test TTA results for fold {fld}...')
 test_preds, _ = learn.TTA(ds_type=DatasetType.Test)
 globals() [model_name + '___test_preds'].append(test_preds[:, 1])
 print("Done!")
 gc.collect()
print("All folds are trained successfully
")
 print_metrics(torch.cat(globals() [model_name + '___val_preds']), torch.cat(globals() [model_name + '___val_labels']))
print("
Writing submission file...")
 test_df_output = pd.concat([test_df, pd.DataFrame(np.mean(np.stack(globals() [model_name + '___test_preds']), axis=0), columns=['target'])], axis=1)
 sample_df.iloc[:,1:] = test_df_output.iloc[:,5]
sample_df.to_csv(f'submission_v{submission_ver}.csv', index=False)
 print(f'File is ready to submit
')
 if submit_after_train:
print("Submitting to Kaggle
")
 !kaggle competitions submit -c siim-isic-melanoma-classification -f 'submission_v{submission_ver}.csv' -m 'Md: {model_name}, Fd: {n_folds}, Bs: {bs}, Sz: {size[0]}, Os: {oversampling_flag}, TTa: {use_tta}'
print('
')
<set_options>
X_train_torch = torch.from_numpy(X_train.values ).type(torch.FloatTensor)
 y_train_torch = torch.from_numpy(y_train.values ).type(torch.LongTensor)
 X_val_torch = torch.from_numpy(X_val.values ).type(torch.FloatTensor)
 y_val_torch = torch.from_numpy(y_val.values ).type(torch.LongTensor)
 x_test_torch = torch.from_numpy(x_test.values ).type(torch.FloatTensor)
 train = torch.utils.data.TensorDataset(X_train_torch,y_train_torch)
 val = torch.utils.data.TensorDataset(X_val_torch, y_val_torch)
 test = torch.utils.data.TensorDataset(x_test_torch)
 data_loader = torch.utils.data.DataLoader(train)
 val_loader = torch.utils.data.DataLoader(val)
 test_loader = torch.utils.data.DataLoader(test )
sns.set()
sns.set_style('dark')
<define_variables>
%%markdown
Input Features --> Fully Connected layer(512 nodes)--> Dropout(50%)--> Fully Connected layer(256 nodes)--> Dropout(50%)--> Fully Connected layer(128 nodes)--> Dropout(50%)--> Fully Connected layer(1 node )
IS_LOCAL = False
USE_REDUCED = False
data_index = 2*int(IS_LOCAL)+ int(USE_REDUCED)
 train_path =('.. /input/santander-customer-transaction-prediction/train.csv',
'.. /input/santandersmall/train_small.csv',
'train.csv',
'train_small.csv')[data_index]
test_path =('.. /input/santander-customer-transaction-prediction/test.csv',
'.. /input/santandersmall/test_small_with_targets.csv',
'test.csv',
'test_small.csv')[data_index]
train_df = pd.read_csv(train_path)
 test_df = pd.read_csv(test_path )<concatenate>
class Titanic_NN(nn.Module):

 def __init__(self, INPUT_NODES):
super(Titanic_NN, self ).__init__()
self.fc1 = nn.Linear(INPUT_NODES,512)
 self.fc2 = nn.Linear(512,256)
 self.dropout = nn.Dropout(0.5)
 self.fc3 = nn.Linear(256, 128)
 self.fc4 = nn.Linear(128,1)
 def forward(self, x):
x = self.fc1(x)
 x = F.relu(x)
 x = self.dropout(x)
 x = F.relu(self.fc2(x))
x = self.dropout(x)
 x = self.fc3(x)
 x = F.relu(x)
 x = self.dropout(x)
 x = self.fc4(x)
 return F.sigmoid(x )
features = [col for col in train_df.columns if col not in ['target', 'ID_code']]
if not 'target' in test_df:
test_df['target'] = -1
all_df = pd.concat([train_df, test_df], sort=False )<count_unique_values>
model = Titanic_NN(INPUT_NODES)
 try:
model.load_state_dict(torch.load(input_path+'titanic_model_4layers'))
except:
pass
error = nn.BCELoss()
optimizer = torch.optim.SGD(model.parameters() , lr=LEARNING_RATE)
 print(model )
unique_count = np.zeros(( test_df.shape[0], len(features)))
 for f, feature in tqdm(enumerate(features), total=len(features)) :
_, i, c = np.unique(test_df[feature], return_counts=True, return_index=True)
 unique_count[i[c == 1], f] += 1
real_sample_indices = np.argwhere(np.sum(unique_count, axis=1)> 0)[:, 0]
synthetic_sample_indices = np.argwhere(np.sum(unique_count, axis=1)== 0)[:, 0]
print('Real:', len(real_sample_indices))
print('Synthetic:', len(synthetic_sample_indices))
del unique_count<concatenate>
def fit(model, data, phase='training', batch_size = 1, is_cuda=False, input_dim = 8):

 if phase == 'training':
model.train()
elif phase == 'validation':
model.eval()
loss_values = 0.0
correct_values = 0
for _,(features, label)in enumerate(data):
if is_cuda:
features, label = features.cuda() , label.cuda()
features, label = Variable(features.view(batch_size, 1, input_dim)) , Variable(label.float().view(-1, 1))
if phase == 'training':
optimizer.zero_grad()
output = model(features)
 loss = error(output, label)
 loss_values += loss.data
if output[0] > 0.5:
predictions = torch.Tensor([1])
 else:
predictions = torch.Tensor([0])
 correct_values += predictions.eq(label.data.view_as(predictions)).cpu().sum()
if phase == 'training':
loss.backward()
optimizer.step()
loss = loss_values / len(data.dataset)
 accuracy = 100.* correct_values / len(data.dataset)
 {phase} loss is {loss:{5}.{2}} and {phase} accuracy is \
=============================================')
 return loss, accuracy
all_real_df = pd.concat([train_df, test_df.iloc[real_sample_indices, :]], sort=False)
 for feature in tqdm(features):
real_series = all_real_df[feature]
counts = real_series.groupby(real_series ).count()
full_series = all_df[feature]
all_df[f'{feature}_count'] = full_series.map(counts)
 del all_real_df
del real_series
del full_series<normalization>
train_loss_list, val_loss_list = [], []
train_accuracy_list, val_accuracy_list = [], []
for epoch in range(EPOCHS):
train_epoch_loss, train_epoch_accuracy = fit(model, data_loader, batch_size=BATCH_SIZE, input_dim=INPUT_NODES)
 val_epoch_loss, val_epoch_accuracy = fit(model, val_loader, phase='validation', batch_size=BATCH_SIZE, input_dim=INPUT_NODES)
 if epoch % 50 == 0:
torch.save(model.state_dict() , 'titanic_model_4layers')
 train_loss_list.append(train_epoch_loss)
 train_accuracy_list.append(train_epoch_accuracy)
 val_loss_list.append(val_epoch_loss)
 val_accuracy_list.append(val_epoch_accuracy )
for feature in tqdm(features):
all_df[feature] = StandardScaler().fit_transform(all_df[feature].values.reshape(-1, 1))
all_df[f'{feature}_count'] = MinMaxScaler().fit_transform(all_df[f'{feature}_count'].values.reshape(-1, 1))<count_values>
accuracies_list.append(accuracies('Neural Network __Validation_Set__', val_accuracy_list[-1]))
for f in range(len(features)) :
features.append(f'{features[f]}_count' )<split>
def predict(model, data):

 model.eval()
test_predictions = list()
for _,(feature,)in enumerate(data):
feature = Variable(feature.view(1, 1, INPUT_NODES))
output = model(feature)
 if output[0] > 0.5:
prediction = 1
else:
prediction = 0
test_predictions.append(prediction)
 return test_predictions
train_df = all_df.iloc[:train_df.shape[0], :]
test_df = all_df.iloc[train_df.shape[0]:, :]
del all_df<choose_model_class>
pred_df = pd.DataFrame(np.c_[np.arange(892, len(test_set)+892)[:,None], predict(model, test_loader)],
columns=['PassengerId', 'Survived'])
 pred_df.to_csv('titanic_submission.csv', index=False )