Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<categorify>
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV
from sklearn.metrics import accuracy_score
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, AdaBoostClassifier
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from lightgbm import LGBMClassifier
from xgboost import XGBClassifier
from tpot import TPOTClassifier
class MyDataset(Dataset):
def __init__(self, df_data, data_dir = './', transform=None):
super().__init__()
self.df = df_data.values
self.data_dir = data_dir
self.transform = transform
def __len__(self):
return len(self.df)
 def __getitem__(self, index):
img_name,label = self.df[index]
img_path = os.path.join(self.data_dir, img_name+'.tif')
 image = cv2.imread(img_path)
 if self.transform is not None:
image = self.transform(image)
 return image, label<concatenate>
train_df = pd.read_csv('.. /input/titanic/train.csv')
 test_df = pd.read_csv('.. /input/titanic/test.csv')
 print('train_df.shape:', train_df.shape)
 print('test_df.shape:', test_df.shape )
trans_train = transforms.Compose([transforms.ToPILImage() ,
transforms.Pad(64, padding_mode='reflect'),
transforms.RandomHorizontalFlip() ,
transforms.RandomVerticalFlip() ,
transforms.RandomRotation(20),
transforms.ToTensor() ,
transforms.Normalize(mean=[0.5, 0.5, 0.5],std=[0.5, 0.5, 0.5])])
 trans_valid = transforms.Compose([transforms.ToPILImage() ,
transforms.Pad(64, padding_mode='reflect'),
transforms.ToTensor() ,
transforms.Normalize(mean=[0.5, 0.5, 0.5],std=[0.5, 0.5, 0.5])])
 dataset_train = MyDataset(df_data=train, data_dir=train_path, transform=trans_train)
 dataset_valid = MyDataset(df_data=val, data_dir=train_path, transform=trans_valid)
 loader_train = DataLoader(dataset = dataset_train, batch_size=batch_size, shuffle=True, num_workers=0)
 loader_valid = DataLoader(dataset = dataset_valid, batch_size=batch_size//2, shuffle=False, num_workers=0 )<choose_model_class>
train_test_data = [train_df, test_df]
class SimpleCNN(nn.Module):
def __init__(self):
super(SimpleCNN, self ).__init__()
self.conv1 = nn.Conv2d(in_channels=3, out_channels=32, kernel_size=3, padding=2)
 self.conv2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3, padding=2)
 self.conv3 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3, padding=2)
 self.conv4 = nn.Conv2d(in_channels=128, out_channels=256, kernel_size=3, padding=2)
 self.conv5 = nn.Conv2d(in_channels=256, out_channels=512, kernel_size=3, padding=2)
 self.bn1 = nn.BatchNorm2d(32)
 self.bn2 = nn.BatchNorm2d(64)
 self.bn3 = nn.BatchNorm2d(128)
 self.bn4 = nn.BatchNorm2d(256)
 self.bn5 = nn.BatchNorm2d(512)
 self.pool = nn.MaxPool2d(kernel_size=2, stride=2)
 self.avg = nn.AvgPool2d(8)
 self.fc = nn.Linear(512 * 1 * 1, 2)
 def forward(self, x):
x = self.pool(F.leaky_relu(self.bn1(self.conv1(x))))
x = self.pool(F.leaky_relu(self.bn2(self.conv2(x))))
x = self.pool(F.leaky_relu(self.bn3(self.conv3(x))))
x = self.pool(F.leaky_relu(self.bn4(self.conv4(x))))
x = self.pool(F.leaky_relu(self.bn5(self.conv5(x))))
x = self.avg(x)
 x = x.view(-1, 512 * 1 * 1)
 x = self.fc(x)
 return x<choose_model_class>
print(train_df.columns.to_list())
 print(test_df.columns.to_list() )
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adamax(model.parameters() , lr=learning_rate )<train_model>
train_null_s = train_df.isnull().sum()
print(train_null_s[train_null_s != 0])
 print('-'*80)
 test_null_s = test_df.isnull().sum()
print(test_null_s[test_null_s != 0] )
total_step = len(loader_train)
 for epoch in range(num_epochs):
for i,(images, labels)in enumerate(loader_train):
images = images.to(device)
 labels = labels.to(device)
 outputs = model(images)
 loss = criterion(outputs, labels)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
if(i+1)% 100 == 0:
print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}' 
.format(epoch+1, num_epochs, i+1, total_step, loss.item()))<train_model>
outlier_detection_field = ['Age', 'Fare']
weight = 2
outlier_indices = []
for col in outlier_detection_field:
q1 = np.nanpercentile(train_df[col], 25)
 q3 = np.nanpercentile(train_df[col], 75)
 iqr = q3-q1
iqr_weight = iqr * weight
lowest_val = q1 - iqr_weight
highest_val = q3 + iqr_weight
outlier_index = train_df[(train_df[col]<lowest_val)|(highest_val<train_df[col])].index
outlier_indices.extend(outlier_index)
 print('{}: {} / {}(record size:{})'.format(col, lowest_val, highest_val, outlier_index.shape[0]))
model.eval()
with torch.no_grad() :
correct = 0
total = 0
for images, labels in loader_valid:
images = images.to(device)
 labels = labels.to(device)
 outputs = model(images)
 _, predicted = torch.max(outputs.data, 1)
 total += labels.size(0)
 correct +=(predicted == labels ).sum().item()
print('Test Accuracy of the model on the 22003 test images: {} %'.format(100 * correct / total))
torch.save(model.state_dict() , 'model.ckpt' )<load_pretrained>
train_df.drop(outlier_indices, axis=0, inplace=True )
dataset_valid = MyDataset(df_data=sub, data_dir=test_path, transform=trans_valid)
 loader_test = DataLoader(dataset = dataset_valid, batch_size=32, shuffle=False, num_workers=0 )<save_to_csv>
train_df.drop('PassengerId', axis=1, inplace=True)
 test_df_PId = test_df['PassengerId']
test_df.drop('PassengerId', axis=1, inplace=True )
model.eval()
preds = []
for batch_i,(data, target)in enumerate(loader_test):
data, target = data.cuda() , target.cuda()
output = model(data)
 pr = output[:,1].detach().cpu().numpy()
for i in pr:
preds.append(i)
 sub.shape, len(preds)
 sub['label'] = preds
sub.to_csv('s.csv', index=False )<import_modules>
print(train_df.columns.to_list())
 print(test_df.columns.to_list() )
import numpy as np
import pandas as pd
from fastai import *
from fastai.vision import *
from torchvision.models import *
import os
import path<define_variables>
minMaxScaler = MinMaxScaler()
for data in train_test_data:
data['Pclass'] = minMaxScaler.fit_transform(data[['Pclass']] )
imgs = os.listdir('.. /input/train')[:5]
arch = resnet50
bs = 64
sz = 48
path = Path('.. /input')
 open_image('.. /input/train/'+imgs[1] )<load_from_csv>
train_df['Pclass'].value_counts()
df = pd.read_csv(path/'train_labels.csv')
 df.head()<load_from_csv>
for data in train_test_data:
data['Title'] = data['Name'].str.extract('([a-zA-Z]+)\.', expand=False )
tfms = get_transforms()
src =(ImageItemList.from_csv(path, 'train_labels.csv', folder = 'train', suffix = '.tif')
.random_split_by_pct() 
.label_from_df() 
.add_test_folder('test'))<normalization>
for data in train_test_data:
data.drop('Name', axis=1, inplace=True )
np.random.seed(50)
 data =(src.transform(tfms, size = sz)
.databunch().normalize(imagenet_stats))<define_variables>
print(train_df['Title'].value_counts())
 print('-'*50)
 print(test_df['Title'].value_counts() )
data.show_batch(rows = 3, figsize=(11,12))<compute_test_metric>
title_mapping = {
'Mr':0,
'Miss':1,
'Mrs':2,
'Master':3,
'Dr':4, 'Rev':4, 'Major':4, 'Mlle':4, 'Col':4, 'Ms':4, 'Countess':4, 'Mme':4, 'Lady':4, 'Sir':4, 'Don':4, 'Jonkheer':4, 'Capt':4, 'Dona':4
}
for data in train_test_data:
data['Title'] = data['Title'].map(title_mapping )
def auc_score(y_pred,y_true,tens=True):
score = roc_auc_score(y_true,torch.sigmoid(y_pred)[:,1])
 if tens:
score = tensor(score)
 return score<choose_model_class>
train_df['Title'].value_counts()
learn = create_cnn(data, arch, metrics = [accuracy, auc_score], model_dir = '/tmp/models/' )<train_model>
minMaxScaler = MinMaxScaler()
for data in train_test_data:
data['Title'] = minMaxScaler.fit_transform(data[['Title']] )
lr = 1e-02
learn.fit_one_cycle(5, slice(lr))<save_model>
train_df['Title'].value_counts()
learn.save('model-1-rn50' )<find_best_params>
for data in train_test_data:
data['Sex'] = data['Sex'].astype('category' ).cat.codes
learn.unfreeze()
learn.lr_find()
learn.recorder.plot()<train_model>
train_df['Sex'].value_counts()
learn.fit_one_cycle(5, slice(1e-05, lr/5))<save_model>
train_df['Age'].isnull().sum()
learn.save('model-2-rn50' )<categorify>
for data in train_test_data:
data['Age'].fillna(train_df.groupby('Title')['Age'].transform('median'), inplace=True )
data =(src.transform(tfms, size = 96)
.databunch().normalize(imagenet_stats))
learn.data = data
data.train_ds[0][0].shape<find_best_params>
train_df['Age'].isnull().sum()
learn.freeze()
learn.lr_find()
learn.recorder.plot()<train_model>
for data in train_test_data:
data['Age'] = pd.qcut(data['Age'], 9 ).cat.codes
lr = 1e-03
learn.fit_one_cycle(5, slice(lr))<save_model>
minMaxScaler = MinMaxScaler()
for data in train_test_data:
data['Age'] = minMaxScaler.fit_transform(data[['Age']] )
learn.save('model-big-1-rn50' )<find_best_params>
for data in train_test_data:
data['FamilySize'] = data['Parch'] + data['SibSp']
learn.unfreeze()
learn.lr_find()
learn.recorder.plot()<train_model>
for data in train_test_data:
data.drop(['SibSp', 'Parch'], axis=1, inplace=True )
learn.fit_one_cycle(5, slice(3e-06, lr/5))<save_model>
learn.save('model-big-2-rn50' )<find_best_params>
minMaxScaler = MinMaxScaler()
for data in train_test_data:
data['FamilySize'] = minMaxScaler.fit_transform(data[['FamilySize']] )
interp = ClassificationInterpretation.from_learner(learn)
 losses,idxs = interp.top_losses()<find_best_params>
print(train_df['Ticket'].value_counts())
 print('-'*80)
 print(train_df['Ticket'].unique().shape )
losses,idxs = interp.top_losses()<compute_test_metric>
for data in train_test_data:
data.drop('Ticket', axis=1, inplace=True )
preds,y = learn.TTA()
acc = accuracy(preds, y)
 print('The validation accuracy is {} %.'.format(acc * 100))<compute_test_metric>
print(train_df['Fare'].isnull().sum())
 print(test_df['Fare'].isnull().sum() )
pred_score = auc_score(preds,y ).item()
print('The validation AUC is {}.'.format(pred_score))<load_from_csv>
for data in train_test_data:
data['Fare'].fillna(train_df.groupby(['Embarked', 'Pclass'])['Fare'].transform('median'), inplace=True )
df1 = pd.read_csv(path/'sample_submission.csv')
 id_list = list(df1.id)
<feature_engineering>
test_df['Fare'].isnull().sum()
preds,y = learn.TTA(ds_type=DatasetType.Test)
 pred_list = list(preds[:,1] )<define_variables>
for data in train_test_data:
data['Fare'] = pd.qcut(data['Fare'], 10 ).astype('category' ).cat.codes
pred_dict = dict(( key, value.item())for(key, value)in zip(learn.data.test_ds.items,pred_list))
pred_ordered = [pred_dict[Path('.. /input/test/' + id + '.tif')] for id in id_list]<save_to_csv>
train_df['Fare'].value_counts()
submissions = pd.DataFrame({'id':id_list,'label':pred_ordered})
 submissions.to_csv("submission_{}.csv".format(pred_score),index = False)
<set_options>
minMaxScaler = MinMaxScaler()
for data in train_test_data:
data['Fare'] = minMaxScaler.fit_transform(data[['Fare']] )
%reload_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
train_df['Fare'].value_counts()
from fastai import *
from fastai.vision import *
from torchvision.models import *
import os
import matplotlib.pyplot as plt<load_from_csv>
print(train_df['Cabin'].value_counts())
 print('-'*80)
 print(train_df['Cabin'].unique().shape)
 print('-'*80)
 print(train_df['Cabin'].str[:1].value_counts() )
path = Path(".. /input")
 labels = pd.read_csv(path/"train_labels.csv")
 labels.head()<count_unique_values>
print(test_df['Cabin'].str[:1].value_counts() )
print(labels["label"].nunique()); classes = list(set(labels["label"])) ; classes<count_values>
for data in train_test_data:
data['Cabin'] = data['Cabin'].str[:1]
for i in classes:
print("Number of items in class {} is {}".format(i,len(labels[labels["label"] == i])) )<feature_engineering>
cabin_mapping={"A":1, "B":2, "C":3, "D":4, "E":5, "F":6, "G":7, "T":8}
for data in train_test_data:
data['Cabin'] = data['Cabin'].map(cabin_mapping )
tfms = get_transforms(do_flip = True,flip_vert = True,max_zoom = 1.1 )<load_from_csv>
print(train_df['Cabin'].value_counts())
 print('-'*80)
 print(train_df.groupby(['Pclass'])['Cabin'].median() )
np.random.seed(123)
 sz = 32
data = ImageDataBunch.from_csv(path, folder = 'train', csv_labels = "train_labels.csv",
test = 'test',suffix=".tif", size = sz,bs = 256,
ds_tfms = tfms)
 data.path = pathlib.Path('.')
 data.normalize(imagenet_stats )<define_variables>
for data in train_test_data:
data['Cabin'].fillna(data.groupby(['Pclass'])['Cabin'].transform('median'), inplace=True )
data.show_batch(rows = 3 )<compute_test_metric>
print(train_df['Cabin'].isnull().sum())
 print('-'*80)
 print(train_df['Cabin'].value_counts() )
def auc_score(y_pred,y_true,tens=True):
score = roc_auc_score(y_true,torch.sigmoid(y_pred)[:,1])
 if tens:
score = tensor(score)
 return score<choose_model_class>
minMaxScaler = MinMaxScaler()
for data in train_test_data:
data['Cabin'] = minMaxScaler.fit_transform(data[['Cabin']] )
arch = models.densenet121
learn = cnn_learner(data,arch,pretrained = True,ps = 0.45,metrics = [auc_score,accuracy] )<train_model>
print(train_df['Embarked'].isnull().sum())
 print(test_df['Embarked'].isnull().sum() )
learn.fit_one_cycle(8,1e-2 )<find_best_params>
for data in train_test_data:
data['Embarked'] = data['Embarked'].fillna('S' )
learn.unfreeze()
learn.lr_find()
learn.recorder.plot()<train_model>
for data in train_test_data:
data['Embarked'] = data['Embarked'].astype('category' ).cat.codes
learn.fit_one_cycle(2,max_lr = slice(1e-5,1e-3))<load_from_csv>
minMaxScaler = MinMaxScaler()
for data in train_test_data:
data['Embarked'] = minMaxScaler.fit_transform(data[['Embarked']] )
newTfms = get_transforms(do_flip = True,flip_vert = True,max_zoom = 1.25)
 newSz = 64
newData = ImageDataBunch.from_csv(path, folder = 'train', csv_labels = "train_labels.csv",
test = 'test',suffix=".tif", size = newSz, ds_tfms = newTfms)
 newData.path = pathlib.Path('.')
 newData.normalize(imagenet_stats)
 learn.data = newData<find_best_params>
y_train_s = train_df['Survived']
x_train_df = train_df.drop('Survived', axis=1 )
learn.freeze()
learn.lr_find()
learn.recorder.plot()<train_model>
x_train, x_test, y_train, y_test = train_test_split(x_train_df, y_train_s, test_size=0.2, random_state=10 )
learn.fit_one_cycle(8,1e-2 )<find_best_params>
def cross_val_score_result(estimator, x, y, scoring, cv):
clf_scores = cross_val_score(estimator, x, y, scoring=scoring, cv=cv)
 clf_scores_mean = np.round(np.mean(clf_scores), 4)
 return clf_scores_mean
learn.unfreeze()
learn.lr_find()
learn.recorder.plot()<train_model>
classifiers = [
DecisionTreeClassifier() ,
RandomForestClassifier() ,
GradientBoostingClassifier() ,
KNeighborsClassifier() ,
SVC() ,
LGBMClassifier() ,
XGBClassifier() ,
AdaBoostClassifier()
]
learn.fit_one_cycle(2,max_lr = slice(1e-3/3,1e-3))<save_model>
best_clf_score = 0
best_clf = None
clf_name = []
clf_mean_score = []
for clf in classifiers:
current_clf_score = cross_val_score_result(clf, x_train, y_train, 'accuracy', 10)
 clf_name.append(clf.__class__.__name__)
 clf_mean_score.append(current_clf_score)
 if current_clf_score > best_clf_score:
best_clf_score = current_clf_score
best_clf = clf
learn.save('stage-2' )<compute_test_metric>
lgbm_clf = LGBMClassifier()
grid_param = {
'learning_rate':[0.005, 0.01, 0.015, 0.02],
'n_estimators':[100, 150, 200],
'bossting_type':['rf', 'gbdt', 'dart', 'goss'],
'max_depth':[10, 15, 20]
}
lgbm_grid = GridSearchCV(lgbm_clf, grid_param, cv=10)
 lgbm_grid.fit(x_train, y_train )
preds,y = learn.TTA()
acc = accuracy(preds, y)
 print('The validation accuracy is {} %.'.format(acc * 100))
pred_score = auc_score(preds,y ).item()
print('The validation AUC is {}.'.format(pred_score))<load_from_csv>
print('best_param:', lgbm_grid.best_params_)
 print('best_score:{:.4f}'.format(lgbm_grid.best_score_))
newTfms = get_transforms(do_flip = True,flip_vert = True,max_zoom = 1.5)
 newSz = 96
newData = ImageDataBunch.from_csv(path, folder = 'train', csv_labels = "train_labels.csv",
test = 'test',suffix=".tif", size = newSz, ds_tfms = newTfms)
 newData.path = pathlib.Path('.')
 newData.normalize(imagenet_stats)
 learn.data = newData<find_best_params>
test_pred = lgbm_grid.best_estimator_.predict(test_df)
 submission = pd.DataFrame({
'PassengerId': test_df_PId,
'Survived': test_pred
} )
learn.freeze()
learn.lr_find()
learn.recorder.plot()<train_model>
submission.to_csv('submission_test.csv', index=False )
<find_best_params><EOS> 
check_submission = pd.read_csv('submission_test.csv')
 check_submission