Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<set_options>
warnings.filterwarnings("ignore")
 %matplotlib inline
warnings.filterwarnings('ignore' )<load_from_csv>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv' )
train_directory = ".. /input/train/train/"
train_df = pd.read_csv('.. /input/train.csv')
 train_df.tail()<prepare_x_and_y>
test.isnull().sum()
X = []
Y = []
imges = train_df['id'].values
for img_id in tqdm_notebook(imges):
X.append(cv2.imread(train_directory + img_id))
Y.append(train_df[train_df['id'] == img_id]['has_cactus'].values[0])
 X = np.asarray(X)
 X = X.astype('float32')
 X /= 255
Y = np.asarray(Y )<import_modules>
test.isnull().sum()
from keras.layers import Conv2D, MaxPool2D, Dense, BatchNormalization, Activation, GlobalAveragePooling2D, Flatten, Dropout
from keras.models import Sequential, Model
from keras.regularizers import l2
from keras.optimizers import Adam<choose_model_class>
train.isnull().sum()
def simple_model() :
model = Sequential()
model.add(Conv2D(32,(3, 3), input_shape=(32, 32, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Conv2D(32,(3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Conv2D(32,(3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Conv2D(32,(3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(MaxPool2D())
 model.add(Conv2D(64,(3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Conv2D(64,(3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Conv2D(64,(3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(MaxPool2D())
 model.add(Conv2D(128,(3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Conv2D(128,(3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Conv2D(128,(3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(MaxPool2D())
 model.add(GlobalAveragePooling2D())
 model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer=Adam(lr=1e-5),
metrics=['accuracy'])
 return model<choose_model_class>
train.isnull().sum()
datagen = ImageDataGenerator(featurewise_center=False,
",
samplewise_center=False,
",
featurewise_std_normalization=False,
",
samplewise_std_normalization=False,
",
zca_whitening=False,
",
zca_epsilon=1e-06,
",
rotation_range=45,
",
width_shift_range=0.2,
",
height_shift_range=0.2,
",
shear_range=0.2,
",
zoom_range=0.2,
",
channel_shift_range=0.,
",
fill_mode='nearest',
validation_split=0.1,
horizontal_flip=True,
vertical_flip=True)
 datagen.fit(X )<train_model>
train['Survived'].value_counts(normalize=True )
model = simple_model()
batch_size = 16
epoch = 1000
train = datagen.flow(X, Y, batch_size=batch_size, subset='training')
 validate = datagen.flow(X, Y, batch_size=batch_size, subset='validation')
 history = model.fit_generator(train,
validation_data=validate,
steps_per_epoch=len(X)/ batch_size,
epochs=epoch,
shuffle=True,
validation_steps=len(X)/batch_size )<data_type_conversions>
train['Survived'].groupby(train['Pclass'] ).mean()
test_directory = ".. /input/test/test/"
X_test = []
Test_imgs = []
for img_id in tqdm_notebook(os.listdir(test_directory)) :
X_test.append(cv2.imread(test_directory + img_id))
Test_imgs.append(img_id)
 X_test = np.asarray(X_test)
 X_test = X_test.astype('float32')
 X_test /= 255<predict_on_test>
train['Title'] = train['Title'] = train.Name.str.extract('([A-Za-z]+)\.', expand=False)
 train['Title'].value_counts()
test_predictions = model.predict(X_test )<save_to_csv>
train['Survived'].groupby(train['Title'] ).mean()
pred_df = pd.DataFrame(test_predictions, columns=['has_cactus'])
 pred_df['id'] = ''
cols = pred_df.columns.tolist()
cols = cols[-1:] + cols[:-1]
pred_df=pred_df[cols]
for i, img in enumerate(Test_imgs):
pred_df.set_value(i,'id',img)
 pred_df.to_csv('submission.csv',index=False )<import_modules>
train['Name_Len'] = train['Name'].apply(lambda x: len(x))
train['Survived'].groupby(pd.qcut(train['Name_Len'],5)).mean()
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import glob
import scipy
import cv2
import random
from sklearn.model_selection import KFold
import keras<load_from_csv>
pd.qcut(train['Name_Len'],5 ).value_counts()
train_data = pd.read_csv('.. /input/train.csv' )<define_variables>
train['Sex'].value_counts(normalize=True )
positive_examples_indexes = train_data[train_data.has_cactus==1].index
negative_examples_indexes = train_data[train_data.has_cactus==0].index<randomize_order>
train['Survived'].groupby(train['Sex'] ).mean()
pos_indexes = positive_examples_indexes.tolist() [:]
random.shuffle(pos_indexes )<define_variables>
train['Survived'].groupby(pd.qcut(train['Age'],5)).mean()
k = 5
third = int(len(pos_indexes)/ k)
 folds_indexes = []
for i in range(k):
start = i*third
end =(i+1)*third
if i == k-1:
end = len(pos_indexes)
 folds_indexes.append(pos_indexes[start:end] )<randomize_order>
pd.qcut(train['Age'],5 ).value_counts()
def image_generator(indexes=None, batch_size = 16, shuffle=True, train=True):
while True:
if train:
temp_indexes = indexes[:]
temp_indexes.extend(negative_examples_indexes.tolist())
 random.shuffle(temp_indexes)
 random.shuffle(temp_indexes)
 else:
temp_indexes = indexes[:]
N = int(len(temp_indexes)/ batch_size)
 for i in range(N):
current_indexes = temp_indexes[i*batch_size:(i+1)*batch_size]
batch_input = []
batch_output = []
for index in current_indexes:
img = mpimg.imread('.. /input/train/train/' + train_data.id[index])
 batch_input += [img]
batch_input += [img[::-1, :, :]]
batch_input += [img[:, ::-1, :]]
batch_input += [np.rot90(img)]
temp_img = np.zeros_like(img)
 temp_img[:28, :, :] = img[4:, :, :]
batch_input += [temp_img]
temp_img = np.zeros_like(img)
 temp_img[:, :28, :] = img[:, 4:, :]
batch_input += [temp_img]
temp_img = np.zeros_like(img)
 temp_img[4:, :, :] = img[:28, :, :]
batch_input += [temp_img]
temp_img = np.zeros_like(img)
 temp_img[:, 4:, :] = img[:, :28, :]
batch_input += [temp_img]
batch_input += [cv2.resize(img[2:30, 2:30, :],(32, 32)) ]
batch_input += [scipy.ndimage.interpolation.rotate(img, 10, reshape=False)]
batch_input += [scipy.ndimage.interpolation.rotate(img, 5, reshape=False)]
for _ in range(11):
batch_output += [train_data.has_cactus[index]]
batch_input = np.array(batch_input)
 batch_output = np.array(batch_output)
 yield(batch_input, batch_output.reshape(-1, 1))<choose_model_class>
train['SibSp'].value_counts()
def build_model() :
model = keras.models.Sequential()
model.add(keras.layers.Conv2D(64,(3, 3), input_shape=(32, 32, 3)))
 model.add(keras.layers.BatchNormalization())
 model.add(keras.layers.LeakyReLU(alpha=0.3))
model.add(keras.layers.Conv2D(64,(3, 3)))
 model.add(keras.layers.BatchNormalization())
 model.add(keras.layers.LeakyReLU(alpha=0.3))
model.add(keras.layers.Conv2D(128,(3, 3)))
 model.add(keras.layers.BatchNormalization())
 model.add(keras.layers.LeakyReLU(alpha=0.3))
model.add(keras.layers.Conv2D(128,(3, 3)))
 model.add(keras.layers.BatchNormalization())
 model.add(keras.layers.LeakyReLU(alpha=0.3))
model.add(keras.layers.Conv2D(256,(3, 3)))
 model.add(keras.layers.BatchNormalization())
 model.add(keras.layers.LeakyReLU(alpha=0.3))
model.add(keras.layers.Conv2D(256,(3, 3)))
 model.add(keras.layers.BatchNormalization())
 model.add(keras.layers.LeakyReLU(alpha=0.3))
model.add(keras.layers.Conv2D(512,(3, 3)))
 model.add(keras.layers.BatchNormalization())
 model.add(keras.layers.LeakyReLU(alpha=0.3))
model.add(keras.layers.Flatten())
 model.add(keras.layers.Dense(100))
model.add(keras.layers.BatchNormalization())
 model.add(keras.layers.LeakyReLU(alpha=0.3))
model.add(keras.layers.Dense(1, activation='sigmoid'))
opt = keras.optimizers.Adam(0.0001)
 model.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])
 return model<train_model>
train['Survived'].groupby(train['SibSp'] ).mean()
models = []
for i in range(k):
print('i =', i)
 model = build_model()
for j in range(k):
if i == j:
continue
current_pos_indexes = folds_indexes[i]
model.fit_generator(image_generator(current_pos_indexes), steps_per_epoch= len(current_pos_indexes)/ 16, epochs=20)
 models.append(model )<find_best_params>
train['Parch'].value_counts()
for model in models:
for j in range(k):
current_pos_indexes = folds_indexes[j]
print(model.evaluate_generator(image_generator(indexes=current_pos_indexes), steps=len(current_pos_indexes)/ 16))<define_variables>
train['Survived'].groupby(train['Parch'] ).mean()
test_files = os.listdir('.. /input/test/test/' )<load_pretrained>
train['Ticket_Len'] = train['Ticket'].apply(lambda x: len(x))
train['Ticket_Len'].value_counts()
preds = []
for _ in range(len(models)) :
preds.append([])
 batch = 40
for i in range(int(4000/batch)) :
images = []
for j in range(batch):
img = mpimg.imread('.. /input/test/test/'+test_files[i*batch + j])
 images += [img]
for k2 in range(len(models)) :
model = models[k2]
out = model.predict(np.array(images))
preds[k2] += [out] 
<define_variables>
train['Ticket_Lett'] = train['Ticket'].apply(lambda x: str(x)[0])
 train['Ticket_Lett'].value_counts()
all_out = np.array(list(map(lambda x: np.array(x ).reshape(-1, 1), preds)) )<feature_engineering>
pd.qcut(train['Fare'], 3 ).value_counts()
all_out = np.mean(all_out, axis=0 )<create_dataframe>
train['Survived'].groupby(pd.qcut(train['Fare'], 3)).mean()
sub_file = pd.DataFrame(data = {'id': test_files, 'has_cactus': all_out.reshape(-1 ).tolist() } )<save_to_csv>
train['Cabin_Lett'] = train['Cabin'].apply(lambda x: str(x)[0])
 train['Cabin_Lett'].value_counts()
sub_file.to_csv('sample_submission.csv', index=False )<import_modules>
train['Survived'].groupby(train['Cabin_Lett'] ).mean()
print(fastai.__version__ )<define_variables>
train['Cabin_num'] = train['Cabin'].apply(lambda x: str(x ).split(' ')[-1][1:])
 train['Cabin_num'].replace('an', np.NaN, inplace = True)
 train['Cabin_num'] = train['Cabin_num'].apply(lambda x: int(x)if not pd.isnull(x)and x != '' else np.NaN )
PATH = Path('.')
 !ls {PATH}<load_from_csv>
pd.qcut(train['Cabin_num'],3 ).value_counts()
df = pd.read_csv(PATH/'.. /input/train.csv'); df.head()<define_variables>
train['Survived'].groupby(pd.qcut(train['Cabin_num'], 3)).mean()
bs = 128<feature_engineering>
def names(train, test):
for i in [train, test]:
i['Name_Len'] = i['Name'].apply(lambda x: len(x))
i['Name_Title'] = i['Name'].apply(lambda x: x.split(',')[1] ).apply(lambda x: x.split() [0])
 del i['Name']
return train, test
tfms = get_transforms(do_flip=True, flip_vert=True )<load_from_csv>
def age_impute(train, test):
for i in [train, test]:
i['Age_Null_Flag'] = i['Age'].apply(lambda x: 1 if pd.isnull(x)else 0)
 data = train.groupby(['Name_Title', 'Pclass'])['Age']
i['Age'] = data.transform(lambda x: x.fillna(x.mean()))
return train, test
data =(ImageList.from_csv(csv_name='train.csv', path=PATH/'.. /input', folder='train/train')
.split_by_rand_pct() 
.label_from_df(cols='has_cactus')
.transform(tfms)
.add_test_folder('test/test')
.databunch(bs=bs)
.normalize(imagenet_stats))<define_variables>
def fam_size(train, test):
for i in [train, test]:
i['Fam_Size'] = np.where(( i['SibSp']+i['Parch'])== 0 , 'Alone',
np.where(( i['SibSp']+i['Parch'])<= 3,'Small', 'Big'))
del i['SibSp']
del i['Parch']
return train, test
data.show_batch(rows=3, figsize=(7, 7))<find_best_params>
def cabin(train, test):
for i in [train, test]:
i['Cabin_Lett'] = i['Cabin'].apply(lambda x: str(x)[0])
 del i['Cabin']
return train, test
data.classes, data.c<choose_model_class>
def cabin_num(train, test):
for i in [train, test]:
i['Cabin_num1'] = i['Cabin'].apply(lambda x: str(x ).split(' ')[-1][1:])
 i['Cabin_num1'].replace('an', np.NaN, inplace = True)
 i['Cabin_num1'] = i['Cabin_num1'].apply(lambda x: int(x)if not pd.isnull(x)and x != '' else np.NaN)
 i['Cabin_num'] = pd.qcut(train['Cabin_num1'],3)
 train = pd.concat(( train, pd.get_dummies(train['Cabin_num'], prefix = 'Cabin_num')) , axis = 1)
 test = pd.concat(( test, pd.get_dummies(test['Cabin_num'], prefix = 'Cabin_num')) , axis = 1)
 del train['Cabin_num']
del test['Cabin_num']
del train['Cabin_num1']
del test['Cabin_num1']
return train, test
learn = cnn_learner(data, models.resnet34, metrics=accuracy, path=PATH )<train_model>
train.Embarked.value_counts()
learn.fit_one_cycle(4 )<train_model>
train['Embarked'].fillna('S', inplace=True )
learn.fit_one_cycle(1 )<predict_on_test>
test[test.Fare.isnull() ]
probs, preds = learn.get_preds(ds_type=DatasetType.Test )<prepare_x_and_y>
train[(train.Pclass==3)&(train.Age>50)].Fare.median()
ilst = data.test_ds.x<define_variables>
test['Fare'].fillna(7.75, inplace=True )
fnames = [item.name for item in ilst.items]; fnames[:10]<create_dataframe>
def dummies(train, test, columns = ['Pclass', 'Sex', 'Embarked', 'Ticket_Lett', 'Cabin_Lett', 'Name_Title', 'Fam_Size']):
for column in columns:
train[column] = train[column].apply(lambda x: str(x))
test[column] = test[column].apply(lambda x: str(x))
good_cols = [column+'_'+i for i in train[column].unique() if i in test[column].unique() ]
train = pd.concat(( train, pd.get_dummies(train[column], prefix = column)[good_cols]), axis = 1)
 test = pd.concat(( test, pd.get_dummies(test[column], prefix = column)[good_cols]), axis = 1)
 del train[column]
del test[column]
return train, test
test_df = pd.DataFrame({'id': fnames, 'has_cactus': probs[:, 1]}); test_df<save_to_csv>
def drop(train, test, bye = ['PassengerId']):
for i in [train, test]:
for z in bye:
del i[z]
return train, test
test_df.to_csv('submission.csv', index=None )<set_options>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv')
 train, test = names(train, test)
 train, test = age_impute(train, test)
 train, test = cabin_num(train, test)
 train, test = cabin(train, test)
 train, test = fam_size(train, test)
 train, test = ticket_grouped(train, test)
 train['Embarked'].fillna('S', inplace=True)
 test['Fare'].fillna(7.75, inplace=True)
 train, test = dummies(train, test, columns = ['Pclass', 'Sex', 'Embarked', 'Ticket_Lett',
'Cabin_Lett', 'Name_Title', 'Fam_Size'])
 train, test = drop(train, test )
%matplotlib inline
train_on_gpu = True 
<install_modules>
rf = RandomForestClassifier(criterion='gini',
n_estimators=700,
min_samples_split=10,
min_samples_leaf=1,
max_features='auto',
oob_score=True,
random_state=1,
n_jobs=-1)
 rf.fit(train.iloc[:, 1:], train.iloc[:, 0])
 print("%.4f" % rf.oob_score_ )
!pip install albumentations > /dev/null 2>&1
!pip install pretrainedmodels > /dev/null 2>&1
!pip install kekas > /dev/null 2>&1
!pip install adabound > /dev/null 2>&1<import_modules>
pd.concat(( pd.DataFrame(train.iloc[:, 1:].columns, columns = ['variable']),
pd.DataFrame(rf.feature_importances_, columns = ['importance'])) ,
axis = 1 ).sort_values(by='importance', ascending = False)[:20]
import albumentations
from albumentations import torch as AT
import pretrainedmodels
import adabound
from kekas import Keker, DataOwner, DataKek
from kekas.transformations import Transformer, to_torch, normalize
from kekas.metrics import accuracy
from kekas.modules import Flatten, AdaptiveConcatPool2d
from kekas.callbacks import Callback, Callbacks, DebuggerCallback
from kekas.utils import DotDict<load_from_csv>
predictions = rf.predict(test)
 predictions = pd.DataFrame(predictions, columns=['Survived'])
 test = pd.read_csv(".. /input/titanic/test.csv")
 predictions = pd.concat(( test.iloc[:, 0], predictions), axis = 1)
 predictions.to_csv('submission.csv', sep=",", index = False )