Kaggle competition: Gallivanters
<SOS> metric: categorizationaccuracy Kaggle data source: gallivanters<drop_column>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
train_df = train_df.drop(['PassengerId'], axis=1 )<categorify>
import numpy as np
import pandas as pd
from datetime import datetime
from collections import Counter
import json
import os
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Conv2D, MaxPool2D, Flatten, Dropout
from tensorflow.keras.preprocessing.image import ImageDataGenerator
deck = {"A": 1, "B": 2, "C": 3, "D": 4, "E": 5, "F": 6, "G": 7, "U": 8}
data = [train_df, test_df]
for dataset in data:
dataset['Cabin'] = dataset['Cabin'].fillna("U0")
 dataset['Deck'] = dataset['Cabin'].map(lambda x: re.compile("([a-zA-Z]+)" ).search(x ).group())
 dataset['Deck'] = dataset['Deck'].map(deck)
 dataset['Deck'] = dataset['Deck'].fillna(0)
 dataset['Deck'] = dataset['Deck'].astype(int )<drop_column>
dim = 256
train_files = []
test_files = []
country_file = ''
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
path = os.path.join(dirname, filename)
 if 'train' in path:
train_files.append(path)
 elif 'test' in path:
test_files.append(path)
 elif 'json' in path:
country_file = path
train_df = train_df.drop(['Cabin'], axis=1)
 test_df = test_df.drop(['Cabin'], axis=1 )<data_type_conversions>
with open(country_file)as json_file:
tmp = json.load(json_file)
 country = {}
for key in tmp.keys() :
country[int(key)] = tmp[key].split(',')[-1]
data = [train_df, test_df]
for dataset in data:
mean = train_df["Age"].mean()
std = test_df["Age"].std()
is_null = dataset["Age"].isnull().sum()
rand_age = np.random.randint(mean - std, mean + std, size = is_null)
 age_slice = dataset["Age"].copy()
age_slice[np.isnan(age_slice)] = rand_age
dataset["Age"] = age_slice
dataset["Age"] = train_df["Age"].astype(int )<count_missing_values>
train_set = []
for f in train_files:
idx = int(f.split('/')[6])
 train_set.append([f, country[idx]])
 train_set = pd.DataFrame(train_set, columns=['Image','Country'])
 test_set = []
for f in test_files:
test_set.append(f)
 test_set = pd.DataFrame(test_set, columns=['Image'] )
train_df["Age"].isnull().sum()<categorify>
train_data_gen = ImageDataGenerator(rescale = 1./255,
shear_range = 0.2,
zoom_range = 0.2,
horizontal_flip = True )
common_value = 'S'
data = [train_df, test_df]
for dataset in data:
dataset['Embarked'] = dataset['Embarked'].fillna(common_value )<data_type_conversions>
train_generator = train_data_gen.flow_from_dataframe(
 dataframe = train_set,
directory="",
x_col="Image",
y_col="Country",
class_mode="categorical",
target_size=(dim,dim),
batch_size=32 )
data = [train_df, test_df]
for dataset in data:
dataset['Fare'] = dataset['Fare'].fillna(0)
 dataset['Fare'] = dataset['Fare'].astype(int )<feature_engineering>
num_classes = len(Counter(train_generator.classes ).keys() )
data = [train_df, test_df]
titles = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
for dataset in data:
dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.', expand=False)
 dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr',\
'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 dataset['Title'] = dataset['Title'].map(titles)
 dataset['Title'] = dataset['Title'].fillna(0 )<drop_column>
classifier = Sequential()
classifier.add(Conv2D(filters = 32, kernel_size =(3,3), activation = 'relu', input_shape =(dim,dim,3)))
 classifier.add(MaxPool2D(pool_size =(2,2)))
 classifier.add(Conv2D(64,(3,3),activation = 'relu'))
classifier.add(Conv2D(64,(3,3),activation = 'relu'))
classifier.add(MaxPool2D(pool_size =(2,2)))
 classifier.add(Conv2D(128,(3,3),activation = 'relu'))
classifier.add(Conv2D(128,(3,3),activation = 'relu'))
classifier.add(MaxPool2D(pool_size =(2,2)))
 classifier.add(Flatten())
 classifier.add(Dense(units = 128, activation = 'relu'))
classifier.add(Dense(units = 128, activation = 'relu'))
classifier.add(Dense(units = num_classes , activation = 'softmax'))
classifier.compile(
 optimizer = 'adam',
loss = 'categorical_crossentropy',
metrics = ['categorical_accuracy','accuracy'] 
)
 classifier.summary()
train_df = train_df.drop(['Name'], axis=1)
 test_df = test_df.drop(['Name'], axis=1 )<categorify>
classifier.fit_generator(train_generator, epochs = 100, steps_per_epoch = 70 )
genders = {"male": 0, "female": 1}
data = [train_df, test_df]
for dataset in data:
dataset['Sex'] = dataset['Sex'].map(genders )<drop_column>
classes = train_generator.class_indices
print(classes )
train_df = train_df.drop(['Ticket'], axis=1)
 test_df = test_df.drop(['Ticket'], axis=1 )<categorify>
Y_pred = []
for idx in range(test_set.shape[0]):
img = image.load_img(path=test_set['Image'][idx],target_size=(dim,dim,3))
img = image.img_to_array(img)
 test_img = img.reshape(( 1,dim,dim,3))
img_class = classifier.predict_classes(test_img)
 prediction = img_class[0]
Y_pred.append(prediction )
ports = {"S": 0, "C": 1, "Q": 2}
data = [train_df, test_df]
for dataset in data:
dataset['Embarked'] = dataset['Embarked'].map(ports )<feature_engineering>
prediction_classes = [ inverted_classes.get(item,item)for item in Y_pred ]
print(prediction_classes )
data = [train_df, test_df]
for dataset in data:
dataset['Age'] = dataset['Age'].astype(int)
 dataset.loc[ dataset['Age'] <= 11, 'Age'] = 0
dataset.loc[(dataset['Age'] > 11)&(dataset['Age'] <= 18), 'Age'] = 1
dataset.loc[(dataset['Age'] > 18)&(dataset['Age'] <= 22), 'Age'] = 2
dataset.loc[(dataset['Age'] > 22)&(dataset['Age'] <= 27), 'Age'] = 3
dataset.loc[(dataset['Age'] > 27)&(dataset['Age'] <= 33), 'Age'] = 4
dataset.loc[(dataset['Age'] > 33)&(dataset['Age'] <= 40), 'Age'] = 5
dataset.loc[(dataset['Age'] > 40)&(dataset['Age'] <= 66), 'Age'] = 6
dataset.loc[ dataset['Age'] > 66, 'Age'] = 6<count_values>
predictions = []
for idx in range(test_set.shape[0]):
predictions.append([test_set['Image'][idx].split('/')[6].split('.')[0],prediction_classes[idx]])
 predictions = pd.DataFrame(predictions, columns=['ID','Country'])
 predictions['ID'] = predictions['ID'].astype(int)
 predictions.sort_values(by=['ID'], inplace=True)
 predictions.to_csv(datetime.now().strftime("gallivanters_%Y%m%d_%H%M%S.csv"), index=False)
 predictions.head(20 )