Kaggle competition: Dogs vs. Cats Redux: Kernels Edition
<SOS> metric: LogLoss Kaggle data source: dogs-vs.-cats-redux:-kernels-edition<categorify>
Conv2D, SeparableConv1D, Add, BatchNormalization, Activation, GlobalAveragePooling2D, LeakyReLU, Flatten
Lambda, Multiply, LSTM, Bidirectional, PReLU, MaxPooling1D
%matplotlib inline
labelEncoder = LabelEncoder()
df_titanic['Sex'] = labelEncoder.fit_transform(df_titanic['Sex'] )<feature_engineering>
input_directory = r".. /input/"
training_dir = input_directory + r"train"
testing_dir = input_directory + r"test"
condition = df_titanic['Age'].isnull()
age_NaN = df_titanic['Age'][condition].index
for age in age_NaN :
condition1 = df_titanic['SibSp'] == df_titanic.iloc[age]["SibSp"]
condition2 = df_titanic['Pclass'] == df_titanic.iloc[age]["Pclass"]
condition3 = df_titanic['Parch'] == df_titanic.iloc[age]["Parch"]
condition = condition1 & condition2 & condition3
new_age = df_titanic['Age'][condition].median()
df_titanic['Age'].iloc[age] = new_age if not np.isnan(new_age)else df_titanic['Age'].median()<feature_engineering>
train_files = os.listdir(training_dir)
 test_files = os.listdir(testing_dir)
 train_labels = []
for file in train_files:
train_labels.append(file.split(".")[0])
 df_train = pd.DataFrame({"id": train_files, "label": train_labels})
 df_train.head()
df_titanic['Age'] =(df_titanic['Age'] - df_titanic['Age'].mean())/ df_titanic['Age'].std()<feature_engineering>
df_test = pd.DataFrame({"id": test_files})
 df_test["label"] = ["cat"]*(len(test_files))
df_test.head()
df_titanic['Title'] = df_titanic['Name'].str.extract('([A-Za-z]+)\.', expand=False)
 df_titanic['Title'].unique()<feature_engineering>
def get_class_files(df, file="id", label = "label", count=5):
label_map = {}
for l in set(df[label]):
label_map[l] = []
for i in range(len(df)) :
label_map[df[label][i]].append(df[file][i])
 return label_map
def select_image_by_label(label_map, image_count_per_label):
label_map_select = {}
for l in label_map:
label_map_select[l] = []
for i in label_map:
num_image = len(label_map[i])
 image_mem = {}
for j in range(image_count_per_label):
image_index = random.randint(0, num_image - 1)
 while image_index in image_mem:
image_index = random.randint(0, num_image - 1)
 label_map_select[i].append(label_map[i][image_index])
 return label_map_select
def plot_sample_image(label_map_select, directory=training_dir, nrows=2, ncols=5, figsize=(16, 4), aspect=None):
fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(16, 4))
i=0
for label in label_map_select:
print(label)
 j=0
for file in label_map_select[label]:
plt.subplot(i+1, ncols, j+1)
 plot_image(file, directory, aspect=aspect)
 j=j+1
plt.tight_layout()
plt.show()
i+=1
def plot_sample_test_image(directory=testing_dir, count=5, nrows=1, ncols=5, figsize=(16, 4), aspect=None):
selected_files = random.sample(os.listdir(testing_dir), count)
 fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(16, 4))
i=0
for file in selected_files:
plt.subplot(1, ncols, i+1)
 path = directory + file
plot_image(file, directory, aspect=aspect)
 i=i+1
plt.tight_layout()
plt.show()
def plot_image(file, directory=None, sub=False, aspect=None):
path = directory + file
img = plt.imread(path)
 plt.imshow(img, aspect=aspect)
 plt.xticks([])
 plt.yticks([])
 plt.title(path.split("/")[-1])
 if sub:
plt.show()
df_titanic['Title'] = df_titanic['Title'].replace(['Don', 'Rev', 'Dr', 'Major', 'Lady', 'Sir',
'Col', 'Capt', 'Countess', 'Jonkheer', 'Dona'], 'Rare' )<categorify>
def reset_graph(model=None):
if model:
try:
del model
except:
return False
tf.reset_default_graph()
K.clear_session()
gc.collect()
return True
def reset_callbacks(checkpoint=None, reduce_lr=None, early_stopping=None, tensorboard=None):
checkpoint = None
reduce_lr = None
early_stopping = None
tensorboard = None
df_titanic["Title"] = df_titanic['Title'].map({"Master":0, "Miss":1, "Ms" : 1, "Mme":1,
"Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 df_titanic['Title'] = df_titanic["Title"].astype(int )<feature_engineering>
classes = ['cat', 'dog']
df_titanic['Surname'] = df_titanic['Name'].map(lambda i: i.split(',')[0] )<drop_column>
def get_data(batch_size=32, target_size=(96,96), class_mode="categorical", training_dir=training_dir, testing_dir=testing_dir, classes=classes, df_train=df_train, df_test=df_test):
print("Generating data following preprocessing...
 ")
 rescale = 1.0/255
train_batch_size = batch_size
test_batch_size = batch_size
train_shuffle = True
val_shuffle = True
test_shuffle = False
train_datagen = ImageDataGenerator(
 horizontal_flip=True,
shear_range=0.2,
zoom_range=0.2,
rescale=rescale,
validation_split=0.25 
)
 train_generator = train_datagen.flow_from_dataframe(
 df_train,
training_dir,
x_col='id',
y_col='label',
has_ext=True,
target_size=target_size,
classes = classes,
class_mode=class_mode,
batch_size=batch_size,
shuffle=True,
seed=42,
subset='training')
 validation_generator = train_datagen.flow_from_dataframe(
 df_train,
training_dir,
x_col='id',
y_col='label',
has_ext=True,
target_size=target_size,
classes = classes,
class_mode=class_mode,
batch_size=batch_size,
shuffle=True,
seed=42,
subset='validation')
 test_datagen = ImageDataGenerator(rescale=rescale)
 test_generator = test_datagen.flow_from_dataframe(
 df_test,
testing_dir,
x_col='id',
y_col='label',
has_ext=True,
target_size=target_size,
classes = classes,
class_mode=class_mode,
batch_size=batch_size,
shuffle=False 
)
 class_weights = get_weight(train_generator.classes)
 steps_per_epoch = len(train_generator)
 validation_steps = len(validation_generator)
 print("
Data batches generated.")
 return train_generator, validation_generator, test_generator, class_weights, steps_per_epoch, validation_steps
def get_weight(y):
class_weight_current = cw.compute_class_weight('balanced', np.unique(y), y)
 return class_weight_current
del df_titanic['Name']<count_missing_values>
def get_conv_model() :
model = Sequential()
model.add(Conv2D(16,(3, 3), activation='relu', padding="same", input_shape=(3,96,96)))
 model.add(Conv2D(16,(3, 3), padding="same", activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(32,(3, 3), activation='relu', padding="same"))
model.add(Conv2D(32,(3, 3), padding="same", activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(64,(3, 3), activation='relu', padding="same"))
model.add(Conv2D(64,(3, 3), padding="same", activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(96,(3, 3), dilation_rate=(2, 2), activation='relu', padding="same"))
model.add(Conv2D(96,(3, 3), padding="valid", activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(128,(3, 3), dilation_rate=(2, 2), activation='relu', padding="same"))
model.add(Conv2D(128,(3, 3), padding="valid", activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Flatten())
 model.add(Dense(64, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(2 , activation='softmax'))
print(model.summary())
 return model
def get_model(model_name, input_shape=(96, 96, 3)) :
inputs = Input(input_shape)
 if model_name == "Xception":
base_model = Xception(include_top=False, input_shape=input_shape)
 elif model_name == "ResNet50":
base_model = ResNet50(include_top=False, input_shape=input_shape)
 elif model_name == "InceptionV3":
base_model = InceptionV3(weights='imagenet', include_top=False, input_shape=input_shape)
 elif model_name == "InceptionResNetV2":
base_model = InceptionResNetV2(include_top=False, input_shape=input_shape)
 if model_name == "DenseNet201":
base_model = DenseNet201(include_top=False, input_shape=input_shape)
 if model_name == "NASNetMobile":
base_model = NASNetMobile(include_top=False, input_shape=input_shape)
 if model_name == "NASNetLarge":
base_model = NASNetLarge(include_top=False, input_shape=input_shape)
 for layer in base_model.layers:
layer.trainable = False
model = Model(inputs=base_model.input, outputs=predictions)
 model.summary()
return model
df_titanic['Cabin'].isnull().sum()<feature_engineering>
def auroc(y_true, y_pred):
return tf.py_func(roc_auc_score,(y_true, y_pred), tf.double )
df_titanic['Ticket'] = df_titanic['Ticket'].map(
 lambda i: i.replace(".","" ).replace("/","" ).strip().split(' ')[0] if not i.isdigit() else "TKT" )<feature_engineering>
model_dir = main_model_dir + time.strftime('%Y-%m-%d %H-%M-%S')+ "/"
log_dir = main_log_dir + time.strftime('%Y-%m-%d %H-%M-%S')
 os.mkdir(model_dir)
 os.mkdir(log_dir)
 model_file = model_dir + "{epoch:02d}-val_acc-{val_acc:.2f}-val_loss-{val_loss:.2f}.hdf5"
model_dir, os.listdir(model_dir), log_dir, os.listdir(log_dir )
df_titanic['Family'] = df_titanic['SibSp'] + df_titanic['Parch'] + 1<categorify>
print("Settting Callbacks")
 checkpoint = ModelCheckpoint(
 model_file,
monitor='val_acc',
save_best_only=True)
 tensorboard = TensorBoard(
 log_dir=log_dir,
update_freq='batch')
 early_stopping = EarlyStopping(
 monitor='val_loss',
patience=2,
verbose=1,
restore_best_weights=True)
 reduce_lr = ReduceLROnPlateau(
 monitor='val_loss',
factor=0.5,
patience=1,
verbose=1)
 callbacks = [reduce_lr, early_stopping, checkpoint]
print("Completed")
 model_dir, os.listdir(model_dir), log_dir, os.listdir(log_dir )
df_titanic['Alone'] = df_titanic['Family'].map(lambda i: 1 if i == 1 else 0)
 df_titanic['Small'] = df_titanic['Family'].map(lambda i: 1 if i == 2 else 0)
 df_titanic['Medium'] = df_titanic['Family'].map(lambda i: 1 if 3 <= i <= 4 else 0)
 df_titanic['Large'] = df_titanic['Family'].map(lambda i: 1 if i >= 5 else 0 )<categorify>
print("Starting...
 ")
 start_time = time.time()
batch_size = 32
target_size =(299, 299)
 train_generator, validation_generator, test_generator, class_weights, steps_per_epoch, validation_steps = get_data(batch_size=batch_size, target_size=target_size, classes=classes, df_test=df_test)
 print("
Completed...
 " )
df_titanic['Pclass'] = df_titanic['Pclass'].astype("category")
 columns = ['Title', 'Surname', 'Cabin', 'Ticket', 'Pclass']
for col in columns:
df_titanic = pd.get_dummies(df_titanic, columns=[col], prefix=col )<drop_column>
loss = 'categorical_crossentropy'
metrics = ['accuracy']
df_titanic.drop(labels = ["PassengerId"], axis = 1, inplace = True )<data_type_conversions>
print("Starting...
 ")
 start_time = time.time()
base_model = InceptionV3(weights='imagenet', include_top=False)
 x = base_model.output
x = GlobalAveragePooling2D()(x)
 x = Dense(1024, activation='relu' )(x)
 predictions = Dense(2, activation='softmax' )(x)
 model = Model(inputs=base_model.input, outputs=predictions)
 for layer in base_model.layers:
layer.trainable = False
print("
Compliling Model...
 ")
 learning_rate = 0.0001
optimizer = Adam(learning_rate)
 model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
 verbose = 1
epochs = 30
print("Trainning Model...
 ")
 history = model.fit_generator(
 train_generator,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
verbose=verbose,
callbacks=callbacks,
validation_data=validation_generator,
validation_steps=validation_steps,
class_weight=class_weights)
 elapsed_time = time.time() - start_time
elapsed_time = time.strftime("%H:%M:%S", time.gmtime(elapsed_time))
print("
Elapsed Time: " + elapsed_time)
 ", date_time(1))
print("
" )
df_titanic_train = df_titanic[:train_size]
df_titanic_test = df_titanic[train_size:]
df_titanic_train['Survived'] = df_titanic_train['Survived'].astype(int)
 del df_titanic_test['Survived']<normalization>
def generate_result(model, test_generator, nsteps=len(test_generator)) :
y_preds = model.predict_generator(test_generator, steps=nsteps, verbose=1)
 return y_preds, y_preds[:,1]
X_train = df_titanic_train.drop(['Survived'], axis=1)
 y_train = df_titanic_train['Survived']
X_test = df_titanic_test
sc = StandardScaler()
sc.fit(X_train)
 X_train = sc.transform(X_train)
 X_test = sc.transform(X_test )<choose_model_class>
y_preds_all, y_preds = generate_result(model, test_generator )
kfold = StratifiedKFold(n_splits=10)
 seed = 20
clfs = []
clfs.append(SVC(random_state=seed))
clfs.append(DecisionTreeClassifier(random_state=seed))
clfs.append(RandomForestClassifier(random_state=seed))
clfs.append(ExtraTreesClassifier(random_state=seed))
clfs.append(GradientBoostingClassifier(random_state=seed))
clfs.append(MLPClassifier(random_state=seed))
clfs.append(KNeighborsClassifier())
 clfs.append(LogisticRegression(random_state=seed))
clfs.append(XGBClassifier(random_state = seed))<compute_train_metric>
df_test = pd.DataFrame({"id": test_generator.filenames, "label": y_preds})
 df_test['id'] = df_test['id'].map(lambda x: x.split('.')[0])
 df_test['id'] = df_test['id'].astype(int)
 df_test = df_test.sort_values('id')
 df_test.to_csv('submission.csv', index=False)
 df_test.head()
<create_dataframe><EOS> 
df_test2 = pd.read_csv("submission.csv")
 df_test2.head()