Kaggle competition: Aerial Cactus Identification
<SOS> metric: AUC Kaggle data source: aerial-cactus-identification<split>
import random
import os
import sklearn.utils
from tqdm import tqdm, tqdm_notebook
import pandas as pd
import cv2 as cv
from tqdm import tqdm
import tensorflow as tf
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Conv2D, Dense, Flatten, BatchNormalization, Dropout, LeakyReLU, DepthwiseConv2D, Flatten
from tensorflow.keras.layers import GlobalAveragePooling2D
from tensorflow.keras.callbacks import ModelCheckpoint,ReduceLROnPlateau,EarlyStopping
import json
import logging
import numpy as np
import matplotlib.pyplot as plt
remain_train = combine[(combine.Predict.isna())&(combine.Survived.notna())]
remain_test = combine[(combine.Predict.isna())&(combine.Survived.isna())]<count_values>
seed(1372)
 set_random_seed(1372)
print("Pclass_Gender distribution in train data:",remain_train.Pclass_Gender.value_counts())
 print("-"*50)
 print("Pclass_Gender distribution in test data:",remain_test.Pclass_Gender.value_counts() )<filter>
def resize_and_save(filename, input_dir, output_dir, size=32):

 image = Image.open(os.path.join(input_dir, filename))
image.save(os.path.join(output_dir, filename))
survival_in_remain = remain_train[(remain_train.Pclass_Gender=="P1-Man")]["Survived"].mean()
survival_in_all = combine[(combine.Pclass_Gender=="P1-Man")]["Survived"].mean()
print(round(survival_in_all,2),round(survival_in_remain,2))<filter>
class Params() :

 def __init__(self, json_path):
self.update(json_path)
 def save(self, json_path):

 with open(json_path, 'w')as f:
json.dump(self.__dict__, f, indent=4)
 def update(self, json_path):

 with open(json_path)as f:
params = json.load(f)
 self.__dict__.update(params)
 @property
def dict(self):

 return self.__dict__
def set_logger(log_path):

 logger = logging.getLogger()
logger.setLevel(logging.INFO)
 if not logger.handlers:
file_handler = logging.FileHandler(log_path)
 file_handler.setFormatter(logging.Formatter('%(asctime)s:%(levelname)s: %(message)s'))
logger.addHandler(file_handler)
 stream_handler = logging.StreamHandler()
stream_handler.setFormatter(logging.Formatter('%(message)s'))
logger.addHandler(stream_handler)
 def save_dict_to_json(d, json_path):

 with open(json_path, 'w')as f:
d = {k: float(v)for k, v in d.items() }
json.dump(d, f, indent=4)
survival_in_remain = remain_train[(remain_train.Pclass_Gender=="P3-Man")]["Survived"].mean()
survival_in_all = combine[(combine.Pclass_Gender=="P3-Man")]["Survived"].mean()
print(round(survival_in_all,2),round(survival_in_remain,2))<feature_engineering>
df = pd.read_csv(csv_dir)
 df['id'] = dataset_dir + '/' + df['id'].astype(str)
 filenames = df['id']
labels = df['has_cactus'].astype(np.float32)
 df = sklearn.utils.shuffle(df,random_state=1372)
 df = df.reset_index(drop=True )
combine.loc[(combine.Predict.isna())&(combine.Survived.isna())&(combine.Pclass_Gender=="P3-Woman"),"Predict"] = 1
combine.loc[(combine.Predict.isna())&(combine.Survived.isna()),"Predict"]=0<create_dataframe>
def _parse_function(filename, label, size):

 image_string = tf.read_file(filename)
 image_decoded = tf.image.decode_jpeg(image_string, channels=3)
 image = tf.image.convert_image_dtype(image_decoded, tf.float32)
 resized_image = tf.image.resize_images(image, [size, size])
 return resized_image, label
final_submit = pd.DataFrame({"PassengerId":combine[combine.Survived.isna() ].PassengerId.values,
"Survived":combine[combine.Survived.isna() ].Predict.values})
 final_submit["Survived"] = final_submit.Survived.astype("int64")
 final_submit.tail(2 )<save_to_csv>
def input_fn(is_training, filenames, labels, params):

 num_samples = len(filenames)
 assert len(filenames)== len(labels), "Filenames and labels should have same length"
parse_fn = lambda f, l: _parse_function(f, l, params.image_size)
 train_fn = lambda f, l: train_preprocess(f, l, params.use_random_flip)
 if is_training:
dataset =(tf.data.Dataset.from_tensor_slices(( tf.constant(filenames), tf.constant(labels)))
.shuffle(num_samples)
.map(parse_fn, num_parallel_calls=params.num_parallel_calls)
.map(train_fn, num_parallel_calls=params.num_parallel_calls)
.batch(params.batch_size)
.repeat() 
.prefetch(32)
)
 else:
dataset =(tf.data.Dataset.from_tensor_slices(( tf.constant(filenames), tf.constant(labels)))
.map(parse_fn)
.batch(params.batch_size)
.repeat() 
.prefetch(32)
)
 return dataset
final_submit.to_csv("Titanic.csv",index = False )<set_options>
params = Params(json_path)
 split = int(len(filenames)*0.15)
 train_dataset = input_fn(True, filenames[:split], labels[:split], params)
 valid_dataset = input_fn(False , filenames[split:], labels[split:], params )
%matplotlib inline
pd.set_option('display.max_rows', 100)
 pd.set_option('display.max_columns', 500)
<load_from_csv>
model = Sequential()
model.add(Conv2D(3, kernel_size = 3, activation = 'relu', input_shape =(32, 32, 3)))
 model.add(Conv2D(filters = 16, kernel_size = 3, activation = 'relu'))
model.add(Conv2D(filters = 16, kernel_size = 3, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(DepthwiseConv2D(kernel_size = 3, strides = 1, padding = 'Same', use_bias = True))
model.add(Conv2D(filters = 32, kernel_size = 1, activation = 'relu'))
model.add(Conv2D(filters = 64, kernel_size = 1, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(DepthwiseConv2D(kernel_size = 3, strides = 2, padding = 'Same', use_bias = True))
model.add(Conv2D(filters = 128, kernel_size = 1, activation = 'relu'))
model.add(Conv2D(filters = 256, kernel_size = 1, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(DepthwiseConv2D(kernel_size = 3, strides = 1, padding = 'Same', use_bias = True))
model.add(Conv2D(filters = 256, kernel_size = 1, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 512, kernel_size = 1, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(DepthwiseConv2D(kernel_size = 3, strides = 2, padding = 'Same', use_bias = True))
model.add(Conv2D(filters = 512, kernel_size = 1, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 512, kernel_size = 1, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(DepthwiseConv2D(kernel_size = 3, strides = 1, padding = 'Same', use_bias = True))
model.add(Conv2D(filters = 1024, kernel_size = 1, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 1024, kernel_size = 1, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Flatten())
 model.add(Dense(512, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(256, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(128, activation = 'elu'))
model.add(Dense(1, activation = 'sigmoid'))
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<load_from_csv>
Adam = tf.keras.optimizers.Adam(lr=params.learning_rate , amsgrad=True)
 model.compile(optimizer = Adam, loss = tf.losses.log_loss, metrics = ['accuracy'])
 model.summary()
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<drop_column>
file_path = 'weights-aerial-cactus.h5'
callbacks = [
ModelCheckpoint(file_path, monitor = 'val_acc', verbose = 1, save_best_only = True, mode = 'max'),
ReduceLROnPlateau(monitor = 'val_loss', factor = 0.2, patience = 3, verbose = 1, mode = 'min', min_lr = 0.00001),
EarlyStopping(monitor = 'val_loss', min_delta = 1e-10, patience = 15, verbose = 1, restore_best_weights = True)
 ]
train.drop(train[(train['GrLivArea'] > 4000)&(train['SalePrice'] < 200000)].index,inplace=True )<count_values>
history = model.fit(train_dataset, validation_data=valid_dataset,
epochs=50,verbose=True,
steps_per_epoch=int(( len(filenames)- split)/params.batch_size),
validation_steps=int(split/params.batch_size),
callbacks = callbacks )
train.dtypes.value_counts()<count_values>
model.load_weights(file_path )
test.dtypes.value_counts()<drop_column>
test_df = pd.read_csv('.. /input/sample_submission.csv')
 X_test = []
images_test = test_df['id'].values
for img_id in tqdm_notebook(images_test):
X_test.append(cv.imread('.. /input/test/test/' + img_id))
X_test = np.asarray(X_test)
 X_test = X_test.astype('float32')
 X_test /= 255
y_test_pred = model.predict_proba(X_test)
 test_df['has_cactus'] = y_test_pred
test_df.to_csv('aerial-cactus-submission_1.csv', index = False)
 for i in range(len(y_test_pred)) :
if y_test_pred[i][0] >= 0.5:
y_test_pred[i][0] = 1.0
else:
y_test_pred[i][0] = 0.0
test_df['has_cactus'] = y_test_pred
test_df.to_csv('aerial-cactus-submission_2.csv', index = False )