Kaggle competition: Google Landmark Retrieval 2020
<SOS> metric: custom_loss Kaggle data source: landmark-retrieval-2020<set_options>
%matplotlib inline
plt.style.use('seaborn-white')
 sns.set_style("white")
<define_variables>
import numpy as np
import pandas as pd
from PIL import Image
import matplotlib.pyplot as plt
import os
import cv2
import glob
import tensorflow as tf
import keras
from keras.models import load_model, save_model
from keras.layers import Input, GlobalAveragePooling2D, GlobalMaxPooling2D
import keras.backend as K
from keras.models import Model, load_model
from keras.applications import VGG16
from keras.applications.vgg16 import preprocess_input
t_start = time.time()
version = 11
basic_name = f'Unet_resnet_version_{version}'
save_model_name = basic_name + '.model'
submission_file = basic_name + '.csv'
print(save_model_name)
 print(submission_file )<train_model>
train_df = pd.read_csv('.. /input/landmark-retrieval-2020/train.csv')
 train_df.head()
img_size_ori = 101
img_size_target = 101
epochs = 75
batch_size = 32
def upsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_target, img_size_target), mode='constant', preserve_range=True)
 def downsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_ori, img_size_ori), mode='constant', preserve_range=True)
<load_from_csv>
train_df[train_df.id == '000171b259e48280']
train_df = pd.read_csv(".. /input/tgs-salt-identification-challenge/train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(".. /input/tgs-salt-identification-challenge/depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]<feature_engineering>
train_df[train_df.landmark_id == 5724]
train_df["images"] = [np.array(load_img(".. /input/tgs-salt-identification-challenge/train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]<feature_engineering>
number_image = train_df[train_df.landmark_id == 107382].count().values[1]
number_image
train_df["masks"] = [np.array(load_img(".. /input/tgs-salt-identification-challenge/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]<feature_engineering>
for number in np.arange(0,number_image):
print(number )
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size_ori, 2 )<categorify>
train_df[train_df.landmark_id == 107382].id.values[1]
def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class )<split>
folder_path_number = []
for i in np.arange(0,3):
folder_path_number.append(train_df[train_df.landmark_id == 107382].id.values[0][i])
 folder_path_number
ids_train, ids_valid, x_train, x_valid, y_train, y_valid, cov_train, cov_test, depth_train, depth_test = train_test_split(
 train_df.index.values,
np.array(train_df.images.map(upsample ).tolist() ).reshape(-1,img_size_target, img_size_target, 1),
np.array(train_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
train_df.coverage.values,
train_df.z.values,
test_size = 0.05 , stratify=train_df.coverage_class, random_state=42 )<define_search_model>
nameofimage = train_df[train_df.landmark_id == 107382].id.values[0]
image_path =(f'.. /input/landmark-retrieval-2020/train/{folder_path_number[0]}/{folder_path_number[1]}/{folder_path_number[2]}/{nameofimage}.jpg')
 image_path
def BatchActivate(x):
x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 return x
def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True):
x = Conv2D(filters, size, strides=strides, padding=padding )(x)
 if activation == True:
x = BatchActivate(x)
 return x
def residual_block(blockInput, num_filters=16, batch_activate = False):
x = BatchActivate(blockInput)
 x = convolution_block(x, num_filters,(3,3))
x = convolution_block(x, num_filters,(3,3), activation=False)
 x = Add()([x, blockInput])
 if batch_activate:
x = BatchActivate(x)
 return x<choose_model_class>
number_image = train_df[train_df.landmark_id == 5724].count().values[1]
length = 8*number_image
fig, axes = plt.subplots(1, number_image, figsize=(length, 6))
for number in np.arange(0,number_image):
folder_path_number = []
for i in np.arange(0,3):
folder_path_number.append(train_df[train_df.landmark_id == 5724].id.values[number][i])
 nameofimage = train_df[train_df.landmark_id == 5724].id.values[number]
image_path =(f'.. /input/landmark-retrieval-2020/train/{folder_path_number[0]}/{folder_path_number[1]}/{folder_path_number[2]}/{nameofimage}.jpg')
 image = cv2.imread(image_path)
 axes[number].imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
axes[number].axis('off')
 axes[number].set_title(f'id of jpg image: {nameofimage}' )
size =(3, 3)
 def build_model(input_layer, start_neurons, DropoutRatio = 0.5):
conv1 = Conv2D(start_neurons * 1, size, activation="relu", padding="same" )(input_layer)
 conv1 = residual_block(conv1,start_neurons * 1)
 conv1 = residual_block(conv1,start_neurons * 1, True)
 pool1 = MaxPooling2D(( 2, 2))(conv1)
 pool1 = Dropout(DropoutRatio/2 )(pool1)
 conv2 = Conv2D(start_neurons * 2, size, activation=None, padding="same" )(pool1)
 conv2 = residual_block(conv2,start_neurons * 2)
 conv2 = residual_block(conv2,start_neurons * 2, True)
 pool2 = MaxPooling2D(( 2, 2))(conv2)
 pool2 = Dropout(DropoutRatio )(pool2)
 conv3 = Conv2D(start_neurons * 4, size, activation=None, padding="same" )(pool2)
 conv3 = residual_block(conv3,start_neurons * 4)
 conv3 = residual_block(conv3,start_neurons * 4, True)
 pool3 = MaxPooling2D(( 2, 2))(conv3)
 pool3 = Dropout(DropoutRatio )(pool3)
 conv4 = Conv2D(start_neurons * 8, size, activation=None, padding="same" )(pool3)
 conv4 = residual_block(conv4,start_neurons * 8)
 conv4 = residual_block(conv4,start_neurons * 8, True)
 pool4 = MaxPooling2D(( 2, 2))(conv4)
 pool4 = Dropout(DropoutRatio )(pool4)
 convm = Conv2D(start_neurons * 16, size, activation=None, padding="same" )(pool4)
 convm = residual_block(convm,start_neurons * 16)
 convm = residual_block(convm,start_neurons * 16, True)
 deconv4 = Conv2DTranspose(start_neurons * 8, size, strides=(2, 2), padding="same" )(convm)
 uconv4 = concatenate([deconv4, conv4])
 uconv4 = Dropout(DropoutRatio )(uconv4)
 uconv4 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(uconv4)
 uconv4 = residual_block(uconv4,start_neurons * 8)
 uconv4 = residual_block(uconv4,start_neurons * 8, True)
 deconv3 = Conv2DTranspose(start_neurons * 4, size, strides=(2, 2), padding="valid" )(uconv4)
 uconv3 = concatenate([deconv3, conv3])
 uconv3 = Dropout(DropoutRatio )(uconv3)
 uconv3 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(uconv3)
 uconv3 = residual_block(uconv3,start_neurons * 4)
 uconv3 = residual_block(uconv3,start_neurons * 4, True)
 deconv2 = Conv2DTranspose(start_neurons * 2, size, strides=(2, 2), padding="same" )(uconv3)
 uconv2 = concatenate([deconv2, conv2])
 uconv2 = Dropout(DropoutRatio )(uconv2)
 uconv2 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(uconv2)
 uconv2 = residual_block(uconv2,start_neurons * 2)
 uconv2 = residual_block(uconv2,start_neurons * 2, True)
 deconv1 = Conv2DTranspose(start_neurons * 1, size, strides=(2, 2), padding="valid" )(uconv2)
 uconv1 = concatenate([deconv1, conv1])
 uconv1 = Dropout(DropoutRatio )(uconv1)
 uconv1 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(uconv1)
 uconv1 = residual_block(uconv1,start_neurons * 1)
 uconv1 = residual_block(uconv1,start_neurons * 1, True)
 output_layer_noActi = Conv2D(1,(1,1), padding="same", activation=None )(uconv1)
 output_layer = Activation('sigmoid' )(output_layer_noActi)
 return output_layer<compute_test_metric>
def get_path(landmark_id):
number_image = train_df[train_df.landmark_id == landmark_id].count().values[1]
image_path = []
for number in np.arange(0,number_image):
folder_path_number = []
for i in np.arange(0,3):
folder_path_number.append(train_df[train_df.landmark_id == landmark_id].id.values[number][i])
 nameofimage = train_df[train_df.landmark_id == landmark_id].id.values[0]
image_path.append(( f'.. /input/landmark-retrieval-2020/train/{folder_path_number[0]}/{folder_path_number[1]}/{folder_path_number[2]}/{nameofimage}.jpg'))
return image_path
def get_iou_vector(A, B):
batch_size = A.shape[0]
metric = []
for batch in range(batch_size):
t, p = A[batch]>0, B[batch]>0
intersection = np.logical_and(t, p)
 union = np.logical_or(t, p)
 iou =(np.sum(intersection > 0)+ 1e-10)/(np.sum(union > 0)+ 1e-10)
 thresholds = np.arange(0.5, 1, 0.05)
 s = []
for thresh in thresholds:
s.append(iou > thresh)
 metric.append(np.mean(s))
return np.mean(metric)
 def my_iou_metric(label, pred):
return tf.py_func(get_iou_vector, [label, pred > 0.3], tf.float64)
 def my_iou_metric_2(label, pred):
return tf.py_func(get_iou_vector, [label, pred > 0.1], tf.float64)
 def my_iou_metric_3(label, pred):
return tf.py_func(get_iou_vector, [label, pred > 0.3], tf.float64 )<compute_train_metric>
print('The number of the unique landmark id in the dataset is', train_df.landmark_id.nunique() )
def lovasz_grad(gt_sorted):

 gts = tf.reduce_sum(gt_sorted)
 intersection = gts - tf.cumsum(gt_sorted)
 union = gts + tf.cumsum(1.- gt_sorted)
 jaccard = 1.- intersection / union
jaccard = tf.concat(( jaccard[0:1], jaccard[1:] - jaccard[:-1]), 0)
 return jaccard
def lovasz_hinge(logits, labels, per_image=True, ignore=None):

 if per_image:
def treat_image(log_lab):
log, lab = log_lab
log, lab = tf.expand_dims(log, 0), tf.expand_dims(lab, 0)
 log, lab = flatten_binary_scores(log, lab, ignore)
 return lovasz_hinge_flat(log, lab)
 losses = tf.map_fn(treat_image,(logits, labels), dtype=tf.float32)
 loss = tf.reduce_mean(losses)
 else:
loss = lovasz_hinge_flat(*flatten_binary_scores(logits, labels, ignore))
return loss
def lovasz_hinge_flat(logits, labels):

 def compute_loss() :
labelsf = tf.cast(labels, logits.dtype)
 signs = 2.* labelsf - 1.
 errors = 1.- logits * tf.stop_gradient(signs)
 errors_sorted, perm = tf.nn.top_k(errors, k=tf.shape(errors)[0], name="descending_sort")
 gt_sorted = tf.gather(labelsf, perm)
 grad = lovasz_grad(gt_sorted)
 loss = tf.tensordot(tf.nn.relu(errors_sorted), tf.stop_gradient(grad), 1, name="loss_non_void")
 return loss
loss = tf.cond(tf.equal(tf.shape(logits)[0], 0),
lambda: tf.reduce_sum(logits)* 0.,
compute_loss,
strict=True,
name="loss" 
)
 return loss
def flatten_binary_scores(scores, labels, ignore=None):

 scores = tf.reshape(scores,(-1,))
labels = tf.reshape(labels,(-1,))
if ignore is None:
return scores, labels
valid = tf.not_equal(labels, ignore)
 vscores = tf.boolean_mask(scores, valid, name='valid_scores')
 vlabels = tf.boolean_mask(labels, valid, name='valid_labels')
 return vscores, vlabels
def lovasz_loss(y_true, y_pred):
y_true, y_pred = K.cast(K.squeeze(y_true, -1), 'int32'), K.cast(K.squeeze(y_pred, -1), 'float32')
 logits = y_pred
loss = lovasz_hinge(logits, y_true, per_image = True, ignore = None)
 return loss<choose_model_class>
files = glob.glob(".. /input/landmark-retrieval-2020/train/a/b/c/*.jpg")
 for i in range(10):
im = cv2.imread(files[i])
 print(im.shape )
def _bn_relu(input):

 norm = BatchNormalization(axis=CHANNEL_AXIS )(input)
 return Activation("relu" )(norm)
 def _conv_bn_relu(**conv_params):

 filters = conv_params["filters"]
kernel_size = conv_params["kernel_size"]
strides = conv_params.setdefault("strides",(1, 1))
kernel_initializer = conv_params.setdefault("kernel_initializer", "he_normal")
 padding = conv_params.setdefault("padding", "same")
 kernel_regularizer = conv_params.setdefault("kernel_regularizer", l2(1.e-4))
def f(input):
conv = Conv2D(filters=filters, kernel_size=kernel_size,
strides=strides, padding=padding,
kernel_initializer=kernel_initializer,
kernel_regularizer=kernel_regularizer )(input)
 return _bn_relu(conv)
 return f
def _bn_relu_conv(**conv_params):

 filters = conv_params["filters"]
kernel_size = conv_params["kernel_size"]
strides = conv_params.setdefault("strides",(1, 1))
kernel_initializer = conv_params.setdefault("kernel_initializer", "he_normal")
 padding = conv_params.setdefault("padding", "same")
 kernel_regularizer = conv_params.setdefault("kernel_regularizer", l2(1.e-4))
def f(input):
activation = _bn_relu(input)
 return Conv2D(filters=filters, kernel_size=kernel_size,
strides=strides, padding=padding,
kernel_initializer=kernel_initializer,
kernel_regularizer=kernel_regularizer )(activation)
 return f
def _shortcut(input, residual):

 input_shape = K.int_shape(input)
 residual_shape = K.int_shape(residual)
 stride_width = int(round(input_shape[ROW_AXIS] / residual_shape[ROW_AXIS]))
stride_height = int(round(input_shape[COL_AXIS] / residual_shape[COL_AXIS]))
equal_channels = input_shape[CHANNEL_AXIS] == residual_shape[CHANNEL_AXIS]
shortcut = input
if stride_width > 1 or stride_height > 1 or not equal_channels:
shortcut = Conv2D(filters=residual_shape[CHANNEL_AXIS],
kernel_size=(1, 1),
strides=(stride_width, stride_height),
padding="valid",
kernel_initializer="he_normal",
kernel_regularizer=l2(0.0001))(input)
 return add([shortcut, residual])
 def basic_block(filters, init_strides=(1, 1), is_first_block_of_first_layer=False):

 def f(input):
if is_first_block_of_first_layer:
conv1 = Conv2D(filters=filters, kernel_size=(3, 3),
strides=init_strides,
padding="same",
kernel_initializer="he_normal",
kernel_regularizer=l2(1e-4))(input)
 else:
conv1 = _bn_relu_conv(filters=filters, kernel_size=(3, 3),
strides=init_strides )(input)
 residual = _bn_relu_conv(filters=filters, kernel_size=(3, 3))(conv1)
 return _shortcut(input, residual)
 return f
def _residual_block(block_function, filters, repetitions, is_first_layer=False):

 def f(input):
for i in range(repetitions):
init_strides =(1, 1)
 if i == 0 and not is_first_layer:
init_strides =(2, 2)
 input = block_function(filters=filters, init_strides=init_strides,
is_first_block_of_first_layer=(is_first_layer and i == 0))(input)
 return input
return f
def _handle_dim_ordering() :
global ROW_AXIS
global COL_AXIS
global CHANNEL_AXIS
if K.image_dim_ordering() == 'tf':
ROW_AXIS = 1
COL_AXIS = 2
CHANNEL_AXIS = 3
else:
CHANNEL_AXIS = 1
ROW_AXIS = 2
COL_AXIS = 3
def _get_block(identifier):
if isinstance(identifier, six.string_types):
res = globals().get(identifier)
 if not res:
raise ValueError('Invalid {}'.format(identifier))
return res
return identifier
class ResnetBuilder(object):
@staticmethod
def build(input_shape, block_fn, repetitions,input_tensor):
_handle_dim_ordering()
if len(input_shape)!= 3:
raise Exception("Input shape should be a tuple(nb_channels, nb_rows, nb_cols)")
 if K.image_dim_ordering() == 'tf':
input_shape =(input_shape[1], input_shape[2], input_shape[0])
 block_fn = _get_block(block_fn)
 if input_tensor is None:
img_input = Input(shape=input_shape)
 else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
 else:
img_input = input_tensor
conv1 = _conv_bn_relu(filters=64, kernel_size=(7, 7), strides=(2, 2))(img_input)
 pool1 = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding="same" )(conv1)
 block = pool1
filters = 64
for i, r in enumerate(repetitions):
block = _residual_block(block_fn, filters=filters, repetitions=r, is_first_layer=(i == 0))(block)
 filters *= 2
block = _bn_relu(block)
 model = Model(inputs=img_input, outputs=block)
 return model
@staticmethod
def build_resnet_34(input_shape,input_tensor):
return ResnetBuilder.build(input_shape, basic_block, [3, 4, 6, 3],input_tensor )<choose_model_class>
vgg = VGG16(input_shape=(224,224,3), weights=None, include_top=False)
 vgg.load_weights(".. /input/keras-pretrained-models/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5")
 input_image = Input(( 224,224,3))
x = vgg(input_image)
 output = GlobalMaxPooling2D()(x)
 model = Model(inputs=[input_image], outputs=[output])
 model.summary()
def UResNet34(input_shape=(128, 128, 1), classes=1, decoder_filters=16, decoder_block_type='upsampling',
encoder_weights="imagenet", input_tensor=None, activation='sigmoid', **kwargs):
backbone = ResnetBuilder.build_resnet_34(input_shape=input_shape,input_tensor=input_tensor)
 input_layer = backbone.input
output_layer = build_model(input_layer, 16,0.5)
 model = Model(input_layer, output_layer)
 c = optimizers.adam(lr = 0.01)
 model.compile(loss="binary_crossentropy", optimizer=c, metrics=[my_iou_metric])
 model.name = 'u-resnet34'
return model<choose_model_class>
class MyModel(tf.keras.Model):
def __init__(self):
super(MyModel, self ).__init__()
self.model = model
@tf.function(input_signature=[
tf.TensorSpec(shape=[None, None, 3], dtype=tf.uint8, name='input_image')
 ])
 def call(self, input_image):
output_tensors = {}
im = tf.image.resize(input_image,(224,224))
im = preprocess_input(im)
 extracted_features = self.model(tf.convert_to_tensor([im], dtype=tf.uint8)) [0]
output_tensors['global_descriptor'] = tf.identity(extracted_features, name='global_descriptor')
 return output_tensors
model1 = UResNet34(input_shape =(1,img_size_target,img_size_target)) 
<concatenate>
m = MyModel()
served_function = m.call
tf.saved_model.save(
 m, export_dir="./my_model", signatures={'serving_default': served_function} )
<train_model><EOS> 
with ZipFile('submission.zip','w')as zip:
zip.write('./my_model/saved_model.pb', arcname='saved_model.pb')
 zip.write('./my_model/variables/variables.data-00000-of-00001', arcname='variables/variables.data-00000-of-00001')
 zip.write('./my_model/variables/variables.index', arcname='variables/variables.index')