Kaggle competition: Dog Breed Identification
<SOS> metric: MulticlassLoss Kaggle data source: dog-breed-identification<compute_test_metric>
print(tf.__version__ )
class FocalLoss(nn.Module):
def __init__(self, gamma=2):
super().__init__()
self.gamma = gamma
def forward(self, logit, target):
target = target.float()
max_val =(-logit ).clamp(min=0)
 loss = logit - logit * target + max_val + \ 
(( -max_val ).exp() +(-logit - max_val ).exp() ).log()
invprobs = F.logsigmoid(-logit *(target * 2.0 - 1.0))
loss =(invprobs * self.gamma ).exp() * loss
if len(loss.size())==2:
loss = loss.sum(dim=1)
 return loss.mean()<choose_model_class>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tensorflow import keras
import cv2
import PIL
import os
import pathlib
import shutil
from IPython.display import Image, display
import plotly.graph_objs as go
import plotly.graph_objects as go
from sklearn.metrics import cohen_kappa_score
from sklearn.model_selection import train_test_split
from tensorflow.keras.models import Sequential, Model,load_model
from tensorflow.keras.applications import vgg16
from tensorflow.keras.applications import resnet50
from tensorflow.keras.applications import xception
from tensorflow.keras.applications import inception_v3
from tensorflow.keras.applications import inception_resnet_v2
from tensorflow.keras.applications import resnet_v2
from tensorflow.keras.applications import nasnet
from tensorflow.keras.preprocessing.image import ImageDataGenerator,load_img, img_to_array
from tensorflow.keras import Input
from tensorflow.keras.layers import Conv2D, MaxPooling2D, Dense, Dropout, Flatten,BatchNormalization,Activation
from tensorflow.keras.layers import GlobalAveragePooling2D, Concatenate
from tensorflow.keras.optimizers import Adam, SGD, RMSprop
from tensorflow.keras.callbacks import ModelCheckpoint, Callback, EarlyStopping, LearningRateScheduler
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.preprocessing import image_dataset_from_directory
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.layers.experimental import preprocessing
import gc
import skimage.io
learn = cnn_learner(data, base_arch=models.resnet152, loss_func=FocalLoss() , metrics=fbeta )<find_best_params>
train_dir = '.. /input/dog-breed-dataset-with-subdirectories-by-class/data/train'
test_dir = '.. /input/dog-breed-dataset-with-subdirectories-by-class/data/test'
train_labels = pd.read_csv('.. /input/dog-breed-identification/labels.csv', index_col = 'id')
 submission=pd.read_csv('.. /input/dog-breed-identification/sample_submission.csv' )
learn.lr_find()
learn.recorder.plot()<train_model>
target, dog_breeds = pd.factorize(train_labels['breed'], sort = True)
 train_labels['target'] = target
print(dog_breeds )
learn.unfreeze()
learn.fit_one_cycle(18, slice(1e-5,1e-2))
learn.freeze()
learn.save('stage-1', return_path=True )<predict_on_test>
train_labels['breed'].value_counts()
valid_preds = learn.get_preds(DatasetType.Valid)
 best_thr = find_best_fixed_threshold(*valid_preds )<categorify>
N_EPOCHS = 50
BATCH_SIZE = 128
IMG_HEIGHT = 331
IMG_WIDTH = 331
test_preds = learn.TTA(ds_type=DatasetType.Test)
 test_df.attribute_ids = join_preds(test_preds, best_thr)
 test_df.head()<save_to_csv>
train_ds = image_dataset_from_directory(
 directory = train_dir,
labels = 'inferred',
label_mode='int',
batch_size=BATCH_SIZE,
image_size=(IMG_HEIGHT, IMG_WIDTH),
shuffle = True,
seed=1234,
validation_split=0.1,
subset="training",
)
test_df.to_csv('submission.csv', index=False )<set_options>
class_names = train_ds.class_names
print(len(class_names))
print(class_names )
WORKERS = 2
CHANNEL = 3
warnings.filterwarnings("ignore")
 SIZE = 156
NUM_CLASSES = 1103
beta_f2=2<load_from_csv>
val_ds = image_dataset_from_directory(
 directory = train_dir,
labels = 'inferred',
label_mode='int',
batch_size=BATCH_SIZE,
image_size=(IMG_HEIGHT, IMG_WIDTH),
shuffle = True,
seed=1234,
validation_split=0.1,
subset="validation",
)
path_to_train = '.. /input/imet-2019-fgvc6/train/'
data = pd.read_csv('.. /input/imet-2019-fgvc6/train.csv')
 train_dataset_info = []
for name, labels in zip(data['id'], data['attribute_ids'].str.split(' ')) :
train_dataset_info.append({
'path':os.path.join(path_to_train, name),
'labels':np.array([int(label)for label in labels])})
 train_dataset_info = np.array(train_dataset_info )<compute_test_metric>
test_ds = image_dataset_from_directory(
 directory = test_dir,
label_mode= None,
batch_size=BATCH_SIZE,
image_size=(IMG_HEIGHT, IMG_WIDTH),
shuffle = False,
seed=1234
)
gamma = 2.0
epsilon = K.epsilon()
def focal_loss(y_true, y_pred):
pt = y_pred * y_true +(1-y_pred)*(1-y_true)
 pt = K.clip(pt, epsilon, 1-epsilon)
 CE = -K.log(pt)
 FL = K.pow(1-pt, gamma)* CE
loss = K.sum(FL, axis=1)
 return loss<define_variables>
del class_names
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
 class data_generator(Sequence):
def create_train(dataset_info, batch_size, shape, augument=True):
assert shape[2] == 3
while True:
dataset_info = shuffle(dataset_info)
 for start in range(0, len(dataset_info), batch_size):
end = min(start + batch_size, len(dataset_info))
batch_images = []
X_train_batch = dataset_info[start:end]
batch_labels = np.zeros(( len(X_train_batch), NUM_CLASSES))
for i in range(len(X_train_batch)) :
image = data_generator.load_image(
 X_train_batch[i]['path'], shape)
 if augument:
image = data_generator.augment(image)
 batch_images.append(image/255.)
batch_labels[i][X_train_batch[i]['labels']] = 1
yield np.array(batch_images, np.float32), batch_labels
def create_valid(dataset_info, batch_size, shape, augument=False):
assert shape[2] == 3
while True:
for start in range(0, len(dataset_info), batch_size):
end = min(start + batch_size, len(dataset_info))
batch_images = []
X_train_batch = dataset_info[start:end]
batch_labels = np.zeros(( len(X_train_batch), NUM_CLASSES))
for i in range(len(X_train_batch)) :
image = data_generator.load_image(
 X_train_batch[i]['path'], shape)
 if augument:
image = data_generator.augment(image)
 batch_images.append(image/255.)
batch_labels[i][X_train_batch[i]['labels']] = 1
yield np.array(batch_images, np.float32), batch_labels
def load_image(path, shape):
image = cv2.imread(path+'.png')
 image = cv2.resize(image,(SIZE, SIZE))
return image
def augment(image):
augment_img = iaa.Sequential([
iaa.OneOf([
iaa.Affine(rotate=0),
iaa.Affine(rotate=(-15,15)) ,
iaa.Crop(px=(0, 16)) ,
iaa.Affine(shear=(-5, 5)) ,
iaa.GaussianBlur(sigma=(0, 0.5)) ,
iaa.Fliplr(0.5),
])], random_order=True)
 image_aug = augment_img.augment_image(image)
 return image_aug 
<import_modules>
plt.figure(figsize=(20, 20))
for images, labels in train_ds.take(1):
for i in range(16):
ax = plt.subplot(4, 4, i + 1)
 plt.imshow(images[i].numpy().astype("uint8"))
plt.title(dog_breeds[labels[i]])
 plt.axis("off" )
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential, load_model
from keras.layers import *
from keras.applications import *
from keras.callbacks import ModelCheckpoint
from keras import metrics
from keras.optimizers import Adam
from keras import backend as K
import keras
from keras.models import Model<choose_model_class>
plt.figure(figsize=(20, 20))
for images, labels in val_ds.take(1):
for i in range(16):
ax = plt.subplot(4, 4, i + 1)
 plt.imshow(images[i].numpy().astype("uint8"))
plt.title(dog_breeds[labels[i]])
 plt.axis("off" )
def create_model(input_shape, n_out):
input_tensor = Input(shape=input_shape)
 base_model = Xception(include_top=False,
weights=None,
input_tensor=input_tensor)
 base_model.load_weights('.. /input/xception/xception_weights_tf_dim_ordering_tf_kernels_notop.h5')
 x = GlobalAveragePooling2D()(base_model.output)
 x = Dropout(0.5 )(x)
 x = Dense(1024, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 final_output = Dense(n_out, activation='sigmoid', name='final_output' )(x)
 model = Model(input_tensor, final_output)
 return model<choose_model_class>
plt.figure(figsize=(20, 20))
for images in test_ds.take(1):
for i in range(16):
ax = plt.subplot(4, 4, i + 1)
 plt.imshow(images[i].numpy().astype("uint8"))
plt.axis("off" )
EarlyStopping, ReduceLROnPlateau,CSVLogger)
 epochs = 35; batch_size = 64
checkpoint = ModelCheckpoint('.. /working/Resnet50_focal.h5', monitor='val_loss', verbose=1,
save_best_only=True, mode='min', save_weights_only = True)
 reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=4,
verbose=1, mode='auto', epsilon=0.0001)
 early = EarlyStopping(monitor="val_loss",
mode="min",
patience=9)
 csv_logger = CSVLogger(filename='.. /working/training_log.csv',
separator=',',
append=True)
 indexes = np.arange(train_dataset_info.shape[0])
 train_indexes, valid_indexes = train_test_split(indexes, test_size=0.15, random_state=8)
 train_generator = data_generator.create_train(
 train_dataset_info[train_indexes], batch_size,(SIZE,SIZE,3), augument=True)
 train_generator_warmup = data_generator.create_train(
 train_dataset_info[train_indexes], batch_size,(SIZE,SIZE,3), augument=False)
 validation_generator = data_generator.create_valid(
 train_dataset_info[valid_indexes], batch_size,(SIZE,SIZE,3), augument=False)
 lrsched = OneCycleScheduler(lr_max=1e-4, n_data_points=len(train_indexes),
epochs=1, batch_size=batch_size, verbose=0)
 callbacks_list = [checkpoint, csv_logger, reduceLROnPlat]<choose_model_class>
AUTOTUNE = tf.data.AUTOTUNE
train_ds = train_ds.prefetch(buffer_size=AUTOTUNE)
 val_ds = val_ds.prefetch(buffer_size=AUTOTUNE)
 test_ds = test_ds.prefetch(buffer_size=AUTOTUNE)
model = create_model(
 input_shape=(SIZE,SIZE,3),
n_out=NUM_CLASSES)
 for layer in model.layers:
layer.trainable = False
for i in range(-5,0):
model.layers[i].trainable = True
model.compile(
 loss='binary_crossentropy',
optimizer=Adam(1e-3))
model.fit_generator(
 train_generator_warmup,
steps_per_epoch=np.ceil(float(len(train_indexes)) / float(batch_size)) ,
epochs=2,
max_queue_size=16, workers=WORKERS, use_multiprocessing=True,
verbose=1 )<choose_model_class>
data_augmentation = Sequential(
 [
preprocessing.RandomFlip("horizontal"),
preprocessing.RandomRotation(0.1),
preprocessing.RandomZoom(0.1),
]
)
for layer in model.layers:
layer.trainable = True
model.compile(loss='binary_crossentropy',
optimizer=Adam(lr=0.0003))
checkpoint = ModelCheckpoint('.. /working/Resnet50_focal.h5', monitor='val_loss', verbose=1,
save_best_only=True, mode='min', save_weights_only = True)
 reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=1,
verbose=1, mode='auto', epsilon=0.0001)
 early = EarlyStopping(monitor="val_loss",
mode="min",
patience=9)
 callbacks_list = [checkpoint, csv_logger, reduceLROnPlat]
model.fit_generator(
 train_generator,
steps_per_epoch=np.ceil(float(len(train_indexes)) / float(batch_size)) ,
validation_data=validation_generator,
validation_steps=np.ceil(float(len(valid_indexes)) / float(batch_size)) ,
epochs=(epochs*0.9),
verbose=1,
max_queue_size=16, workers=WORKERS, use_multiprocessing=True,
callbacks=callbacks_list)
 model.compile(loss='binary_crossentropy',
optimizer=SGD(0.0001,0.9))
model.fit_generator(
 train_generator,
steps_per_epoch=np.ceil(float(len(train_indexes)) / float(batch_size)) ,
validation_data=validation_generator,
validation_steps=np.ceil(float(len(valid_indexes)) / float(batch_size)) ,
epochs=(epochs*0.1),
verbose=1,
max_queue_size=16, workers=WORKERS, use_multiprocessing=True,
callbacks=callbacks_list )<load_from_csv>
base_model_1 = xception.Xception(weights='imagenet', include_top=False, input_shape=(IMG_HEIGHT, IMG_WIDTH,3))
base_model_2 = inception_v3.InceptionV3(weights='imagenet', include_top=False, input_shape=(IMG_HEIGHT, IMG_WIDTH,3))
base_model_3 = inception_resnet_v2.InceptionResNetV2(weights='imagenet', include_top=False, input_shape=(IMG_HEIGHT, IMG_WIDTH,3))
base_model_5 = nasnet.NASNetLarge(weights='imagenet', include_top=False, input_shape=(IMG_HEIGHT, IMG_WIDTH,3))
base_model_1.trainable = False
base_model_2.trainable = False
base_model_3.trainable = False
base_model_5.trainable = False
inputs = Input(shape=(IMG_HEIGHT, IMG_WIDTH, 3))
aug_inputs = data_augmentation(inputs)
 x1 = xception.preprocess_input(aug_inputs)
 x1 = base_model_1(x1, training=False)
 x1 = GlobalAveragePooling2D()(x1)
 x2 = inception_v3.preprocess_input(aug_inputs)
 x2 = base_model_2(x2, training=False)
 x2 = GlobalAveragePooling2D()(x2)
 x3 = inception_resnet_v2.preprocess_input(aug_inputs)
 x3 = base_model_3(x3, training=False)
 x3 = GlobalAveragePooling2D()(x3)
 x5 = nasnet.preprocess_input(aug_inputs)
 x5 = base_model_5(x5, training=False)
 x5 = GlobalAveragePooling2D()(x5)
 x = Concatenate()([x1, x2, x3, x5])
 x = Dropout (.7 )(x)
 outputs = Dense(120, activation='softmax' )(x)
 model = Model(inputs, outputs)
 display(model.summary() )
submit = pd.read_csv('.. /input/imet-2019-fgvc6/sample_submission.csv')
 model.load_weights('.. /working/Resnet50_focal.h5')
 predicted = []<categorify>
optimizer = Adam(learning_rate=0.001)
 model.compile(loss="sparse_categorical_crossentropy", metrics=['accuracy'], optimizer=optimizer )
BATCH = 512
fullValGen = data_generator.create_valid(
 train_dataset_info[valid_indexes], BATCH,(SIZE,SIZE,3))
n_val = round(train_dataset_info.shape[0]*0.15)//BATCH
print(n_val)
 lastFullValPred = np.empty(( 0, NUM_CLASSES))
lastFullValLabels = np.empty(( 0, NUM_CLASSES))
for i in tqdm(range(n_val+1)) :
im, lbl = next(fullValGen)
 scores = model.predict(im)
 lastFullValPred = np.append(lastFullValPred, scores, axis=0)
 lastFullValLabels = np.append(lastFullValLabels, lbl, axis=0)
 print(lastFullValPred.shape, lastFullValLabels.shape )<compute_train_metric>
EarlyStop_callback = EarlyStopping(min_delta=0.001, patience=10, restore_best_weights=True)
def my_f2(y_true, y_pred):
assert y_true.shape[0] == y_pred.shape[0]
tp = np.sum(( y_true == 1)&(y_pred == 1))
tn = np.sum(( y_true == 0)&(y_pred == 0))
fp = np.sum(( y_true == 0)&(y_pred == 1))
fn = np.sum(( y_true == 1)&(y_pred == 0))
p = tp /(tp + fp + K.epsilon())
 r = tp /(tp + fn + K.epsilon())
 f2 =(1+beta_f2**2)*p*r /(p*beta_f2**2 + r + 1e-15)
 return f2
def find_best_fixed_threshold(preds, targs, do_plot=True):
score = []
thrs = np.arange(0, 0.5, 0.01)
 for thr in tqdm(thrs):
score.append(my_f2(targs,(preds > thr ).astype(int)))
 score = np.array(score)
 pm = score.argmax()
best_thr, best_score = thrs[pm], score[pm].item()
print(f'thr={best_thr:.3f}', f'F2={best_score:.3f}')
 if do_plot:
plt.plot(thrs, score)
 plt.vlines(x=best_thr, ymin=score.min() , ymax=score.max())
 plt.text(best_thr+0.03, best_score-0.01, f'$F_{2}=${best_score:.3f}', fontsize=14);
plt.show()
return best_thr, best_score<compute_train_metric>
history = model.fit(
 train_ds,
epochs=N_EPOCHS,
validation_data=val_ds,
callbacks=[EarlyStop_callback]
)
best_thr, best_score = find_best_fixed_threshold(lastFullValPred, lastFullValLabels, do_plot=True )<predict_on_test>
wrong_pred_images = np.array([])
 actual_labels = np.array([])
 predicted_labels = np.array([])
 batch = 1
for images, labels in val_ds:
batch_predictions_probs = model.predict_on_batch(images)
 batch_predictions = np.argmax(batch_predictions_probs, axis=1)
 mask =(batch_predictions != labels.numpy())
 print("No of wrong predictions on batch {}: {}".format(batch, mask.sum()))
wrong_pred_indices = np.arange(len(batch_predictions)) [mask]
print(wrong_pred_indices)
 if len(wrong_pred_images)== 0:
wrong_pred_images = images.numpy() [wrong_pred_indices]
actual_labels = labels.numpy() [wrong_pred_indices]
predicted_labels = batch_predictions[wrong_pred_indices]
else:
wrong_pred_images = np.append(wrong_pred_images, images.numpy() [wrong_pred_indices], axis = 0)
 actual_labels = np.append(actual_labels, labels.numpy() [wrong_pred_indices], axis = 0)
 predicted_labels = np.append(predicted_labels, batch_predictions[wrong_pred_indices], axis = 0)
 batch = batch + 1
print(wrong_pred_images.shape)
 print(actual_labels.shape)
 print(predicted_labels.shape )
for i, name in tqdm(enumerate(submit['id'])) :
path = os.path.join('.. /input/imet-2019-fgvc6/test/', name)
 image = data_generator.load_image(path,(SIZE,SIZE,3))
score_predict = model.predict(image[np.newaxis]/255.)
label_predict = np.arange(NUM_CLASSES)[score_predict[0]>=best_thr]
str_predict_label = ' '.join(str(l)for l in label_predict)
 predicted.append(str_predict_label )<save_to_csv>
predictions = model.predict(
 test_ds,
batch_size = BATCH_SIZE,
verbose=1
)
submit['attribute_ids'] = predicted
submit.to_csv('submission.csv', index=False )<load_from_csv>
submission.loc[:, dog_breeds] = predictions
submission.head()
<import_modules><EOS> 
submission.to_csv('submission.csv', index=False)