Kaggle competition: Quick, Draw! Doodle Recognition Challenge
<SOS> metric: MAP@{K} Kaggle data source: quick,-draw!-doodle-recognition-challenge<save_to_csv>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import cv2
import keras
import math
df_final_season.to_csv('Submission.csv', index=False )<import_modules>
test = pd.read_csv('.. /input/quickdraw-doodle-recognition/test_simplified.csv')
 print(test.shape)
 test.head()
import math, os, re, warnings, random, glob
import numpy as np
import pandas as pd
import tensorflow as tf
import tensorflow.keras.layers as L
import tensorflow.keras.backend as K
from tensorflow.keras import Sequential
from kaggle_datasets import KaggleDatasets 
<set_options>
def img_to_np(img_str, ht, wt, lw, pad):
strokes = eval(img_str)
 ht_ = ht - 2*pad
wt_ = wt - 2*pad
img = np.zeros(( ht, wt), np.uint8)
 for s in strokes:
sx =(np.array(s[0])* wt_ / 256 ).round().astype('int')+ pad
sy =(np.array(s[1])* ht_ / 256 ).round().astype('int')+ pad
for i in range(len(sx)- 1):
p1 =(sx[i], sy[i])
 p2 =(sx[i+1], sy[i+1])
 img = cv2.line(img, p1, p2,(255, 0, 0), lw, lineType=cv2.LINE_AA)
 return img
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print(f'Running on TPU {tpu.master() }')
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<define_variables>
test_imgs = np.zeros(shape =(test.shape[0], 64, 64, 1))
BATCH_SIZE = 16 * REPLICAS
HEIGHT = 512
WIDTH = 512
CHANNELS = 3
N_CLASSES = 5
TTA_STEPS = 3
IMAGE_SIZE = [512, 512]
SEED =555
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
AUG_BATCH = BATCH_SIZE<normalization>
%%time
for i, row in test.iterrows() :
test_imgs[i,:,:,0] = img_to_np(row.drawing, 64, 64, 1, 2)/ 255
def data_augment(image, label):
image = tf.image.rot90(image,k=np.random.randint(4))
image = tf.image.random_flip_left_right(image , seed=SEED)
 image= image = tf.image.random_flip_up_down(image, seed=SEED)
 IMG_SIZE=IMAGE_SIZE[0]
image = tf.image.resize_with_crop_or_pad(image, IMG_SIZE + 6, IMG_SIZE + 6)
 image = tf.image.random_crop(image, size=[IMG_SIZE, IMG_SIZE, 3])
 image = tf.image.random_brightness(image, max_delta=0.5)
 image = tf.image.random_saturation(image, 0, 2, seed=SEED)
 image = tf.image.adjust_saturation(image, 3)
 return image, label 
<normalization>
%%time
probs = cnn.predict(test_imgs)
 print(probs.shape )
def get_name(file_path):
parts = tf.strings.split(file_path, os.path.sep)
 name = parts[-1]
return name
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def resize_image(image, label):
image = tf.image.resize(image, [HEIGHT, WIDTH])
 image = tf.reshape(image, [HEIGHT, WIDTH, CHANNELS])
 return image, label
def process_path(file_path):
name = get_name(file_path)
 img = tf.io.read_file(file_path)
 img = decode_image(img)
 return img, name
def get_dataset(files_path, shuffled=False, tta=False, extension='jpg'):
dataset = tf.data.Dataset.list_files(f'{files_path}*{extension}', shuffle=shuffled)
 dataset = dataset.map(process_path, num_parallel_calls=AUTO)
 if tta:
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.map(resize_image, num_parallel_calls=AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False, tta=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 if tta:
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.map(resize_image, num_parallel_calls=AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 image_name = example['image_name']
return image, image_name
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n )<load_from_csv>
N_train = probs.shape[0]
top_3_probs = np.zeros(shape=(N_train, 3))
for i in range(N_train):
p = probs[i, :]
top_classes = np.argpartition(p, -3)[-3:]
top_classes = top_classes[np.argsort(p[top_classes])]
top_classes = np.flip(top_classes)
 top_probs = p[top_classes]
top_3_probs[i,:] = top_probs
print(top_3_probs[:10, :].round(2))
print(top_3_probs.shape)
 plt.figure(figsize=[10,6])
 for i in range(3):
plt.subplot(3,1,i+1)
 plt.hist(top_3_probs[:,i], color='orchid', edgecolor='k', bins = np.arange(0, 1.01, 0.025))
plt.yscale('log')
 plt.show()
database_base_path = '/kaggle/input/cassava-leaf-disease-classification/'
submission = pd.read_csv(f'{database_base_path}sample_submission.csv')
 display(submission.head())
 TEST_FILENAMES = tf.io.gfile.glob(f'{database_base_path}test_tfrecords/*.tfrec')
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 print(f'GCS: test: {NUM_TEST_IMAGES}')
<find_best_params>
N_train = probs.shape[0]
predictions = []
t = 0.35
for i in range(N_train):
p = probs[i, :]
top_classes = np.argpartition(p, -3)[-3:]
top_classes = top_classes[np.argsort(p[top_classes])]
top_classes = np.flip(top_classes)
 top_probs = p[top_classes]
sel = top_probs > t
sel[0] = True
predictions.append(top_classes[sel])
 print(len(predictions))
model_path_list = glob.glob('/kaggle/input/casavaleafclassificationdensenet201e20f3/*.h5')
 model_path_list.sort()
print('Models to predict:')
 print(*model_path_list, sep='
' )<load_pretrained>
submission = pd.read_csv('.. /input/quickdraw-doodle-recognition/sample_submission.csv')
 submission.head()
models = []
i = 0
for model_path in model_path_list:
print(model_path)
 K.clear_session()
models.append(keras.models.load_model(model_path)) 
<predict_on_test>
label_lookup_df = pd.read_csv('.. /input/models-and-submissions/Quick Draw Models/label_lookup.csv')
 label_lookup = {k:v for k,v in zip(label_lookup_df.index.values, label_lookup_df.label.values)}
label_lookup[0]
print(" TTA_STEPS = {} ".format(TTA_STEPS))
if TTA_STEPS > 0:
for step in range(TTA_STEPS):
test_ds = get_test_dataset(ordered=True, tta=True)
 print(f'TTA step {step+1}/{TTA_STEPS}')
 test_images_ds = test_ds.map(lambda image, image_name: image)
 probabilities = np.average([models[i].predict(test_images_ds)for i in range(len(models)) ], axis = 0)
 else:
test_ds = get_test_dataset(ordered=True, tta=True)
 test_images_ds = test_ds.map(lambda image, image_name: image)
 probabilities = np.average([models[i].predict(test_images_ds)for i in range(1)], axis = 0)
 predictions = np.argmax(probabilities, axis=-1)
<save_to_csv>
%%time
for i in range(N_train):
classes = predictions[i]
words_list = [label_lookup[c] for c in classes]
words_string = ' '.join(words_list)
 submission.loc[i, 'word'] = words_string
submission.head()
print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='image_id,label', comments='' )<define_variables>
submission.to_csv('submission.csv', index=False )
<import_modules><EOS> 
submission.to_csv('submission.csv', index=False )