Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<feature_engineering>
import math, os, re, warnings, random, glob
import numpy as np
import pandas as pd
import tensorflow as tf
import tensorflow.keras.layers as L
import tensorflow.keras.backend as K
from tensorflow.keras import Sequential
from kaggle_datasets import KaggleDatasets
new_data['spoken_languages_count']=new_data['spoken_languages'].apply(lambda x:len(x))<groupby>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print(f'Running on TPU {tpu.master() }')
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )
new_data.loc[train.index].groupby('spoken_languages_count' ).revenue.median()<categorify>
BATCH_SIZE = 16 * REPLICAS
HEIGHT = 512
WIDTH = 512
CHANNELS = 3
N_CLASSES = 5
TTA_STEPS = 3
IMAGE_SIZE = [512, 512]
SEED =555
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
AUG_BATCH = BATCH_SIZE
status_du=pd.get_dummies(new_data['status'] )<concatenate>
def data_augment(image, label):
image = tf.image.rot90(image,k=np.random.randint(4))
image = tf.image.random_flip_left_right(image , seed=SEED)
 image= image = tf.image.random_flip_up_down(image, seed=SEED)
 IMG_SIZE=IMAGE_SIZE[0]
image = tf.image.resize_with_crop_or_pad(image, IMG_SIZE + 6, IMG_SIZE + 6)
 image = tf.image.random_crop(image, size=[IMG_SIZE, IMG_SIZE, 3])
 image = tf.image.random_brightness(image, max_delta=0.5)
 image = tf.image.random_saturation(image, 0, 2, seed=SEED)
 image = tf.image.adjust_saturation(image, 3)
 return image, label
new_data=pd.concat([new_data,status_du],axis=1)
<drop_column>
def get_name(file_path):
parts = tf.strings.split(file_path, os.path.sep)
 name = parts[-1]
return name
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def resize_image(image, label):
image = tf.image.resize(image, [HEIGHT, WIDTH])
 image = tf.reshape(image, [HEIGHT, WIDTH, CHANNELS])
 return image, label
def process_path(file_path):
name = get_name(file_path)
 img = tf.io.read_file(file_path)
 img = decode_image(img)
 return img, name
def get_dataset(files_path, shuffled=False, tta=False, extension='jpg'):
dataset = tf.data.Dataset.list_files(f'{files_path}*{extension}', shuffle=shuffled)
 dataset = dataset.map(process_path, num_parallel_calls=AUTO)
 if tta:
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.map(resize_image, num_parallel_calls=AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False, tta=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 if tta:
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.map(resize_image, num_parallel_calls=AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 image_name = example['image_name']
return image, image_name
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n )
new_data=new_data.drop(['status'],axis=1)
 new_data.head()<drop_column>
database_base_path = '/kaggle/input/cassava-leaf-disease-classification/'
submission = pd.read_csv(f'{database_base_path}sample_submission.csv')
 display(submission.head())
 TEST_FILENAMES = tf.io.gfile.glob(f'{database_base_path}test_tfrecords/*.tfrec')
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 print(f'GCS: test: {NUM_TEST_IMAGES}')
new_data=new_data.drop(['original_language'],axis=1 )<count_missing_values>
model_path_list = glob.glob('/kaggle/input/casavaleafclassificationdensenet201e20f3/*.h5')
 model_path_list.sort()
print('Models to predict:')
 print(*model_path_list, sep='
' )
sum(new_data['geres_name'].isna() )<data_type_conversions>
models = []
i = 0
for model_path in model_path_list:
print(model_path)
 K.clear_session()
models.append(keras.models.load_model(model_path))
new_data['geres_name']=new_data['geres_name'].fillna('Unknow' )<feature_engineering>
print(" TTA_STEPS = {} ".format(TTA_STEPS))
if TTA_STEPS > 0:
for step in range(TTA_STEPS):
test_ds = get_test_dataset(ordered=True, tta=True)
 print(f'TTA step {step+1}/{TTA_STEPS}')
 test_images_ds = test_ds.map(lambda image, image_name: image)
 probabilities = np.average([models[i].predict(test_images_ds)for i in range(len(models)) ], axis = 0)
 else:
test_ds = get_test_dataset(ordered=True, tta=True)
 test_images_ds = test_ds.map(lambda image, image_name: image)
 probabilities = np.average([models[i].predict(test_images_ds)for i in range(1)], axis = 0)
 predictions = np.argmax(probabilities, axis=-1)
<groupby><EOS> 
print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='image_id,label', comments='' )