Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<categorify>
!pip install --quiet /kaggle/input/kerasapplications
!pip install --quiet /kaggle/input/efficientnet-git
class RandomLighting(Transform):
def __init__(self, b, c, tfm_y=TfmType.NO):
super().__init__(tfm_y)
 self.b, self.c = b, c
def set_state(self):
self.store.b_rand = rand0(self.b)
 self.store.c_rand = rand0(self.c)
 def do_transform(self, x, is_y):
if is_y and self.tfm_y != TfmType.PIXEL: return x
b = self.store.b_rand
c = self.store.c_rand
c = -1 /(c - 1)if c < 0 else c + 1
x = lighting(x, b, c)
 return x
def get_data(sz, bs, test_names=test_names, test_dir=TEST):
aug_tfms = [RandomRotateZoom(deg=20, zoom=2, stretch=1),
RandomLighting(0.05, 0.05, tfm_y=TfmType.NO),
RandomBlur(blur_strengths=3,tfm_y=TfmType.NO),
RandomFlip(tfm_y=TfmType.NO)]
tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.NO,
aug_tfms=aug_tfms)
 ds = ImageData.get_ds(HWIDataset,(tr_n[:-(len(tr_n)% bs)], TRAIN), 
(val_n, TRAIN), tfms, test=(test_names, test_dir))
md = ImageData("./", ds, bs, num_workers=nw, classes=None)
 return md 
<choose_model_class>
print("Tensorflow version " + tf.__version__ )
batch_size = 64
md = get_data(384, batch_size)
 learn = ConvLearner.pretrained(arch, md)
 learn.opt_fn = optim.Adam<train_model>
strategy = tf.distribute.get_strategy()
AUTOTUNE = tf.data.experimental.AUTOTUNE
GCS_PATH = ".. /input/cassava-leaf-disease-classification"
IMAGE_SIZE = [512, 512]
RESIZE_IMAGE_SIZE = [512, 512]
CLASSES = ['0', '1', '2', '3', '4']
WEIGHTS_PATH = ".. /input/cassava-leaf-disease-resnet-weights/EfficientNetB4-best-08-0.8890.h5"
BATCH_SIZE = 16
learn.fit(lr, 2, cycle_len=3)
 learn.unfreeze()
lrs = np.array([lr/10, lr/20, lr/40])
 learn.fit(lrs, 4, cycle_len=4, use_clr=(20, 16))
learn.save(MODEL_PATH )<prepare_output>
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
seed = 0
seed_everything(seed)
 warnings.filterwarnings('ignore' )
preds_v,y_v = learn.TTA(is_test=False,n_aug=2)
 preds_v = np.stack(preds_v, axis=-1)
 preds_v = np.exp(preds_v)
 preds_v = preds_v.mean(axis=-1)
 y_v += 1<prepare_output>
def decode_image(image):
image = tf.image.decode_jpeg(image, channels=3)
 image = tf.cast(image, tf.float32)
 image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
TEST=TRAIN
total_new_whale = len(new_whale_df.index.values)
 md = get_data(384, batch_size, test_names=new_whale_df.index.values[:int(total_new_whale*0.2)], test_dir=TRAIN)
 learn.set_data(md)
 preds_w,y_w = learn.TTA(is_test=True,n_aug=2)
 preds_w = np.stack(preds_w, axis=-1)
 preds_w = np.exp(preds_w)
 preds_w = preds_w.mean(axis=-1 )<concatenate>
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE)
 return dataset
def data_val_augment(image, label):
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 if not IMAGE_SIZE == RESIZE_IMAGE_SIZE:
image = tf.image.resize(image, RESIZE_IMAGE_SIZE)
 return image, label
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.map(data_val_augment, num_parallel_calls=AUTOTUNE)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTOTUNE)
 return dataset
y = np.concatenate([y_v,y_w])
 preds = np.concatenate([preds_v, preds_w],axis=0)
 preds = np.concatenate([np.zeros(( preds.shape[0],1)) , preds],axis=1 )<find_best_params>
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n )
def map5(X, y):
score = 0
for i in range(X.shape[0]):
pred = X[i].argsort() [-5:][::-1]
for j in range(pred.shape[0]):
if pred[j] == y[i]:
score +=(5 - j)/5
break
return score/X.shape[0]
best_th = 0
best_score = 0
for th in np.arange(0.1, 0.801, 0.01):
preds[:,0] = th
score = map5(preds, y)
 if score > best_score:
best_score = score
best_th = th
print("Threshold = {:.3f}, MAP5 = {:.3f}".format(th,score)) 
<feature_engineering>
def read_tfrecord(example, labeled):
tfrecord_format = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64)
 } if labeled else {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string)
 }
example = tf.io.parse_single_example(example, tfrecord_format)
 image = decode_image(example['image'])
 if labeled:
label = tf.cast(example['target'], tf.int32)
 return image, label
idnum = example['image_name']
return image, idnum
TEST = '.. /input/test/'
md = get_data(384, batch_size, test_names=test_names, test_dir=TEST)
 learn.set_data(md)
 preds_t,y_t = learn.TTA(is_test=True,n_aug=8)
 preds_t = np.stack(preds_t, axis=-1)
 preds_t = np.exp(preds_t)
 preds_t = preds_t.mean(axis=-1)
 preds_t = np.concatenate([np.zeros(( preds_t.shape[0],1)) +best_th, preds_t],axis=1 )<save_to_csv>
test_ds = get_test_dataset(ordered=True)
 print('Computing predictions...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilities = trained_model.predict(test_images_ds)
 predictions = np.argmax(probabilities, axis=-1)
 print(predictions )
<import_modules><EOS> 
print('Generating submission.csv file...')
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='image_id,label', comments='')
 !head submission.csv