Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<feature_engineering>
!pip3 install --no-dependencies.. /input/efficientnetcassava/Keras_Applications-1.0.8-py3-none-any.whl
!pip3 install --no-dependencies.. /input/efficientnetcassava/efficientnet-1.1.1-py3-none-any.whl
h2ws = {}
for p, w in tagged.items() :
if w != new_whale:
h = p2h[p]
if h not in h2ws: h2ws[h] = []
if w not in h2ws[h]: h2ws[h].append(w)
 known = sorted(list(h2ws.keys()))
h2i = {}
for i, h in enumerate(known): h2i[h] = i
fknown = branch_model.predict_generator(FeatureGen(known), max_queue_size=20, workers=10, verbose=0)
 fsubmit = branch_model.predict_generator(FeatureGen(submit), max_queue_size=20, workers=10, verbose=0)
 score = head_model.predict_generator(ScoreGen(fknown, fsubmit), max_queue_size=20, workers=10, verbose=0)
 score = score_reshape(score, fknown, fsubmit)
 prepare_submission(0.99, 'submission.csv' )<install_modules>
import re
import numpy as np
import pandas as pd
import os
import json
!pip install fastai==0.7.0 --no-deps
!pip install torch==0.4.1 torchvision==0.2.1
!pip install torchtext==0.2.3<define_variables>
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras import models
from tensorflow.keras import layers
from tensorflow.keras import losses
from sklearn.model_selection import train_test_split
from efficientnet.keras import EfficientNetB3 as EfficientNet
MODEL_NAME = 'Densenet201'
TRAIN = '.. /input/train/'
TEST = '.. /input/test/'
LABELS = '.. /input/train.csv'
SAMPLE_SUB = '.. /input/sample_submission.csv'
arch = dn201
num_workers = 8<feature_engineering>
AUTO = tf.data.experimental.AUTOTUNE
SIZE = 600
ORIGINAL_WIDTH = 800
ORIGINAL_HEIGHT = 600
CHANNELS = 3
BATCH_SIZE = 32
df = pd.read_csv(LABELS ).set_index('Image')
 new_whale_df = df[df.Id == "new_whale"]
train_df = df[~(df.Id == "new_whale")]
unique_labels = np.unique(train_df.Id.values)
 labels_dict = dict()
labels_list = []
for i in range(len(unique_labels)) :
labels_dict[unique_labels[i]] = i
labels_list.append(unique_labels[i])
 print("Number of classes: {}".format(len(unique_labels)))
 train_df.Id = train_df.Id.apply(lambda x: labels_dict[x])
 train_labels = np.asarray(train_df.Id.values)
 test_names = [f for f in os.listdir(TEST)] 
<feature_engineering>
def decode_image(path):
image = tf.io.read_file(path)
 image = tf.image.decode_jpeg(image, channels=3)
 image =(tf.cast(image, tf.float32)/ 255.0)
 image = tf.image.resize(image, [ORIGINAL_HEIGHT, ORIGINAL_WIDTH])
 image = tf.reshape(image, [ORIGINAL_HEIGHT, ORIGINAL_WIDTH , CHANNELS])
 return image
def normalize(x):
x = tf.image.resize(x, [ORIGINAL_HEIGHT, ORIGINAL_WIDTH])
 x = tf.reshape(x, [ORIGINAL_HEIGHT, ORIGINAL_WIDTH, CHANNELS])
 return x
train_df['image_name'] = train_df.index
rs = np.random.RandomState(42)
 perm = rs.permutation(len(train_df))
tr_n = train_df['image_name'].values
val_n = train_df['image_name'].values[perm][:1000]
print('Train/val:', len(tr_n), len(val_n))
print('Train classes', len(train_df.loc[tr_n].Id.unique()))
print('Val classes', len(train_df.loc[val_n].Id.unique()))<prepare_x_and_y>
load_dir = "/kaggle/input/cassava-leaf-disease-classification"
sub_df = pd.read_csv(load_dir + '/sample_submission.csv')
 sub_df['paths'] = load_dir + "/test_images/" + sub_df.image_id
class HWIDataset(FilesDataset):
def __init__(self, fnames, path, transform):
self.train_df = train_df
super().__init__(fnames, transform, path)
 def get_x(self, i):
img = open_image(os.path.join(self.path, self.fnames[i]))
img = cv2.resize(img,(self.sz, self.sz))
return img
def get_y(self, i):
if(self.path == TEST): return 0
return self.train_df.loc[self.fnames[i]]['Id']
def get_c(self):
return len(unique_labels )<categorify>
def load_dataset(augment=False):
test_dataset =tf.data.Dataset.from_tensor_slices(sub_df.paths.values ).map(decode_image, num_parallel_calls=AUTO)
 if augment:
test_dataset = test_dataset.map(lambda x: data_aug(x), num_parallel_calls=AUTO)
 else:
test_dataset = test_dataset.map(lambda x:normalize(x))
return test_dataset.batch(BATCH_SIZE ).prefetch(AUTO )
class RandomLighting(Transform):
def __init__(self, b, c, tfm_y=TfmType.NO):
super().__init__(tfm_y)
 self.b, self.c = b, c
def set_state(self):
self.store.b_rand = rand0(self.b)
 self.store.c_rand = rand0(self.c)
 def do_transform(self, x, is_y):
if is_y and self.tfm_y != TfmType.PIXEL: return x
b = self.store.b_rand
c = self.store.c_rand
c = -1 /(c - 1)if c < 0 else c + 1
x = lighting(x, b, c)
 return x
def get_data(sz, batch_size):

 aug_tfms = [RandomRotateZoom(deg=20, zoom=2, stretch=1),
RandomLighting(0.2, 0.2, tfm_y=TfmType.NO),
RandomBlur(blur_strengths=3,tfm_y=TfmType.NO),
RandomFlip(tfm_y=TfmType.NO)]
tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.NO,
aug_tfms=aug_tfms)
 ds = ImageData.get_ds(HWIDataset,(tr_n[:-(len(tr_n)% batch_size)], TRAIN), 
(val_n, TRAIN), tfms, test=(test_names, TEST))
md = ImageData("./", ds, batch_size, num_workers=num_workers, classes=None)
 return md<load_pretrained>
def load_model(i):
inputs = layers.Input(shape=(ORIGINAL_HEIGHT, ORIGINAL_WIDTH, 3))
model = Sequential([
EfficientNet(include_top=False,weights=None, input_tensor=inputs),
layers.GlobalAveragePooling2D(name="avg_pool"),
layers.BatchNormalization() ,
layers.Dropout(0.3, name="top_dropout"),
layers.Dense(5, activation="softmax", name="pred")
 ])
 model.load_weights(F".. /input/efficientnetcassava/EfficientNetB3_tl_best_weights_{i}.h5")
 model.compile(loss=losses.SparseCategoricalCrossentropy() , optimizer=tf.optimizers.Adam(lr=0.001), metrics=['accuracy'])
 return model
image_size = 224
batch_size = 48
md = get_data(image_size, batch_size)
 extra_fc_layers_size = []
learn = ConvLearner.pretrained(arch, md, xtra_fc=extra_fc_layers_size)
 learn.opt_fn = optim.Adam<init_hyperparams>
n_models = 5
models = []
for i in range(n_models):
models.append(load_model(i))
print('Number of layer groups:', len(learn.get_layer_groups()), '\t(first 2 groups is pretrained backbone)')
 print('This is our extra thin on top of the backbone Resnet50 architecture:')
 learn.get_layer_groups() [2]<train_model>
preds = []
test_dataset = load_dataset()
for i in range(n_models):
preds.append(models[i].predict(test_dataset, verbose=1))
for i in range(10):
test_dataset_augmented = load_dataset(augment=True)
 for i in range(n_models):
preds.append(models[i].predict(test_dataset_augmented, verbose=1))
preds = np.mean(preds, axis=0)
 preds
<define_variables><EOS> 
sub_df['label'] = preds.argmax(axis=1)
 sub_df.drop(columns='paths' ).to_csv('submission.csv', index=False)
 !head submission.csv