Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<train_model>
warnings.filterwarnings('ignore')
base_lr = 1e-5
fc_lr = 1e-3
lrs = [base_lr, base_lr, fc_lr]
learn.fit(lrs, n_cycle=6, cycle_len=1)
 learn.save('weights_v2' )<prepare_output>
training_folder = '.. /input/cassava-leaf-disease-classification/train_images/'
best_th = 0.38
preds_t,y_t = learn.TTA(is_test=True,n_aug=8)
 preds_t = np.stack(preds_t, axis=-1)
 preds_t = np.exp(preds_t)
 preds_t = preds_t.mean(axis=-1)
 preds_t = np.concatenate([np.zeros(( preds_t.shape[0],1)) +best_th, preds_t],axis=1)
 np.save('preds_dn201.npy', preds_t )<save_to_csv>
samples_df = pd.read_csv(".. /input/cassava-leaf-disease-classification/train.csv")
 samples_df = shuffle(samples_df, random_state=42)
 samples_df["filepath"] = training_folder+samples_df["image_id"]
samples_df.head()
sample_df = pd.read_csv(SAMPLE_SUB)
 sample_list = list(sample_df.Image)
 labels_list = ["new_whale"]+labels_list
pred_list = [[labels_list[i] for i in p.argsort() [-5:][::-1]] for p in preds_t]
pred_dic = dict(( key, value)for(key, value)in zip(learn.data.test_ds.fnames,pred_list))
pred_list_cor = [' '.join(pred_dic[id])for id in sample_list]
df = pd.DataFrame({'Image':sample_list,'Id': pred_list_cor})
 df.to_csv('sub_{}.csv'.format(MODEL_NAME), header=True, index=False)
 df.head()<define_variables>
training_percentage = 0.8
training_item_count = int(len(samples_df)*training_percentage)
 validation_item_count = len(samples_df)-int(len(samples_df)*training_percentage)
 training_df = samples_df[:training_item_count]
validation_df = samples_df[training_item_count:]
sub_files = [
'.. /input/publickernel/sub_766.csv',
'.. /input/publickernel/sub_771.csv'
]
sub_weight = [
0.766**2,
0.771**2
]<save_to_csv>
batch_size = 8
image_size = 512
input_shape =(image_size, image_size, 3)
 dropout_rate = 0.4
classes_to_predict = sorted(training_df.label.unique() )
Hlabel = 'Image'
Htarget = 'Id'
npt = 5
place_weights = {}
for i in range(npt):
place_weights[i] =(1 /(i + 1))
print(place_weights)
 lg = len(sub_files)
 sub = [None]*lg
for i, file in enumerate(sub_files):
print("Reading {}: w={} - {}".format(i, sub_weight[i], file))
reader = csv.DictReader(open(file,"r"))
sub[i] = sorted(reader, key=lambda d: str(d[Hlabel]))
out = open("sub_ns.csv", "w", newline='')
 writer = csv.writer(out)
 writer.writerow([Hlabel,Htarget])
 for p, row in enumerate(sub[0]):
target_weight = {}
for s in range(lg):
row1 = sub[s][p]
for ind, trgt in enumerate(row1[Htarget].split(' ')) :
target_weight[trgt] = target_weight.get(trgt,0)+(place_weights[ind]*sub_weight[s])
 tops_trgt = sorted(target_weight, key=target_weight.get, reverse=True)[:npt]
writer.writerow([row1[Hlabel], " ".join(tops_trgt)])
 out.close()<install_modules>
training_data = tf.data.Dataset.from_tensor_slices(( training_df.filepath.values, training_df.label.values))
validation_data = tf.data.Dataset.from_tensor_slices(( validation_df.filepath.values, validation_df.label.values))
!pip install fastai==0.7.0 --no-deps
!pip install torch==0.4.1 torchvision==0.2.1<define_variables>
def load_image_and_label_from_path(image_path, label):
img = tf.io.read_file(image_path)
 img = tf.image.decode_jpeg(img, channels=3)
 return img, label
AUTOTUNE = tf.data.experimental.AUTOTUNE
training_data = training_data.map(load_image_and_label_from_path, num_parallel_calls=AUTOTUNE)
 validation_data = validation_data.map(load_image_and_label_from_path, num_parallel_calls=AUTOTUNE)
 training_data_batches = training_data.shuffle(buffer_size=1000 ).batch(batch_size ).prefetch(buffer_size=AUTOTUNE)
 validation_data_batches = validation_data.shuffle(buffer_size=1000 ).batch(batch_size ).prefetch(buffer_size=AUTOTUNE )
MODEL_NAME = 'Resnext50'
TRAIN = '.. /input/humpback-whale-identification/train/'
TEST = '.. /input/humpback-whale-identification/test/'
LABELS = '.. /input/humpback-whale-identification/train.csv'
SAMPLE_SUB = '.. /input/humpback-whale-identification/sample_submission.csv'
arch = resnext50
num_workers = 4<feature_engineering>
adapt_data = tf.data.Dataset.from_tensor_slices(training_df.filepath.values)
 def adapt_mode(image_path):
img = tf.io.read_file(image_path)
 img = tf.image.decode_jpeg(img, channels=3)
 img = layers.experimental.preprocessing.Rescaling(1.0 / 255 )(img)
 return img
adapt_data = adapt_data.map(adapt_mode, num_parallel_calls=AUTOTUNE)
 adapt_data_batches = adapt_data.shuffle(buffer_size=1000 ).batch(batch_size ).prefetch(buffer_size=AUTOTUNE )
df = pd.read_csv(LABELS ).set_index('Image')
 new_whale_df = df[df.Id == "new_whale"]
train_df = df[~(df.Id == "new_whale")]
unique_labels = np.unique(train_df.Id.values)
 labels_dict = dict()
labels_list = []
for i in range(len(unique_labels)) :
labels_dict[unique_labels[i]] = i
labels_list.append(unique_labels[i])
 print("Number of classes: {}".format(len(unique_labels)))
 train_df.Id = train_df.Id.apply(lambda x: labels_dict[x])
 train_labels = np.asarray(train_df.Id.values)
 test_names = [f for f in os.listdir(TEST)]<feature_engineering>
data_augmentation_layers = tf.keras.Sequential(
 [
layers.experimental.preprocessing.RandomCrop(height=image_size, width=image_size),
layers.experimental.preprocessing.RandomFlip("horizontal_and_vertical"),
layers.experimental.preprocessing.RandomRotation(0.25),
layers.experimental.preprocessing.RandomZoom(( -0.2, 0)) ,
layers.experimental.preprocessing.RandomContrast(( 0.2,0.2))
]
)
train_df['image_name'] = train_df.index
rs = np.random.RandomState(42)
 perm = rs.permutation(len(train_df))
tr_n = train_df['image_name'].values
val_n = train_df['image_name'].values[perm][:1000]
print('Train/val:', len(tr_n), len(val_n))
print('Train classes', len(train_df.loc[tr_n].Id.unique()))
print('Val classes', len(train_df.loc[val_n].Id.unique()))<load_pretrained>
efficientnet = EfficientNetB3(weights=".. /input/keras-efficientnetb3-no-top-weights/efficientnetb3_notop.h5",
include_top=False,
input_shape=input_shape,
drop_connect_rate=dropout_rate)
 inputs = Input(shape=input_shape)
 augmented = data_augmentation_layers(inputs)
 efficientnet = efficientnet(augmented)
 pooling = layers.GlobalAveragePooling2D()(efficientnet)
 dropout = layers.Dropout(dropout_rate )(pooling)
 outputs = Dense(len(classes_to_predict), activation="softmax" )(dropout)
 model = Model(inputs=inputs, outputs=outputs)
 model.summary()
PICKL = '.. /input/cropimg/bounding-box.pickle'
with open(PICKL, 'rb')as f:
crop_boxs = pickle.load(f )<data_type_conversions>
epochs = 20
pil2tensor = transforms.ToTensor()
class HWIDataset(FilesDataset):
def __init__(self, fnames, path, transform):
self.train_df = train_df
super().__init__(fnames, transform, path)
 def get_x(self, i):
img_cr = make_bbox_image(os.path.join(self.path, self.fnames[i])).convert("RGB")
 img = pil2tensor(img_cr ).numpy().transpose(1, 2, 0)
 try:
img = cv2.resize(img,(self.sz, self.sz))
except Exception as e:
print(self.fnames[i])
 print(str(e))
return img
def get_y(self, i):
if(self.path == TEST): return 0
return self.train_df.loc[self.fnames[i]]['Id']
def get_c(self):
return len(unique_labels)
<categorify>
decay_steps = int(round(len(training_df)/batch_size)) *epochs
cosine_decay = CosineDecay(initial_learning_rate=1e-4, decay_steps=decay_steps, alpha=0.3)
 callbacks = [ModelCheckpoint(filepath='best_model.h5', monitor='val_loss', save_best_only=True)]
model.compile(loss="sparse_categorical_crossentropy", optimizer=tf.keras.optimizers.Adam(cosine_decay), metrics=["accuracy"] )
class RandomLighting(Transform):
def __init__(self, b, c, tfm_y=TfmType.NO):
super().__init__(tfm_y)
 self.b, self.c = b, c
def set_state(self):
self.store.b_rand = rand0(self.b)
 self.store.c_rand = rand0(self.c)
 def do_transform(self, x, is_y):
if is_y and self.tfm_y != TfmType.PIXEL: return x
b = self.store.b_rand
c = self.store.c_rand
c = -1 /(c - 1)if c < 0 else c + 1
x = lighting(x, b, c)
 return x
def get_data(sz, batch_size):

 aug_tfms = [RandomRotateZoom(deg=20, zoom=2, stretch=1),
RandomLighting(0.2, 0.2, tfm_y=TfmType.NO),
RandomBlur(blur_strengths=3,tfm_y=TfmType.NO),
RandomFlip(tfm_y=TfmType.NO)]
tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.NO,
aug_tfms=aug_tfms)
 ds = ImageData.get_ds(HWIDataset,(tr_n[:-(len(tr_n)% batch_size)], TRAIN), 
(val_n, TRAIN), tfms, test=(test_names, TEST))
md = ImageData("./", ds, batch_size, num_workers=num_workers, classes=None)
 return md 
<load_from_zip>
history = model.fit(training_data_batches,
epochs = epochs,
validation_data=validation_data_batches,
callbacks=callbacks )
!wget http://files.fast.ai/models/weights.tgz
!tar -zxvf weights.tgz
!mkdir /opt/conda/lib/python3.6/site-packages/fastai/weights/
!cp weights/resnext_50_32x4d.pth /opt/conda/lib/python3.6/site-packages/fastai/weights/
!rm -rf weights weights.tgz<load_pretrained>
model.load_weights("./best_model.h5" )
image_size = 448
batch_size = 8
md = get_data(image_size, batch_size)
 extra_fc_layers_size = []
learn = ConvLearner.pretrained(arch, md, xtra_fc=extra_fc_layers_size)
 learn.opt_fn = optim.Adam<set_options>
submission_df.to_csv("submission.csv", index=False )