Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<define_variables>
package_path = '.. /input/pytorch-image-models/pytorch-image-models-master'
!pip install fastai==0.7.0 --no-deps
!pip install torch==0.4.1 torchvision==0.2.1
MODEL_NAME = 'Resnet50'
TRAIN = '.. /input/humpback-whale-identification/train/'
TEST = '.. /input/humpback-whale-identification/test/'
LABELS = '.. /input/humpback-whale-identification/train.csv'
SAMPLE_SUB = '.. /input/humpback-whale-identification/sample_submission.csv'
BBOX = '.. /input/generating-whale-bounding-boxes/bounding_boxes.csv'
arch = resnet50
num_workers = 4<feature_engineering>
from glob import glob
from sklearn.model_selection import GroupKFold, StratifiedKFold
import cv2
from skimage import io
import torch
from torch import nn
import os
from datetime import datetime
import time
import random
import cv2
import torchvision
from torchvision import transforms
import pandas as pd
import numpy as np
from tqdm import tqdm
import matplotlib.pyplot as plt
from torch.utils.data import Dataset,DataLoader
from torch.utils.data.sampler import SequentialSampler, RandomSampler
from torch.cuda.amp import autocast, GradScaler
import sklearn
import warnings
import joblib
from sklearn.metrics import roc_auc_score, log_loss
from sklearn import metrics
import warnings
import cv2
import pydicom
import timm
from scipy.ndimage.interpolation import zoom
from sklearn.metrics import log_loss
df = pd.read_csv(LABELS ).set_index('Image')
 new_whale_df = df[df.Id == "new_whale"]
train_df = df[~(df.Id == "new_whale")]
unique_labels = np.unique(train_df.Id.values)
 labels_dict = dict()
labels_list = []
for i in range(len(unique_labels)) :
labels_dict[unique_labels[i]] = i
labels_list.append(unique_labels[i])
 print("Number of classes: {}".format(len(unique_labels)))
 train_df.Id = train_df.Id.apply(lambda x: labels_dict[x])
 train_labels = np.asarray(train_df.Id.values)
 test_names = [f for f in os.listdir(TEST)]<feature_engineering>
CFG = {
'fold_num': 10,
'seed': 719,
'model_arch': 'tf_efficientnet_b3_ns',
'img_size': 512,
'epochs': 32,
'train_bs': 32,
'valid_bs': 32,
'lr': 0.03*1e-4,
'num_workers': 4,
'accum_iter': 2,
'verbose_step': 1,
'device': 'cuda:0',
'tta': 1,
'used_epochs': [6,7,8,9],
'weights': [1,1,1,1],
'PseEpochs':3,
'weight_decay':1e-6,
'T_0': 10,
'min_lr': 1e-7,
}
train_df['image_name'] = train_df.index
bbox_df = pd.read_csv(BBOX ).set_index('Image')
 rs = np.random.RandomState(42)
 perm = rs.permutation(len(train_df))
tr_n = train_df['image_name'].values
val_n = train_df['image_name'].values[perm][:1000]
print('Train/val:', len(tr_n), len(val_n))
print('Train classes', len(train_df.loc[tr_n].Id.unique()))
print('Val classes', len(train_df.loc[val_n].Id.unique()))<categorify>
train = pd.read_csv('.. /input/cassava-leaf-disease-classification/train.csv')
 train
class RandomLighting(Transform):
def __init__(self, b, c, tfm_y=TfmType.NO):
super().__init__(tfm_y)
 self.b, self.c = b, c
def set_state(self):
self.store.b_rand = rand0(self.b)
 self.store.c_rand = rand0(self.c)
 def do_transform(self, x, is_y):
if is_y and self.tfm_y != TfmType.PIXEL: return x
b = self.store.b_rand
c = self.store.c_rand
c = -1 /(c - 1)if c < 0 else c + 1
x = lighting(x, b, c)
 return x
def get_data(sz, batch_size):

 aug_tfms = [RandomRotateZoom(deg=20, zoom=2, stretch=1),
RandomLighting(0.2, 0.2, tfm_y=TfmType.NO),
RandomBlur(blur_strengths=3,tfm_y=TfmType.NO),
RandomFlip(tfm_y=TfmType.NO)]
tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.NO,
aug_tfms=aug_tfms)
 ds = ImageData.get_ds(HWIDataset,(tr_n[:-(len(tr_n)% batch_size)], TRAIN), 
(val_n, TRAIN), tfms, test=(test_names, TEST))
md = ImageData("./", ds, batch_size, num_workers=num_workers, classes=None)
 return md 
<load_pretrained>
train.label.value_counts()
image_size = 384
batch_size = 32
md = get_data(image_size, batch_size)
 extra_fc_layers_size = []
learn = ConvLearner.pretrained(arch, md, xtra_fc=extra_fc_layers_size)
 learn.opt_fn = optim.Adam<init_hyperparams>
submission = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 submission.head()
print('Number of layer groups:', len(learn.get_layer_groups()), '\t(first 2 groups is pretrained backbone)')
 print('This is our extra thin on top of the backbone Resnet50 architecture:')
 learn.get_layer_groups() [2]<train_model>
class CassavaDataset(Dataset):
def __init__(
 self, df, data_root, transforms=None, output_label=True 
):
super().__init__()
self.df = df.reset_index(drop=True ).copy()
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index: int):
if self.output_label:
target = self.df.iloc[index]['label']
path = "{}/{}".format(self.data_root, self.df.iloc[index]['image_id'])
 img = get_img(path)
 if self.transforms:
img = self.transforms(image=img)['image']
if self.output_label == True:
return img, target
else:
return img
base_lr = 1e-4
fc_lr = 1e-3
lrs = [base_lr, base_lr, fc_lr]
learn.fit(lrs=lrs, n_cycle=2, cycle_len=None)
 learn.unfreeze()
learn.fit(lrs, n_cycle=16, cycle_len=None )<define_variables>
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
 HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
 def get_train_transforms() :
return Compose([
RandomResizedCrop(CFG['img_size'], CFG['img_size']),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
ShiftScaleRotate(p=0.5),
HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),
RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
CoarseDropout(p=0.5),
Cutout(p=0.5),
ToTensorV2(p=1.0),
], p=1.)
def get_valid_transforms() :
return Compose([
CenterCrop(CFG['img_size'], CFG['img_size'], p=1.) ,
Resize(CFG['img_size'], CFG['img_size']),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)
def get_inference_transforms() :
return Compose([
RandomResizedCrop(CFG['img_size'], CFG['img_size']),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)
best_th = 0.38<prepare_output>
class CassvaImgClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, n_class)
 def forward(self, x):
x = self.model(x)
 return x
preds_t,y_t = learn.TTA(is_test=True,n_aug=8)
 preds_t = np.stack(preds_t, axis=-1)
 preds_t = np.exp(preds_t)
 preds_t = preds_t.mean(axis=-1)
 preds_t = np.concatenate([np.zeros(( preds_t.shape[0],1)) +best_th, preds_t],axis=1)
 np.save("preds.npy",preds_t )<save_to_csv>
if __name__ == '__main__':
seed_everything(CFG['seed'])
 folds = StratifiedKFold(n_splits=CFG['fold_num'] ).split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
if fold > 0:
break
print('Inference fold {} started'.format(fold))
valid_ = train.loc[val_idx,:].reset_index(drop=True)
 valid_ds = CassavaDataset(valid_, '.. /input/cassava-leaf-disease-classification/train_images/', transforms=get_inference_transforms() , output_label=False)
 test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms() , output_label=False)
 val_loader = torch.utils.data.DataLoader(
 valid_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 tst_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique() ).to(device)
 val_preds = []
tst_preds = []
for i, epoch in enumerate(CFG['used_epochs']):
model.load_state_dict(torch.load('.. /input/fork-pytorch-efficientnet-baseline-train-amp-a/{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch)))
 with torch.no_grad() :
for _ in range(CFG['tta']):
tst_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, tst_loader, device)]
tst_preds = np.mean(tst_preds, axis=0)
 tst_preds = softmax(tst_preds)
 test['image_id'] = glob.glob('.. /input/cassava-leaf-disease-classification/test_images/*.jpg')
 del model
torch.cuda.empty_cache()
sample_df = pd.read_csv(SAMPLE_SUB)
 sample_list = list(sample_df.Image)
 labels_list = ["new_whale"]+labels_list
pred_list = [[labels_list[i] for i in p.argsort() [-5:][::-1]] for p in preds_t]
pred_dic = dict(( key, value)for(key, value)in zip(learn.data.test_ds.fnames,pred_list))
pred_list_cor = [' '.join(pred_dic[id])for id in sample_list]
df = pd.DataFrame({'Image':sample_list,'Id': pred_list_cor})
 df.to_csv('submission_{}.csv'.format(MODEL_NAME), header=True, index=False)
 df.head()<import_modules>
test['label'] = np.argmax(tst_preds, axis=1)
from fastai.conv_learner import *
from fastai.dataset import *
from tqdm import tqdm
import pandas as pd
import numpy as np
import os
from sklearn.model_selection import train_test_split, StratifiedShuffleSplit
import matplotlib.pyplot as plt
import math<define_variables>
def rect_path(path):
return '.. /input/cassava-leaf-disease-classification/train_images/'+path
train['image_id'] = train['image_id'].apply(rect_path )
MODEL_PATH = 'Resnet18_v1'
TRAIN = '.. /input/train/'
TEST = '.. /input/test/'
LABELS = '.. /input/train.csv'
SAMPLE_SUB = '.. /input/sample_submission.csv' 
<choose_model_class>
train = pd.concat([train,test] ).reset_index()
train
arch = resnet34
nw = 4<feature_engineering>
class CassavaPseDataset(Dataset):
def __init__(self, df, data_root,
transforms=None,
output_label=True,
one_hot_label=False,
do_fmix=False,
fmix_params={
'alpha': 1.,
'decay_power': 3.,
'shape':(CFG['img_size'], CFG['img_size']),
'max_soft': True,
'reformulate': False
},
do_cutmix=False,
cutmix_params={
'alpha': 1,
} 
):
super().__init__()
self.df = df.reset_index(drop=True ).copy()
self.transforms = transforms
self.data_root = data_root
self.do_fmix = do_fmix
self.fmix_params = fmix_params
self.do_cutmix = do_cutmix
self.cutmix_params = cutmix_params
self.output_label = output_label
self.one_hot_label = one_hot_label
if output_label == True:
self.labels = self.df['label'].values
if one_hot_label is True:
self.labels = np.eye(self.df['label'].max() +1)[self.labels]
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index: int):
if self.output_label:
target = self.labels[index]
img = get_img(self.df.loc[index]['image_id'])
 if self.transforms:
img = self.transforms(image=img)['image']
if self.do_fmix and np.random.uniform(0., 1., size=1)[0] > 0.5:
with torch.no_grad() :
lam = np.clip(np.random.beta(self.fmix_params['alpha'], self.fmix_params['alpha']),0.6,0.7)
 mask = make_low_freq_image(self.fmix_params['decay_power'], self.fmix_params['shape'])
 mask = binarise_mask(mask, lam, self.fmix_params['shape'], self.fmix_params['max_soft'])
 fmix_ix = np.random.choice(self.df.index, size=1)[0]
fmix_img = get_img("{}/{}".format(self.data_root, self.df.iloc[fmix_ix]['image_id']))
if self.transforms:
fmix_img = self.transforms(image=fmix_img)['image']
mask_torch = torch.from_numpy(mask)
 img = mask_torch*img+(1.-mask_torch)*fmix_img
rate = mask.sum() /CFG['img_size']/CFG['img_size']
target = rate*target +(1.-rate)*self.labels[fmix_ix]
if self.do_cutmix and np.random.uniform(0., 1., size=1)[0] > 0.5:
with torch.no_grad() :
cmix_ix = np.random.choice(self.df.index, size=1)[0]
cmix_img = get_img("{}/{}".format(self.data_root, self.df.iloc[cmix_ix]['image_id']))
if self.transforms:
cmix_img = self.transforms(image=cmix_img)['image']
lam = np.clip(np.random.beta(self.cutmix_params['alpha'], self.cutmix_params['alpha']),0.3,0.4)
 bbx1, bby1, bbx2, bby2 = rand_bbox(( CFG['img_size'], CFG['img_size']), lam)
 img[:, bbx1:bbx2, bby1:bby2] = cmix_img[:, bbx1:bbx2, bby1:bby2]
rate = 1 -(( bbx2 - bbx1)*(bby2 - bby1)/(CFG['img_size'] * CFG['img_size']))
target = rate*target +(1.-rate)*self.labels[cmix_ix]
if self.output_label == True:
return img, target
else:
return img
df = pd.read_csv(LABELS ).set_index('Image')
 new_whale_df = df[df.Id == "new_whale"]
train_df = df[~(df.Id == "new_whale")]
unique_labels = np.unique(train_df.Id.values)
 labels_dict = dict()
labels_list = []
for i in range(len(unique_labels)) :
labels_dict[unique_labels[i]] = i
labels_list.append(unique_labels[i])
 print("Number of classes: {}".format(len(unique_labels)))
 train_names = train_df.index.values
train_df.Id = train_df.Id.apply(lambda x: labels_dict[x])
 train_labels = np.asarray(train_df.Id.values)
 test_names = [f for f in os.listdir(TEST)]<randomize_order>
def prepare_dataloader(df, trn_idx, val_idx, data_root='.. /input/cassava-leaf-disease-classification/train_images/'):
train_ = df.loc[trn_idx,:].reset_index(drop=True)
 valid_ = df.loc[val_idx,:].reset_index(drop=True)
 train_ds = CassavaPseDataset(train_, data_root, transforms=get_train_transforms() , output_label=True, one_hot_label=False, do_fmix=False, do_cutmix=False)
 valid_ds = CassavaPseDataset(valid_, data_root, transforms=get_valid_transforms() , output_label=True)
 train_loader = torch.utils.data.DataLoader(
 train_ds,
batch_size=CFG['train_bs'],
pin_memory=False,
drop_last=False,
shuffle=True,
num_workers=CFG['num_workers'], 
)
 val_loader = torch.utils.data.DataLoader(
 valid_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 return train_loader, val_loader
def train_one_epoch(epoch, model, loss_fn, optimizer, train_loader, device, scheduler=None, schd_batch_update=False):
model.train()
t = time.time()
running_loss = None
pbar = tqdm(enumerate(train_loader), total=len(train_loader))
for step,(imgs, image_labels)in pbar:
imgs = imgs.to(device ).float()
target_ = image_labels.numpy()
image_labels = image_labels.to(device ).long()
with autocast() :
image_preds = model(imgs)
 loss = loss_fn(image_preds, image_labels)
 scaler.scale(loss ).backward()
if running_loss is None:
running_loss = loss.item()
else:
running_loss = running_loss *.99 + loss.item() *.01
if(( step + 1)% CFG['accum_iter'] == 0)or(( step + 1)== len(train_loader)) :
scaler.step(optimizer)
 scaler.update()
optimizer.zero_grad()
if scheduler is not None and schd_batch_update:
scheduler.step()
if(( step + 1)% CFG['verbose_step'] == 0)or(( step + 1)== len(train_loader)) :
description = f'epoch {epoch} loss: {running_loss:.4f}'
pbar.set_description(description)
 if scheduler is not None and not schd_batch_update:
scheduler.step()
def valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False):
model.eval()
t = time.time()
loss_sum = 0
sample_num = 0
image_preds_all = []
image_targets_all = []
pbar = tqdm(enumerate(val_loader), total=len(val_loader))
for step,(imgs, image_labels)in pbar:
imgs = imgs.to(device ).float()
image_labels = image_labels.to(device ).long()
image_preds = model(imgs)
 image_preds_all += [torch.argmax(image_preds, 1 ).detach().cpu().numpy() ]
image_targets_all += [image_labels.detach().cpu().numpy() ]
loss = loss_fn(image_preds, image_labels)
 loss_sum += loss.item() *image_labels.shape[0]
sample_num += image_labels.shape[0]
if(( step + 1)% CFG['verbose_step'] == 0)or(( step + 1)== len(val_loader)) :
description = f'epoch {epoch} loss: {loss_sum/sample_num:.4f}'
pbar.set_description(description)
 image_preds_all = np.concatenate(image_preds_all)
 image_targets_all = np.concatenate(image_targets_all)
 print('validation multi-class accuracy = {:.4f}'.format(( image_preds_all==image_targets_all ).mean()))
if scheduler is not None:
if schd_loss_update:
scheduler.step(loss_sum/sample_num)
 else:
scheduler.step()
dup = []
for idx,row in train_df.iterrows() :
if labels_count[row['Id']] < 5:
dup.extend([idx]*math.ceil(( 5 - labels_count[row['Id']])/labels_count[row['Id']]))
train_names = np.concatenate([train_names, dup])
 train_names = train_names[np.random.RandomState(seed=42 ).permutation(train_names.shape[0])]
len(train_names )<split>
if __name__ == '__main__':
seed_everything(CFG['seed'])
 folds = StratifiedKFold(n_splits=CFG['fold_num'], shuffle=True, random_state=CFG['seed'] ).split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
if fold>0:
break
test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
print('Training with {} started'.format(fold))
print(len(trn_idx), len(val_idx))
train_loader, val_loader = prepare_dataloader(train, trn_idx, val_idx, data_root='.. /input/cassava-leaf-disease-classification/train_images/')
 device = torch.device(CFG['device'])
 tst_preds = []
loss_tr = nn.CrossEntropyLoss().to(device)
 loss_fn = nn.CrossEntropyLoss().to(device)
 for i, epoch in enumerate(CFG['used_epochs']):
model.load_state_dict(torch.load('.. /input/fork-pytorch-efficientnet-baseline-train-amp-a/{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch)))
 model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique() , pretrained=False ).to(device)
 scaler = GradScaler()
optimizer = torch.optim.Adam(model.parameters() , lr=CFG['lr'], weight_decay=CFG['weight_decay'])
 scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=CFG['T_0'], T_mult=1, eta_min=CFG['min_lr'], last_epoch=-1)
 for _ in range(CFG['PseEpochs']):
if len(test)>2:
train_one_epoch(epoch, model, loss_tr, optimizer, train_loader, device, scheduler=scheduler, schd_batch_update=False)
 with torch.no_grad() :
for _ in range(CFG['tta']):
tst_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, tst_loader, device)]
tst_preds = np.mean(tst_preds, axis=0)
 del model, optimizer, train_loader, val_loader, scaler, scheduler
torch.cuda.empty_cache()
sss = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42069)
 for train_idx, val_idx in sss.split(train_names, np.zeros(train_names.shape)) :
tr_n, val_n = train_names[train_idx], train_names[val_idx]
print(len(tr_n), len(val_n))<set_options>
test['label'] = np.argmax(tst_preds, axis=1)
 test.head()
<prepare_x_and_y><EOS> 
test.to_csv('submission.csv', index=False )