Kaggle competition: Cassava Leaf Disease Classification
n_split=6
skf=StratifiedKFold(n_splits=n_split)
 models=[]
historys=[]
epochs = 200
batch_size = 32
if debug:
epochs=3
sub_model_list=[1,0,2,3,5,2]
for i in range(n_split):
print('reading '+str(i)+' model')
 if i==1:
data_root='baseline-0-760-0-143-6fold-split-1st/'
elif i==2:
data_root='fork-of-baseline-0-760-0-143-6fold-split-2/'
else:
data_root='baseline-0-760-0-143-6fold-split-'+str(i)+'/'
model=load_model('.. /input/'+data_root+str(i)+'_'+str(sub_model_list[i])+'_keras.model',{'bce_dice_loss': bce_dice_loss})
 models.append(model)
<predict_on_test>
%matplotlib inline
pd.set_option('display.max_rows', None)
 pd.set_option('display.max_columns', None )
threshes=[]
for i,(trdex,valdex)in enumerate(skf.split(X=train_df.index.values,y=train_df.coverage_class.values)) :
ids_valid=train_df.index.values[valdex]
y_valid=np.array(train_df.loc[ids_valid].masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1)
 x_valid=np.array(train_df.loc[ids_valid].images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1)
 model = models[i]
preds_valid = model.predict(x_valid ).reshape(-1, img_size_target, img_size_target)
 preds_valid = np.array([downsample(x)for x in preds_valid])
 y_valid = np.array([downsample(x)for x in y_valid])
 thresholds = np.linspace(0, 1, 50)
 ious = np.array([iou_metric_batch(y_valid, np.int32(preds_valid > threshold)) for threshold in tqdm_notebook(thresholds)])
 threshold_best_index = np.argmax(ious[9:-10])+ 9
iou_best = ious[threshold_best_index]
threshold_best = thresholds[threshold_best_index]
threshes.append(threshold_best)
 print(i,threshold_best,iou_best)
 print(thresholds,ious )<categorify>
data_dir = '/kaggle/input/cassava-leaf-disease-classification'
train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
sub = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))
def RLenc(img, order='F', format=True):

 bytes = img.reshape(img.shape[0] * img.shape[1], order=order)
 runs = []
r = 0
pos = 1
for c in bytes:
if(c == 0):
if r != 0:
runs.append(( pos, r))
pos += r
r = 0
pos += 1
else:
r += 1
if r != 0:
runs.append(( pos, r))
pos += r
r = 0
if format:
z = ''
for rr in runs:
z += '{} {} '.format(rr[0], rr[1])
 return z[:-1]
else:
return runs<predict_on_test>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
preds_test=np.zeros([len(x_test),img_size_target, img_size_target],dtype=np.float32)
 avg_thres=0
test_batch_length=1000
for i in range(n_split):
model = models[i]
avg_thres+=threshes[i]
for b in range(int(len(x_test)/test_batch_length)) :
print(str(i)+' split: '+str(b)+' batch')
 x_test_batch=x_test[b*test_batch_length:(b+1)*test_batch_length]
preds_test_batch = model.predict(x_test_batch)
 preds_test[b*test_batch_length:(b+1)*test_batch_length]+=(preds_test_batch ).astype(np.float32 ).squeeze()
preds_test/=n_split
avg_thres/=n_split<compute_test_metric>
train['label'].value_counts().sort_index()
pred_dict = {idx: RLenc(np.round(downsample(preds_test[i])> avg_thres)) for i, idx in enumerate(tqdm_notebook(test_df.index.values)) }<save_to_csv>
class CassavaDataset(Dataset):
def __init__(self, data_dir, transform=None, phase='train', df=None):
self.df = df
self.data_dir = data_dir
self.transform = transform
self.phase = phase
if self.phase == 'test':
img_dir = 'test_images'
else:
img_dir = 'train_images'
self.img_path = glob.glob(os.path.join(self.data_dir, img_dir, '*.jpg'))
def __len__(self):
if self.df is None:
return len(self.img_path)
 else:
return len(self.df)
 def __getitem__(self, idx):
if self.phase == 'test':
target_img_path = self.img_path[idx]
else:
row = self.df.iloc[idx]
target_img_id = row['image_id']
target_img_path = os.path.join(self.data_dir, 'train_images', f'{target_img_id}')
 img = cv2.imread(target_img_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img_id = os.path.basename(target_img_path)
 if self.transform is not None:
img = self.transform(img, self.phase)
 else:
img = torch.from_numpy(img.transpose(( 2, 0, 1)))
 img = img / 255.
 if self.phase == 'test':
return img, img_id
else:
label = self.df[self.df['image_id'] == img_id]['label'].values
label = torch.tensor(label, dtype=torch.long)
 return img, label
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv('submission.csv' )<set_options>
class ImageTransform:
def __init__(self, img_size=224, mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)) :
self.transform = {
'train': albu.Compose([
albu.RandomShadow(p=0.5),
albu.RandomResizedCrop(img_size, img_size, interpolation=cv2.INTER_AREA),
albu.ColorJitter(p=0.5),
albu.CLAHE(p=0.5),
albu.HorizontalFlip(p=0.5),
albu.VerticalFlip(p=0.5),
albu.Transpose(p=0.5),
albu.ShiftScaleRotate(p=0.5),
albu.OneOf([
albu.Blur(p=1.0),
albu.GaussianBlur(p=1.0)
 ], p=0.5),
albu.CoarseDropout(max_height=15, max_width=15, min_holes=3, p=0.5),
albu.Normalize(mean, std),
ToTensorV2() ,
], p=1.0),
'val': albu.Compose([
albu.Resize(img_size, img_size),
albu.Normalize(mean, std),
ToTensorV2() ,
], p=1.0),
'test': albu.Compose([
albu.Resize(img_size, img_size),
albu.HorizontalFlip(p=0.5),
albu.VerticalFlip(p=0.5),
albu.Normalize(mean, std),
ToTensorV2() ,
], p=1.0)
 }
def __call__(self, img, phase='train'):
augmented = self.transform[phase](image=img)
 augmented = augmented['image']
return augmented
plt.style.use('seaborn-white')
 sns.set_style("white")
<normalization>
transform = ImageTransform()
dataset = CassavaDataset(data_dir, transform, phase='train', df=train)
 img, label = dataset.__getitem__(0)
 print(img.size() , label)
 print(img.max())
 print(img.min() )
img_size_ori = 101
img_size_target = 128
def upsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_target, img_size_target), mode='constant', preserve_range=True)
 def downsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_ori, img_size_ori), mode='constant', preserve_range=True)
 def iouMetric(true, pred):
true = K.batch_flatten(true)
 pred = K.batch_flatten(pred)
 pred = K.cast(K.greater(pred, 0.5), K.floatx())
 intersec = true * pred
iou = K.sum(intersec)/(K.sum(true + pred - intersec)+ K.epsilon())
 return iou<compute_test_metric>
dataloader = DataLoader(dataset, batch_size=8)
 imgs, labels = next(iter(dataloader))
print(imgs.size() )
def dice_coef(y_true, y_pred):
y_true_f = K.flatten(y_true)
 y_pred = K.cast(y_pred, 'float32')
 y_pred_f = K.cast(K.greater(K.flatten(y_pred), 0.5), 'float32')
 intersection = y_true_f * y_pred_f
score = 2.* K.sum(intersection)/(K.sum(y_true_f)+ K.sum(y_pred_f))
return score
def dice_loss(y_true, y_pred):
smooth = 1.
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = y_true_f * y_pred_f
score =(2.* K.sum(intersection)+ smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 return 1.- score
def bce_dice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)+ dice_loss(y_true, y_pred)
 def bce_logdice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)- K.log(1.- dice_loss(y_true, y_pred))
def weighted_bce_loss(y_true, y_pred, weight):
epsilon = 1e-7
y_pred = K.clip(y_pred, epsilon, 1.- epsilon)
 logit_y_pred = K.log(y_pred /(1.- y_pred))
loss = weight *(logit_y_pred *(1.- y_true)+
K.log(1.+ K.exp(-K.abs(logit_y_pred)))+ K.maximum(-logit_y_pred, 0.))
 return K.sum(loss)/ K.sum(weight)
 def weighted_dice_loss(y_true, y_pred, weight):
smooth = 1.
 w, m1, m2 = weight, y_true, y_pred
intersection =(m1 * m2)
 score =(2.* K.sum(w * intersection)+ smooth)/(K.sum(w * m1)+ K.sum(w * m2)+ smooth)
 loss = 1.- K.sum(score)
 return loss
def weighted_bce_dice_loss(y_true, y_pred):
y_true = K.cast(y_true, 'float32')
 y_pred = K.cast(y_pred, 'float32')
 averaged_mask = K.pool2d(
 y_true, pool_size=(50, 50), strides=(1, 1), padding='same', pool_mode='avg')
 weight = K.ones_like(averaged_mask)
 w0 = K.sum(weight)
 weight = 5.* K.exp(-5.* K.abs(averaged_mask - 0.5))
w1 = K.sum(weight)
 weight *=(w0 / w1)
 loss = weighted_bce_loss(y_true, y_pred, weight)+ dice_loss(y_true, y_pred)
 return loss<load_from_csv>
transform = ImageTransform()
dataset = CassavaDataset(data_dir, transform, phase='test', df=None)
 img, label = dataset.__getitem__(0)
 print(img.size() , label)
 print(img.max())
 print(img.min() )
train_df = pd.read_csv(".. /input/tgs-salt-identification-challenge/train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(".. /input/tgs-salt-identification-challenge/depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]<feature_engineering>
dataloader = DataLoader(dataset, batch_size=1)
 imgs, labels = next(iter(dataloader))
print(imgs.size() )
train_df["images"] = [np.array(load_img(".. /input/tgs-salt-identification-challenge/train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]<feature_engineering>
class CassavaDataModule(pl.LightningDataModule):
def __init__(self, data_dir, cfg, transform, cv, fold):
super(CassavaDataModule, self ).__init__()
self.data_dir = data_dir
self.cfg = cfg
self.transform = transform
self.cv = cv
self.fold = fold
def prepare_data(self):
self.df = pd.read_csv(os.path.join(self.data_dir, 'train.csv'))
def setup(self, stage=None):
self.df['fold'] = -1
for i,(trn_idx, val_idx)in enumerate(self.cv.split(self.df, self.df['label'])) :
self.df.loc[val_idx, 'fold'] = i
train = self.df[self.df['fold'] != self.fold].reset_index(drop=True)
 val = self.df[self.df['fold'] == self.fold].reset_index(drop=True)
 self.train_dataset = CassavaDataset(self.data_dir, self.transform, phase='train', df=train)
 self.val_dataset = CassavaDataset(self.data_dir, self.transform, phase='val', df=val)
 self.test_dataset = CassavaDataset(self.data_dir, self.transform, phase='test', df=None)
 def train_dataloader(self):
return DataLoader(self.train_dataset,
batch_size=self.cfg.train['batch_size'],
pin_memory=True,
num_workers=4,
shuffle=True)
 def val_dataloader(self):
return DataLoader(self.val_dataset,
batch_size=self.cfg.train['batch_size'],
pin_memory=True,
num_workers=4,
shuffle=False)
 def test_dataloader(self):
batch_size = min(len(self.test_dataset), self.cfg.train['batch_size'])
 return DataLoader(self.test_dataset,
batch_size=batch_size,
pin_memory=True,
num_workers=4,
shuffle=False )
train_df["masks"] = [np.array(load_img(".. /input/tgs-salt-identification-challenge/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]<feature_engineering>
class Timm_model(nn.Module):
def __init__(self, model_name='efficientnet_b0', pretrained=True, out_dim=5):
super(Timm_model, self ).__init__()
self.base = create_model(model_name, pretrained=pretrained)
 if 'efficientnet' in model_name:
self.base.classifier = nn.Linear(in_features=self.base.classifier.in_features, out_features=out_dim)
 elif 'vit' in model_name:
self.base.head = nn.Linear(in_features=self.base.head.in_features, out_features=out_dim)
 else:
self.base.fc = nn.Linear(in_features=self.base.fc.in_features, out_features=out_dim)
 def forward(self, x):
return self.base(x )
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size_ori, 2 )<categorify>
z = torch.randn(4, 3, 224, 224)
 model = Timm_model(pretrained=False)
 out = model(z)
 print(out.size() )
def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class )<split>
class CassavaLightningSystem(pl.LightningModule):
def __init__(self, net, cfg, experiment=None):
super(CassavaLightningSystem, self ).__init__()
self.net = net
self.cfg = cfg
self.experiment = experiment
self.criterion = nn.CrossEntropyLoss()
self.best_loss = 1e+9
self.best_acc = None
self.epoch_num = 0
self.acc_fn = metrics.Accuracy()
self.loss_list = []
self.acc_list = []
def configure_optimizers(self):
self.optimizer = optim.AdamW(self.parameters() , lr=self.cfg.train['lr'], weight_decay=1e-5)
 self.scheduler = StepLR(self.optimizer, step_size=2, gamma=0.5)
 return [self.optimizer], [self.scheduler]
def forward(self, x):
return self.net(x)
 def step(self, batch):
inp, label = batch
out = self.forward(inp)
 loss = self.criterion(out, label.squeeze())
 return loss, label, torch.sigmoid(out)
 def training_step(self, batch, batch_idx):
loss, label, logits = self.step(batch)
 if self.experiment is not None:
logs = {'train/loss': loss.item() }
self.experiment.log_metrics(logs, step=batch_idx)
 return {'loss': loss, 'logits': logits, 'labels': label}
def validation_step(self, batch, batch_idx):
loss, label, logits = self.step(batch)
 if self.experiment is not None:
val_logs = {'val/loss': loss.item() }
self.experiment.log_metrics(val_logs, step=batch_idx)
 return {'val_loss': loss, 'logits': logits, 'labels': label}
def validation_epoch_end(self, outputs):
avg_loss = torch.stack([x['val_loss'] for x in outputs] ).mean()
logits = torch.cat([x['logits'] for x in outputs])
 labels = torch.cat([x['labels'] for x in outputs])
 acc = self.acc_fn(logits, labels.squeeze())
 print(f'Epoch: {self.epoch_num} Loss: {avg_loss.item() :.4f} Acc {acc.item() :.4f}')
 self.loss_list.append(avg_loss.item())
 self.acc_list.append(acc.item())
 if self.experiment is not None:
logs = {'val/epoch_loss': avg_loss, 'val/epoch_acc': acc}
self.experiment.log_metrics(logs, step=self.epoch_num)
 if self.best_loss > avg_loss:
self.best_loss = avg_loss.item()
self.best_acc = acc.item()
expname = self.cfg.exp['exp_name']
filename = f'{expname}_epoch_{self.epoch_num}_loss_{self.best_loss:.3f}_acc_{self.best_acc:.3f}.pth'
torch.save(self.net.state_dict() , filename)
 if self.experiment is not None:
self.experiment.log_model(name=filename, file_or_folder='./'+filename)
 os.remove(filename)
 self.epoch_num += 1
return {'avg_val_loss': avg_loss}
def test_step(self, batch, batch_idx):
inp, img_id = batch
out = self.forward(inp)
 logits = torch.sigmoid(out)
 return {'preds': logits, 'image_id': img_id}
def test_epoch_end(self, outputs):
preds = torch.cat([x['preds'] for x in outputs])
 preds = preds.detach().cpu().numpy()
preds = pd.DataFrame(preds, columns=[f'label_{c}' for c in range(5)])
 img_ids = [x['image_id'] for x in outputs]
img_ids = [list(x)for x in img_ids]
img_ids = list(itertools.chain.from_iterable(img_ids))
self.sub = preds
self.sub.insert(0, 'image_id', img_ids)
 return None
def optimizer_step(self, epoch, batch_idx, optimizer, optimizer_idx,
optimizer_closure, on_tpu, using_native_amp, using_lbfgs):
if self.trainer.global_step < 500:
lr_scale = min(1., float(self.trainer.global_step + 1)/ float(500))
for pg in optimizer.param_groups:
pg['lr'] = lr_scale * self.cfg.train['lr']
optimizer.step(closure=optimizer_closure)
 optimizer.zero_grad()
ids_train, ids_valid, x_train, x_valid, y_train, y_valid, cov_train, cov_test, depth_train, depth_test = train_test_split(
 train_df.index.values,
np.array(train_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
np.array(train_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
train_df.coverage.values,
train_df.z.values,
test_size=0.2, stratify=train_df.coverage_class, random_state=1337 )<define_search_model>
class cfg:
exp = {
'exp_name': 'test'
}
data = {
'img_size': 256,
'n_splits': 5
}
train = {
'batch_size': 64,
'epoch': 10,
'seed': 42,
'lr': 0.005,
'model_name': 'efficientnet_b0'
}
def conv_block(m, dim, acti, bn, res, do=0):
	n = Conv2D(dim, 3, activation=acti, padding='same' )(m)
 	n = BatchNormalization()(n)if bn else n
	n = Dropout(do )(n)if do else n
	n = Conv2D(dim, 3, activation=acti, padding='same' )(n)
 	n = BatchNormalization()(n)if bn else n
	return Concatenate()([m, n])if res else n
def level_block(m, dim, depth, inc, acti, do, bn, mp, up, res):
	if depth > 0:
		n = conv_block(m, dim, acti, bn, res)
 		m = MaxPooling2D()(n)if mp else Conv2D(dim, 3, strides=2, padding='same' )(n)
 		m = level_block(m, int(inc*dim), depth-1, inc, acti, do, bn, mp, up, res)
 		if up:
			m = UpSampling2D()(m)
 			m = Conv2D(dim, 2, activation=acti, padding='same' )(m)
 		else:
			m = Conv2DTranspose(dim, 3, strides=2, activation=acti, padding='same' )(m)
 		n = Concatenate()([n, m])
 		m = conv_block(n, dim, acti, bn, res)
 	else:
		m = conv_block(m, dim, acti, bn, res, do)
 	return m
def UNet(img_shape, out_ch=1, start_ch=64, depth=4, inc_rate=2., activation='relu',
		 dropout=0.5, batchnorm=False, maxpool=True, upconv=True, residual=False):
	i = Input(shape=img_shape)
 	o = level_block(i, start_ch, depth, inc_rate, activation, dropout, batchnorm, maxpool, upconv, residual)
 	o = Conv2D(out_ch, 1, activation='sigmoid' )(o)
 	return Model(inputs=i, outputs=o )<create_dataframe>
data_dir = '/kaggle/input/cassava-leaf-disease-classification'
seed_everything(cfg.train['seed'])
 cv = StratifiedKFold(n_splits=cfg.data['n_splits'], shuffle=True, random_state=cfg.train['seed'])
 transform = ImageTransform(img_size=cfg.data['img_size'] )
model = UNet(( img_size_target,img_size_target,1),start_ch=16,depth=5,batchnorm=True )<choose_model_class>
def main(data_dir, transform, cfg, cv, fold, TTA=5):
net = Timm_model(model_name=cfg.train['model_name'], pretrained=False)
 dm = CassavaDataModule(data_dir, cfg, transform, cv, fold=fold)
 model = CassavaLightningSystem(net, cfg, experiment=None)
 trainer = Trainer(
 logger=False,
max_epochs=cfg.train['epoch'],
gpus=1,
amp_backend='apex',
amp_level='O2', 
)
 trainer.fit(model, datamodule=dm)
 if TTA > 0:
for i in range(TTA):
trainer.test(model, datamodule=dm)
 if i == 0:
res = model.sub
else:
for j in range(5):
res[f'label_{j}'] += model.sub[f'label_{j}']
else:
trainer.test(model, datamodule=dm)
 res = model.sub
res.to_csv(f'submission_fold{fold}.csv', index=False)
 del net, dm
torch.cuda.empty_cache()
return model
sgd = SGD(lr=0.01, decay=1e-4, momentum=0.9, nesterov=True)
 model.compile(loss=weighted_bce_dice_loss, optimizer="adam", metrics=["accuracy",iouMetric] )<define_search_model>
TTA = 3
models = []
for fold in range(cfg.data['n_splits']):
m = main(data_dir, transform, cfg, cv, fold, TTA)
 models.append(m)
 del m
<train_model><EOS> 
sub_paths = glob.glob('submission_fold*')
 for i, path in enumerate(sub_paths):
tmp = pd.read_csv(path)
 if i == 0:
res = tmp
else:
for j in range(5):
res[f'label_{j}'] += tmp[f'label_{j}']
label_cols = [c for c in res.columns if c != 'image_id']
res['label'] = np.argmax(res[label_cols].values, axis=1)
 res = res[['image_id', 'label']]
res.to_csv('submission.csv', index=False )