Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<choose_model_class>
%matplotlib inline
model.compile(optimizer=optimizer, loss="categorical_crossentropy", metrics=["accuracy"] )<choose_model_class>
import torch
import torch.nn as nn
import torch.optim as optim
from torch.optim.lr_scheduler import CosineAnnealingLR, CosineAnnealingWarmRestarts
import torch.utils.data as data
from torch.utils.data import DataLoader
import torchvision
from torchvision import transforms
from torch.cuda.amp import autocast, GradScaler
reduce_lr = LearningRateScheduler(lambda x: 1e-3 * 0.9 ** x )<train_model>
!pip install '.. /input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'
decays = [(lambda x: 1e-3 * 0.9 ** x )(x)for x in range(10)]
i=1
for lr in decays:
print("Epoch " + str(i)+" Learning Rate: " + str(lr))
i+=1<choose_model_class>
DEBUG = False
INFERENCE = True
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 SEED = 87
seed_everything(SEED)
 TRAIN = '.. /input/cassava-leaf-disease-classification/train.csv'
TRAIN_IMGS = '.. /input/cassava-leaf-disease-classification/train_images'
TEST = '.. /input/cassava-leaf-disease-classification/sample_submission.csv'
TEST_IMGS = '.. /input/cassava-leaf-disease-classification/test_images'
cfg = {'model':{'arch': 'efficientnet-b0'},
'data':{'img_size': 224},
'optim':{'lr': 1e-3,
'T_0': 5,
'T_mult': 1,
'eta_min': 1e-6},
'train':{'batch_size': 64,
'shuffle': True,
'num_epochs': 5,
'patience': 2,
'use_checkpoint': False,
'checkpoint_path': '.. /input/cassava-checkpoints-effnet'},
'val':{'batch_size': 64,
'shuffle': False},
'test':{'batch_size': 64,
'shuffle': False,
'TTA': True,
'num_TTA': 5,
'checkpoint': '.. /input/cassava-checkpoints-effnet'}}
early_stopping = EarlyStopping(
 min_delta=0.001,
patience=20,
restore_best_weights=True,
)<define_variables>
if INFERENCE:
test_df = pd.DataFrame()
test_df['image_id'] = list(os.listdir(TEST_IMGS + '/'))
test_df.loc[:, 'img_path'] = TEST_IMGS + '/' + test_df.image_id
if DEBUG:
test_df = pd.read_csv(TRAIN ).head(128)
 test_df.loc[:, 'img_path'] = TRAIN_IMGS + '/' + test_df.image_id
else:
train_df = pd.read_csv(TRAIN)
 train_df.loc[:, 'img_path'] = TRAIN_IMGS + '/' + train_df.image_id
if DEBUG:
train_df = train_df.head(200)
 idx_set = [idx for idx in range(len(train_df)) ]
train_idx, val_idx = train_test_split(idx_set, test_size=0.2)
 val_df = train_df.iloc[val_idx].reset_index(drop=True)
 train_df = train_df.iloc[train_idx].reset_index(drop=True)
 val_df['pred'] = np.nan
batch_size = 64
epochs = 50<train_model>
if(DEBUG)&(not INFERENCE):
num_labels = len(train_df.loc[:, 'label'].unique())
 print('train ds', train_df.loc[:, 'label'].value_counts(normalize=True ).values)
 print('val ds', val_df.loc[:, 'label'].value_counts(normalize=True ).values )
history = model.fit_generator(datagen.flow(x_train, y_train, batch_size = batch_size), epochs = epochs,
validation_data =(x_test, y_test), verbose=1,
steps_per_epoch=x_train.shape[0] // batch_size,
callbacks = [reduce_lr] )<import_modules>
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
 class AlbumTransform() :
def __init__(self, mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225), resize=cfg['data']['img_size']):
self.data_transform = {
'train': Compose([
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
ShiftScaleRotate(p=0.5),
HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),
RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=mean, std=std, max_pixel_value=255.0, p=1.0),
CoarseDropout(p=0.5),
Cutout(p=0.5),
RandomResizedCrop(resize, resize),
ToTensorV2(p=1.0),
], p=1.0),
'val': Compose([
CenterCrop(resize, resize),
Normalize(mean=mean, std=std, max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0)
 ], p=1.0),
'test': Compose([
CenterCrop(resize, resize),
Normalize(mean=mean, std=std, max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0)
 ], p=1.0),
'test_tta': Compose([
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
ShiftScaleRotate(p=0.5),
HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),
RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=mean, std=std, max_pixel_value=255.0, p=1.0),
CoarseDropout(p=0.5),
Cutout(p=0.5),
RandomResizedCrop(resize, resize),
ToTensorV2(p=1.0),
], p=1.0)
 }
def __call__(self, img, phase='train'):
return self.data_transform[phase](image=img)['image']
import matplotlib.pyplot as plt<save_to_csv>
class CassavaDataset(data.Dataset):
def __init__(self, df, transform=None, phase='train'):
self.transform = transform
self.phase= phase
self.df = df
def __len__(self):
return len(self.df)
 def __getitem__(self, index):
img_path = self.df.iloc[index].loc['img_path']
img = cv2.imread(img_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img_transformed = self.transform(img, self.phase)
 if(self.phase=='train' or self.phase=='val'):
label = self.df.iloc[index].loc['label']
return img_transformed, label
else:
return img_transformed
pred_digits_test = np.argmax(model.predict(df_test),axis=1)
 image_id_test = []
for i in range(len(pred_digits_test)) :
image_id_test.append(i+1)
 d = {'ImageId':image_id_test,'Label':pred_digits_test}
answer = pd.DataFrame(d)
 answer.to_csv('answer.csv',index=False )<set_options>
transform = AlbumTransform()
if INFERENCE:
if cfg['test']['TTA']:
test_ds = CassavaDataset(test_df, transform=transform, phase='test_tta')
 else:
test_ds = CassavaDataset(test_df, transform=transform, phase='test')
 test_dl = DataLoader(test_ds, batch_size=cfg['test']['batch_size'],
shuffle=cfg['test']['shuffle'], num_workers=os.cpu_count() , pin_memory=True)
 dataloaders = {'test': test_dl}
else:
train_ds = CassavaDataset(train_df, transform=transform, phase='train')
 val_ds = CassavaDataset(val_df, transform=transform, phase='val')
 train_dl = DataLoader(train_ds, batch_size=cfg['train']['batch_size'],
shuffle=cfg['train']['shuffle'], num_workers=os.cpu_count() , pin_memory=True)
 val_dl = DataLoader(val_ds, batch_size=cfg['val']['batch_size'],
shuffle=cfg['val']['shuffle'], num_workers=os.cpu_count() , pin_memory=True)
 dataloaders = {'train': train_dl, 'val': val_dl}
if(DEBUG)&(not INFERENCE):
iterator = iter(train_dl)
 inputs, labels = next(iterator)
 print('input size: ', inputs.size())
 print('label size: ', labels.size() )
%matplotlib inline
%load_ext autoreload
%autoreload 2 
<load_from_csv>
class CassavaNet(nn.Module):
def __init__(self):
super().__init__()
if INFERENCE:
self.model = EfficientNet.from_name(cfg['model']['arch'], in_channels=3)
 else:
self.model = EfficientNet.from_pretrained(cfg['model']['arch'], in_channels=3)
 num_in_features = self.model._fc.in_features
self.model._fc = nn.Linear(num_in_features, 5)
 def forward(self, x):
x = self.model(x)
 return x
train_data = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 test_data = pd.read_csv("/kaggle/input/digit-recognizer/test.csv" )<train_model>
model = CassavaNet()
print("Training Data : ")
 train_data.head(3 ).iloc[:,:17]<feature_engineering>
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(params = model.parameters() , lr=cfg['optim']['lr'] )
train_data_norm = train_data.iloc[:, 1:] / 255.0
test_data_norm = test_data / 255.0<define_variables>
class RunOneEpoch() :
def __init__(self, model, dataloaders, criterion, optimizer, scheduler=None):
self.model = model
self.dataloaders = dataloaders
self.criterion = criterion.to(device)
 self.optimizer = optimizer
self.scheduler = scheduler
def train(self, phase='train'):
batch_loss = 0.0
batch_corrects = 0
epoch_loss = 0.0
epoch_corrects = 0
if phase=='train':
model.train()
else:
model.eval()
model.to(device)
 scaler = GradScaler()
dl = self.dataloaders[phase]
preds_list = np.array([])
 progress_bar = tqdm(dl)
 for i,(inputs, labels)in enumerate(progress_bar):
inputs = inputs.to(device, non_blocking=True)
 labels = labels.to(device, non_blocking=True)
 optimizer.zero_grad()
with torch.set_grad_enabled(phase=='train'):
with autocast() :
outputs = self.model(inputs)
 loss = self.criterion.forward(outputs, labels)
 _, preds = torch.max(outputs, axis=1)
 if phase=='train':
scaler.scale(loss ).backward()
scaler.step(self.optimizer)
 scaler.update()
else:
preds_list = np.append(preds_list, preds.detach().cpu().numpy())
 batch_loss = loss.item() * cfg[phase]['batch_size']
batch_corrects = torch.sum(preds==labels.detach())
 batch_corrects = batch_corrects.detach().cpu().numpy()
batch_acc = batch_corrects / cfg[phase]['batch_size']
num_batch =(i + 1)* cfg[phase]['batch_size']
epoch_loss += batch_loss
epoch_corrects += batch_corrects
train_loss = epoch_loss / num_batch
train_acc = epoch_corrects / num_batch
batch_loss = loss.item()
progress_bar.set_description(f"{phase}　| Batch: loss: {round(batch_loss, 4)} acc: {round(batch_acc, 4)}, Epoch: loss: {round(train_loss, 4)} acc: {round(train_acc, 4)}")
 if phase=='train':
if self.scheduler: self.scheduler.step()
return train_loss, train_acc
else:
return train_loss, train_acc, preds_list
def inference(self, phase='test'):
model.eval()
model.to(device)
 dl = self.dataloaders[phase]
preds_list = np.array([])
 progress_bar = tqdm(dl)
 for i, inputs in enumerate(progress_bar):
inputs = inputs.to(device, non_blocking=True)
 optimizer.zero_grad()
with torch.set_grad_enabled(phase=='train'):
outputs = self.model(inputs)
 _, preds = torch.max(outputs, axis=1)
 preds_list = np.append(preds_list, preds.cpu().numpy())
 return preds_list
def inference_TTA(self, phase='test'):
model.eval()
dl = self.dataloaders[phase]
preds_list = np.array([])
 outputs_list = np.zeros(( len(dl.dataset), 5))
progress_bar = tqdm(dl)
 for tta in range(cfg['test']['num_TTA']):
outputs_tta = np.array([])
 for i, inputs in enumerate(progress_bar):
inputs = inputs.to(device, non_blocking=True)
 with torch.set_grad_enabled(phase=='train'):
outputs = self.model(inputs)
 outputs_tta = np.append(outputs_tta, outputs.detach().cpu().numpy())
 outputs_tta = outputs_tta.reshape(-1, 5)
 outputs_list = outputs_list + outputs_tta
outputs_list = torch.Tensor(outputs_list)
 _, preds_list = torch.max(outputs_list, axis=1)
 return preds_list, outputs_list
num_examples_train = train_data.shape[0]
num_examples_test = test_data.shape[0]
n_h = 32
n_w = 32
n_c = 3<define_variables>
def get_checkpoint(folder, checkpoint_name):
checkpoint_path = os.path.join(folder, checkpoint_name)
 checkpoint = torch.load(checkpoint_path, map_location=device)
 return checkpoint
<categorify><EOS> 
%%time
if INFERENCE:
if cfg['test']['TTA']:
outputs = []
model = CassavaNet().to(device)
 try:
checkpoint = get_checkpoint(cfg['test']['checkpoint'], f'checkpoint.pth')
 model.load_state_dict(checkpoint['model'])
 except:
print('No checkpoint.')
 criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(params = model.parameters() , lr=cfg['optim']['lr'])
 run = RunOneEpoch(model=model, dataloaders=dataloaders,
criterion=criterion, optimizer=optimizer)
 _, outputs = run.inference_TTA()
_, preds = torch.max(outputs, axis=1)
 test_df.loc[:, 'label'] = preds
test_df.loc[:, 'label'] = test_df.loc[:, 'label'].astype('uint8')
 test_df.drop('img_path', inplace=True, axis=1)
 test_df.to_csv('submission.csv', index=False)
 else:
preds = run.inference()
test_df.loc[:, 'label'] = preds
test_df.loc[:, 'label'] = test_df.loc[:, 'label'].astype('uint8')
 test_df.drop('img_path', inplace=True, axis=1)
 test_df.to_csv('submission.csv', index=False)
 else:
if cfg['train']['use_checkpoint']:
checkpoint = get_checkpoint(cfg['train']['checkpoint_path'], f'checkpoint.pth')
 model.load_state_dict(checkpoint['model'])
 optimizer.load_state_dict(checkpoint['optimizer'])
 optimizer.param_groups[0]['lr'] = cfg['optim']['lr']
print('Loaded checkpoint state.')
 scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=cfg['optim']['T_0'],
T_mult=cfg['optim']['T_mult'],
eta_min=cfg['optim']['eta_min'])
 run = RunOneEpoch(model=model, dataloaders=dataloaders,
criterion=criterion, optimizer=optimizer, scheduler=scheduler)
 loss_train = []; acc_train = []
loss_val = []; acc_val = []
patience = cfg['train']['patience']; num_patience = cfg['train']['patience']
preds_oof = []
checkpoints = []
for i in range(cfg['train']['num_epochs']):
print(f'--------- Epoch {i}')
 loss, acc = run.train(phase='train')
 loss_train.append(loss); acc_train.append(acc)
 loss, acc, preds = run.train(phase='val')
 loss_val.append(loss); acc_val.append(acc)
 preds_oof.append(preds)
 save_list = {'epoch': i,
'model': run.model.state_dict() ,
'optimizer': run.optimizer.state_dict()
}
checkpoints.append(save_list)
 if i == 0:pass
else:
if loss > loss_val[-2]:
patience -= 1
if patience == 0:
print(f'Early stopping at epoch {i}.Checkpoint is at epoch {i-num_patience}')
 try:
loss_val_series = pd.Series(loss_val)
 idxmin_loss = loss_val_series.idxmin()
val_df.loc[:, 'pred'] = preds_oof[idxmin_loss]
torch.save(checkpoints[idxmin_loss], f'checkpoint.pth')
 except:
val_df.loc[:, 'pred'] = preds_oof[-1]
torch.save(checkpoints[-1], f'checkpoint.pth')
 break
else:
patience = cfg['train']['patience']
if i==cfg['train']['num_epochs']-1:
print('Reached maximum train epoch.')
 val_df.loc[:, 'pred'] = preds_oof[-1]
torch.save(checkpoints[-1], f'checkpoint.pth')
 fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(16, 6))
ax[0].plot(loss_train, label='train', marker='o')
 ax[1].plot(acc_train, label='train', marker='o')
 ax[0].plot(loss_val, label='val', marker='o')
 ax[1].plot(acc_val, label='val', marker='o')
 ax[0].set_xlabel('Epoch'); ax[1].set_xlabel('Epoch')
 ax[0].set_ylabel('CE Loss'); ax[1].set_ylabel('Accuracy')
 ax[0].set_title(f'Cross Entropy Loss'); ax[1].set_title('Accuracy')
 ax[0].set_xlim(0,); ax[1].set_xlim(0,)
 ax[0].legend() ; ax[1].legend()
plt.show()
acc_idx = val_df.label==val_df.pred
acc = len(val_df.loc[acc_idx])/ len(val_df)
 print(f'OOF Accuracy: {round(acc, 5)}' )