Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<split>
!pip install -q efficientnet_pytorch
def train_validate_split(digit_df, test_ratio=0.2):
assert isinstance(digit_df, pd.DataFrame)
 digit_num = len(digit_df)
 overall_indices = [num for num in range(digit_num)]
overall_class_num = digits_per_class(digit_df, overall_indices)
 test_class_num = [int(num*test_ratio)for num in overall_class_num]
tmp_test_class_num = [0 for num in range(10)]
shuffle(overall_indices)
 train_indices = []
val_indices = []
for idx in overall_indices:
tmp_label = digit_df.iloc[idx, 0]
if tmp_test_class_num[tmp_label] < test_class_num[tmp_label]:
val_indices.append(idx)
 tmp_test_class_num[tmp_label] += 1
else:
train_indices.append(idx)
 return train_indices, val_indices<split>
import os
import time
import random
import numpy as np
import pandas as pd
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import roc_auc_score
from pathlib import Path
from PIL import Image, ImageEnhance, ImageOps, ImageDraw
from torchvision import transforms, models
from efficientnet_pytorch import EfficientNet
import torch
from torch import optim
from torch.optim.lr_scheduler import ReduceLROnPlateau
from torch import nn
from torch.utils.data import Dataset, DataLoader
from torch.utils.data.sampler import WeightedRandomSampler
from torch import cuda
train_data, val_data = train_validate_split(data_torch.digit_df)
 train_class_num = digits_per_class(data_torch.digit_df, train_data)
 val_class_num = digits_per_class(data_torch.digit_df, val_data)
 for i, num in enumerate(train_class_num, 0):
print("Number of Images for Digit ", i, "- Train: ", num, "Validate: ", val_class_num[i])
 print("Train Images: ", sum(train_class_num), "Validate Images: ", sum(val_class_num))<load_pretrained>
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 class AverageMeter() :
def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
train_sampler = sampler.SubsetRandomSampler(train_data)
 train_dataloader = DataLoader(data_torch,
batch_size=4,
shuffle=False,
sampler=train_sampler,
num_workers=4 )<choose_model_class>
class MelanomaDataset(Dataset):
def __init__(self, base_dir, info, transform=None, phase='train'):
self.base_dir = base_dir
self.info = info
self.transform = transform
self.phase = phase
def __len__(self):
return len(self.info)
 def __getitem__(self, index):
p = Path(self.base_dir, self.info.loc[index, 'image_name'] + '.jpg')
 img = Image.open(p)
 img_transformed = self.transform(img, self.phase)
 return {
'inputs': img_transformed,
'labels': torch.tensor(self.info.loc[index, 'target'], dtype=torch.int64)
 }
class BasicLeNet(nn.Module):

 def __init__(self):
super(BasicLeNet, self ).__init__()
self.features = nn.Sequential(
 nn.Conv2d(1, 6, 5),
nn.ReLU(inplace=True),
nn.MaxPool2d(2),
nn.Conv2d(6, 16, 5),
nn.ReLU(inplace=True),
nn.MaxPool2d(2)
)
 self.classifier = nn.Sequential(
 nn.Linear(16*4*4, 120),
nn.ReLU(inplace=True),
nn.Linear(120, 84),
nn.ReLU(inplace=True),
nn.Linear(84, 10)
)
 def forward(self, x):

 x = self.features(x)
 x = x.view(x.size(0), 16*4*4)
 x = self.classifier(x)
 return x<train_model>
def load_model(model, out_features):
net = None
if model.startswith('efficientnet'):
net = EfficientNet.from_pretrained(model, num_classes=out_features)
 for name, param in net.named_parameters() :
if '_fc' in name:
param.requires_grad = True
else:
param.requires_grad = False
elif model == 'vgg19':
net = models.vgg19(pretrained=True)
 net.classifier[6] = nn.Linear(in_features=net.classifier[6].weight.size(1), out_features=out_features)
 for name, param in net.named_parameters() :
if name in ['classifier.6.weight', 'classifier.6.bias']:
param.requires_grad = True
else:
param.requires_grad = False
elif model == 'resnext':
net = models.resnext101_32x8d(pretrained=True)
 net.fc = nn.Linear(in_features=net.fc.weight.size(1), out_features=out_features)
 for name, param in net.named_parameters() :
if 'fc' in name:
param.requires_grad = True
else:
param.requires_grad = False
elif model == 'densenet':
net = models.densenet161(pretrained=True)
 net.classifier = nn.Linear(in_features=net.classifier.weight.size(1), out_features=out_features)
 for name, param in net.named_parameters() :
if 'classifier' in name:
param.requires_grad = True
else:
param.requires_grad = False
return net
def training(network, criterion, optimizer, epoch_num, test=True):
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print("Start Training with", device, epoch_num, "overall epoch")
 network.to(device)

composed_transform = transforms.Compose([Regularize() , ToTensor() ])
 digit_dataset = DigitDataset('train.csv', '.. /input/', train=True, transform=composed_transform)
 if test:
train_indices, val_indices = train_validate_split(digit_dataset.digit_df)
 train_sampler = sampler.SubsetRandomSampler(train_indices)
 val_sampler = sampler.SubsetRandomSampler(val_indices)
 train_dataloader = DataLoader(
 digit_dataset,
batch_size=32,
shuffle=False,
sampler=train_sampler,
num_workers=4,
pin_memory=True 
)
 val_dataloader = DataLoader(
 digit_dataset,
batch_size=32,
shuffle=False,
sampler=val_sampler,
num_workers=4,
pin_memory=True 
)
 print("Training with validation, ", "Overall Data:", len(train_indices)+len(val_indices))
print("Training Data:", len(train_indices), "Validate Data:", len(val_indices))
else:
train_dataloader = DataLoader(
 digit_dataset,
batch_size=32,
shuffle=True,
num_workers=4,
pin_memory=True 
)
 val_dataloader = None
print("Training all data, ", "Overall Data:", len(digit_dataset))

 batch_num = 0
ita = []
loss_avg = []
val_acc = []
for epoch in range(epoch_num):
running_loss = 0.0
for i, data in enumerate(train_dataloader, 0):
digits, labels = data
digits, labels = digits.to(device), labels.to(device)
 optimizer.zero_grad()
outputs = network(digits)
 loss = criterion(outputs, labels)
 loss.backward()
optimizer.step()
running_loss += loss.item()
batch_num += 1
if test == True and i % 500 == 499:
ita.append(batch_num)
 loss_avg.append(running_loss/500.)
val_acc.append(validating(network, val_dataloader))
running_loss = 0.
 if test:
train_accuracy = validating(network, train_dataloader)
 val_accuracy = validating(network, val_dataloader)
 print('Training accuracy: %.5f' %(train_accuracy))
print('Validation accuracy: %.5f' %(val_accuracy))
return network, ita, loss_avg, val_acc<categorify>
def eval_model(model, dataset, batch_size, criterion, num_workers, device=None):
dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)
 losses = AverageMeter()
preds = []
targets = []
with torch.no_grad() :
for data in dataloader:
inputs = data['inputs'].to(device)
 labels = data['labels'].to(device)
 outputs = model(inputs)
 loss = criterion(outputs, labels)
 losses.update(loss.item() , inputs.size(0))
preds.append(outputs[:, 1])
 targets.append(labels)
 preds = torch.cat(preds, dim=-1 ).cpu().numpy()
targets = torch.cat(targets, dim=-1 ).cpu().numpy()
auc = roc_auc_score(targets, preds)
 return losses.avg, auc
def train_model(
 model_id,
dataset_train,
dataset_valid,
batch_size,
model,
criterion,
optimizer,
scheduler,
num_epochs,
freezed_epochs,
base_dir,
num_workers,
sampler,
device,
early_stopping 
):
model.to(device)
 if sampler != None:
dataloader_train = DataLoader(dataset_train, batch_size=batch_size, sampler=sampler, num_workers=num_workers)
 else:
dataloader_train = DataLoader(dataset_train, batch_size=batch_size, shuffle=True, num_workers=num_workers)
 for epoch in range(1, num_epochs + 1):
losses = AverageMeter()
s_time = time.time()
if epoch == freezed_epochs + 1:
model.load_state_dict(torch.load(f'./state_dict_{model_id}.pt', map_location=device))
for param in model.parameters() :
param.requires_grad = True
for g in optimizer.param_groups:
g['lr'] = 1e-4
model.train()
for data in dataloader_train:
optimizer.zero_grad()
inputs = data['inputs'].to(device)
 labels = data['labels'].to(device)
 outputs = model(inputs)
 loss = criterion(outputs, labels)
 loss.backward()
optimizer.step()
losses.update(loss.item() , inputs.size(0))
model.eval()
loss_valid, auc_valid = eval_model(model, dataset_valid, batch_size, criterion, num_workers, device)
 if epoch == 1 or auc_valid > max_auc:
saved = True
max_auc = auc_valid
torch.save(model.state_dict() , f'./state_dict_{model_id}.pt')
 counter = 0
else:
saved = False
counter += 1
e_time = time.time()
print(f'epoch: {epoch}, loss_train: {losses.avg:.4f}, loss_valid: {loss_valid:.4f}, auc_valid: {auc_valid:.4f}, saved: {saved}, {(e_time - s_time):.4f}sec')
 if epoch < freezed_epochs + 1:
counter = 0
else:
scheduler.step(auc_valid)
 if early_stopping != None:
if counter == early_stopping:
break
def predict(dataset, batch_size, model, device=None):
dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False)
 model.to(device)
 model.eval()
preds = []
with torch.no_grad() :
for data in dataloader:
inputs = data['inputs'].to(device)
 outputs = model(inputs)
 preds.append(outputs)
 preds = torch.cat(preds)
 return preds
def validating(network, loader):
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 correct_num = 0
total_num = 0
for i, data in enumerate(loader, 0):
digits, labels = data
total_num += labels.size(0)
 digits, labels = digits.to(device), labels.to(device)
 outputs = network(digits)
 _, predicted = torch.max(outputs, 1)
 correct_num +=(( predicted == labels ).sum().to("cpu")).item()
accuracy = correct_num / total_num
return accuracy<train_on_grid>
seed_everything(123)
 torch.backends.cudnn.benchmark = True
config = {
'INPUT_DIR' : '.. /input/jpeg-melanoma-256x256',
'MODEL' : 'efficientnet-b0',
'SIZE' : 256,
'BATCH_SIZE' : 128,
'NUM_FOLDS' : 5,
'NUM_EPOCHS' : 20,
'FREEZED_EPOCHS' : 5,
'LEARNING_RATE' : 1e-3,
'EARLY_STOPPING' : 3,
'UNIFORM_AUGMENT': True,
'TTA' : 5,
'NUM_WORKERS' : 4,
'DEVICE' : 'cuda'
}
lenet = BasicLeNet()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(lenet.parameters())
 lenet, batch_ita, loss_list, val_acc_list = training(lenet, criterion, optimizer, 30 )<load_from_csv>
train = pd.read_csv(Path(config['INPUT_DIR'], 'train.csv'))
test = pd.read_csv(Path(config['INPUT_DIR'], 'test.csv'))
sub = pd.read_csv(Path(config['INPUT_DIR'], 'sample_submission.csv'))
transform = ImageTransform(config['SIZE'], config['UNIFORM_AUGMENT'] )
<split><EOS> 
prediction = pd.DataFrame()
skf = StratifiedKFold(n_splits=config['NUM_FOLDS'], shuffle=True)
 for i,(tr_idx, va_idx)in enumerate(skf.split(train, train['target'])) :
tr = train.loc[tr_idx, :]
va = train.loc[va_idx, :]
tr.reset_index(drop=True, inplace=True)
 va.reset_index(drop=True, inplace=True)
 dataset_train = MelanomaDataset(Path(config['INPUT_DIR'], 'train'), tr, transform=transform, phase='train')
 dataset_valid = MelanomaDataset(Path(config['INPUT_DIR'], 'train'), va, transform=transform, phase='valid')
 net = load_model(config['MODEL'], 2)
 criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters() , lr=config['LEARNING_RATE'])
 scheduler = ReduceLROnPlateau(optimizer=optimizer, mode='max', patience=2, factor=0.2)
 class_sample_count = np.array([len(np.where(tr['target'] == t)[0])for t in np.unique(tr['target'])])
 weight = 1./ class_sample_count
samples_weight = np.array([weight[t] for t in tr['target']])
 sampler = WeightedRandomSampler(samples_weight, len(samples_weight))
print(f"---- fold: {i + 1} ------------")
 train_model(
 f"{config['MODEL']}_{i + 1}",
dataset_train,
dataset_valid,
config['BATCH_SIZE'],
net,
criterion,
optimizer,
scheduler,
config['NUM_EPOCHS'],
config['FREEZED_EPOCHS'],
config['INPUT_DIR'],
config['NUM_WORKERS'],
sampler,
config['DEVICE'],
config['EARLY_STOPPING'] 
)
 test['target'] = 0
dataset_test = MelanomaDataset(Path(config['INPUT_DIR'], 'test'), test, transform=transform, phase='test')
 for _ in range(config['TTA']):
pred_test = predict(dataset_test, config['BATCH_SIZE'], net, device=config['DEVICE'])
 pred_test = pd.DataFrame(torch.softmax(pred_test, 1)[:, 1].cpu().numpy())
 prediction = pd.concat([prediction, pred_test], axis=1)
 sub['target'] = prediction.mean(axis=1)
 sub.to_csv('./submission.csv', index=False )