Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<choose_model_class>
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py
!python pytorch-xla-env-setup.py --version nightly --apt-packages libomp5 libopenblas-dev
!pip install wtfml
!pip install efficientnet_pytorch
model.compile(optimizer='adam',loss='categorical_crossentropy',metrics=['accuracy'] )<choose_model_class>
warnings.filterwarnings("ignore" )
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )<init_hyperparams>
import torch
import torch.nn as nn
import torchvision.models as models
import torchvision
import cv2
import numpy as np
import pandas as pd
import os
from torch.utils.data import DataLoader,TensorDataset,Dataset
import matplotlib.pyplot as plt
import albumentations
from sklearn import model_selection
from sklearn.metrics import roc_auc_score
from efficientnet_pytorch import EfficientNet
from wtfml.utils import EarlyStopping
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )<train_model>
train_df = pd.read_csv('/kaggle/input/jpeg-melanoma-384x384/train.csv')
 train_df.head()
epochs = 250
batch_size=64
history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_test,Y_test),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size
,callbacks=[learning_rate_reduction] )<predict_on_test>
df = train_df.sample(frac=1 ).reset_index(drop=True)
 df['kfold'] = -1
y = train_df.target.values
kf = model_selection.StratifiedKFold(n_splits=5,shuffle=True)
 idx = kf.get_n_splits(X=df,y=y)
 print(idx)
 for fold,(x,y)in enumerate(kf.split(X=df,y=y)) :
df.loc[y,'kfold'] = fold
predictions = model.predict(test)
 predictions = np.argmax(predictions,axis = 1)
 predictions = pd.Series(predictions,name="Label" )<save_to_csv>
df = pd.read_csv('/kaggle/input/tpu-csv/train_fold_tpu.csv')
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),predictions],axis = 1)
 submission.to_csv("submissions_mnist.csv",index=False)
 print("Your file is saved." )<import_modules>
class CustomDataset(Dataset):
def __init__(self,path,name,target,aug):
super(CustomDataset,self ).__init__()
self.path = path
self.name = name
self.target = target
self.aug = aug
def __len__(self):
return len(self.name)
 def __getitem__(self,index):
im_name = self.name[index]
y = self.target[index]
img_path = os.path.join(self.path,im_name + '.jpg')
 img = cv2.resize(cv2.imread(img_path),dsize=(384,384))
image = self.aug(image=img)
 l = image['image']
image = np.transpose(l,(2, 0, 1)).astype(np.float32)
 return torch.tensor(image,dtype=torch.float),torch.tensor(y)
import pandas as pd
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.optim import lr_scheduler
from torch.autograd import Variable
from torch.utils.data import DataLoader, Dataset
from torchvision import transforms
from torchvision.utils import make_grid
import math, random, numbers
from PIL import Image<define_variables>
class Data_Loader() :
def __init__(self,path,name,target,aug):
self.path = path
self.name = name
self.target = target
self.aug = aug
self.dataset = CustomDataset(self.path,self.name,self.target,self.aug)
 def get(self,batch_size,shuffle,num_workers):
sampler = torch.utils.data.distributed.DistributedSampler(self.dataset,
num_replicas = xm.xrt_world_size() ,
rank = xm.get_ordinal() ,
shuffle = shuffle)
 dataloader = torch.utils.data.DataLoader(self.dataset,
batch_size=batch_size,
shuffle=False,
sampler=sampler,
num_workers=num_workers)
 return dataloader
INPUT_DIR = '.. /input/digit-recognizer'
BATCH_SIZE = 64
N_EPOCHS = 50<load_from_csv>
class EffNet(nn.Module):
def __init__(self,model='b4'):
super(EffNet,self ).__init__()
model_name = 'efficientnet' + model
self.feature = EfficientNet.from_pretrained("efficientnet-b4")
 self.drop = nn.Dropout(0.3)
 self.l0 = nn.Linear(1792,1)
 def forward(self,img):
batch_size = img.shape[0]
x = self.feature.extract_features(img)
 x = nn.functional.adaptive_avg_pool2d(x,1 ).reshape(batch_size,-1)
 x = self.drop(x)
 out = self.l0(x)
 return out
train_df = pd.read_csv(INPUT_DIR + '/train.csv')
 n_train = len(train_df)
 n_pixels = len(train_df.columns)- 1
n_class = len(set(train_df['label']))
print('Number of training samples: {0}'.format(n_train))
print('Number of training pixels: {0}'.format(n_pixels))
print('Number of classes: {0}'.format(n_class))<load_from_csv>
class FocalLoss(nn.Module):
def __init__(self,alpha=1,gamma=2):
super(FocalLoss,self ).__init__()
self.alpha = alpha
self.gamma = gamma
def forward(self,preds,truth):
criterion = nn.BCEWithLogitsLoss()
logits = criterion(preds,truth.unsqueeze(-1 ).type_as(preds))
pt = torch.exp(-logits)
 focal_loss = self.alpha*(1-pt)**self.gamma*logits
return torch.mean(focal_loss )
test_df = pd.read_csv(INPUT_DIR + '/test.csv')
 n_test = len(test_df)
 n_pixels = len(test_df.columns)
 print('Number of train samples: {0}'.format(n_test))
print('Number of test pixels: {0}'.format(n_pixels))<load_from_csv>
def train(model,fold):
batch_t = 128
batch_v = 128
best_score = 0
device = xm.xla_device()
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225)
 train_transpose = albumentations.Compose([
albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True),
albumentations.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=15),
albumentations.Flip(p=0.5)
 ])
 valid_transpose = albumentations.Compose(
 [ albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True)
 ])
 image_path = '/kaggle/input/jpeg-melanoma-384x384/train/'
train_df = df[df.kfold!=fold].reset_index(drop=True)
 valid_df = df[df.kfold==fold].reset_index(drop=True)
 train_im = train_df.image_name.values.tolist()
train_y = train_df.target.values
valid_im = valid_df.image_name.values.tolist()
valid_y = valid_df.target.values
train_dataset = Data_Loader(image_path,train_im,train_y,
train_transpose ).get(batch_size=batch_t,shuffle=True,num_workers=4)
 valid_dataset = Data_Loader(image_path,valid_im,valid_y,
valid_transpose ).get(batch_size=batch_v,shuffle=False,num_workers=4)
 model = model.to(device)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-4)
 schedular = torch.optim.lr_scheduler.ReduceLROnPlateau(
 optimizer,
patience=3,
threshold=0.001,
mode="max" 
)
 es = EarlyStopping(patience=5, mode="max",tpu=True)
 criterion = nn.BCEWithLogitsLoss()
epochs = 15
best_score = 0
for epoch in range(epochs):
model.train()
batch = 0
for train_data,label in train_dataset:
train_data = train_data.to(device)
 label = torch.tensor(label,dtype = torch.float32)
 label = label.to(device)
 optimizer.zero_grad()
out = model(train_data)
 loss = criterion(out,label.unsqueeze(1 ).type_as(out))
batch +=1
del train_data,label
gc.collect()
if batch%100==0 : print("EPOCH {} Loss {} batch {}".format(epoch,loss.item() ,batch))
loss.backward()
xm.optimizer_step(optimizer,barrier=True)
 gc.collect()
model.eval()
preds = []
batch = 0
for valid_data,valid_label in valid_dataset:
valid_data = valid_data.to(device)
 valid_label = torch.tensor(valid_label,dtype = torch.float32)
 valid_label = valid_label.to(device)
 batch +=1
with torch.no_grad() :
out = model(valid_data)
 loss = criterion(out,valid_label.unsqueeze(1 ).type_as(out))
preds.append(out.cpu())
 if batch%50==0 : xm.master_print('Valid Loss {} batch {}'.format(loss.item() ,batch))
del valid_data,valid_label
gc.collect()
gc.collect()
pred=np.vstack(( preds)).ravel()
auc_score = roc_auc_score(valid_y.astype(np.float32),pred)
 print("EPOCH {} AUC Score {}".format(epoch,auc_score))
schedular.step(auc_score)
 es(auc_score, model, model_path=f"model_fold_{fold}.bin")
 if es.early_stop:
print("Early stopping")
 break
gc.collect()
class MNIST_data(Dataset):
def __init__(self, file_path,
transform = transforms.Compose([transforms.ToPILImage() , transforms.ToTensor() ,
transforms.Normalize(mean=(0.5,), std=(0.5,)) ])
):
df = pd.read_csv(file_path)
 if len(df.columns)== n_pixels:
self.X = df.values.reshape(( -1,28,28)).astype(np.uint8)[:,:,:,None]
self.y = None
else:
self.X = df.iloc[:,1:].values.reshape(( -1,28,28)).astype(np.uint8)[:,:,:,None]
self.y = torch.from_numpy(df.iloc[:,0].values)
 self.transform = transform
def __len__(self):
return len(self.X)
 def __getitem__(self, idx):
if self.y is not None:
return self.transform(self.X[idx]), self.y[idx]
else:
return self.transform(self.X[idx] )<normalization>
model = EffNet()
class RandomRotation(object):
def __init__(self, degrees, resample=False, expand=False, center=None):
if isinstance(degrees, numbers.Number):
if degrees < 0:
raise ValueError("If degrees is a single number, it must be positive.")
 self.degrees =(-degrees, degrees)
 else:
if len(degrees)!= 2:
raise ValueError("If degrees is a sequence, it must be of len 2.")
 self.degrees = degrees
self.resample = resample
self.expand = expand
self.center = center
@staticmethod
def get_params(degrees):
angle = np.random.uniform(degrees[0], degrees[1])
 return angle
def __call__(self, img):
def rotate(img, angle, resample=False, expand=False, center=None):
return img.rotate(angle, resample, expand, center)
 angle = self.get_params(self.degrees)
 return rotate(img, angle, self.resample, self.expand, self.center )<normalization>
def _mp_fn(rank, flags):
torch.set_default_tensor_type('torch.FloatTensor')
 a = train(model,0)
 FLAGS={}
xmp.spawn(_mp_fn, args=(FLAGS,), nprocs=8, start_method='fork' )
class RandomShift(object):
def __init__(self, shift):
self.shift = shift
@staticmethod
def get_params(shift):
hshift, vshift = np.random.uniform(-shift, shift, size=2)
 return hshift, vshift
def __call__(self, img):
hshift, vshift = self.get_params(self.shift)
 return img.transform(img.size, Image.AFFINE,(1,0,hshift,0,1,vshift), resample=Image.BICUBIC, fill=1 )<load_from_csv>
train(model,0)
 train(model,1)
 train(model,2)
 train(model,3)
 train(model,4 )
train_dataset = MNIST_data(INPUT_DIR + '/train.csv', transform=transforms.Compose(
 [transforms.ToPILImage() , RandomRotation(degrees=20), RandomShift(3),
transforms.ToTensor() , transforms.Normalize(mean=(0.5,), std=(0.5,)) ]))
test_dataset = MNIST_data(INPUT_DIR + '/test.csv')
 train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE, shuffle=False )<define_search_model>
def image_aug(path,image_name,valid=False):
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225)
 train_transpose = albumentations.Compose([
albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True),
albumentations.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=15),
albumentations.Flip(p=0.5)
 ])
 valid_transpose = albumentations.Compose(
 [ albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True)
 ])
 if valid==True :
im_path = os.path.join(path,image_name + '.jpg')
 img = cv2.resize(cv2.imread(im_path),dsize=(384,384))
aug = valid_transpose(image=img)
 l = aug['image']
else:
im_path = os.path.join(path,image_name + '.jpg')
 img = cv2.resize(cv2.imread(im_path),dsize=(384,384))
aug = train_transpose(image=img)
 l =aug['image']
image = np.transpose(l,(2, 0, 1)).astype(np.float32)
 return torch.tensor(image, dtype=torch.float)
 class Data_Loader(Dataset):
def __init__(self,image_path,im_name,target,valid=False):
self.name = im_name
self.target = target
self.path = image_path
self.valid = valid
def __len__(self):
return(len(self.name))
def __getitem__(self,index):
if self.valid==False:
im = self.name[index]
self.train_y = self.target[index]
im_tensor = image_aug(self.path,im)
 return im_tensor,self.train_y
else:
im = self.name[index]
self.valid_y = self.target[index]
im_tensor = image_aug(self.path,im,valid=True)
 return im_tensor,self.valid_y
def train(fold):
batch_t = 16
batch_v = 16
best_score = 0
device = 'cuda'
image_path = '/kaggle/input/jpeg-melanoma-384x384/train/'
train_df = df[df.kfold!=fold].reset_index(drop=True)
 valid_df = df[df.kfold==fold].reset_index(drop=True)
 train_im = train_df.image_name.values.tolist()
train_y = train_df.target.values
valid_im = valid_df.image_name.values.tolist()
valid_y = valid_df.target.values
train_dataset = Data_Loader(image_path,train_im,train_y)
 train_dataset = DataLoader(train_dataset,batch_t,shuffle=False,num_workers=4)
 valid_dataset = Data_Loader(image_path,valid_im,valid_y,valid=True)
 valid_dataset = DataLoader(valid_dataset,batch_v,shuffle=False,num_workers=4)
 model = EffNet()
model = model.cuda()
optimizer = torch.optim.Adam(model.parameters() , lr=1e-4)
 schedular = torch.optim.lr_scheduler.ReduceLROnPlateau(
 optimizer,
patience=3,
threshold=0.001,
mode="max" 
)
 es = EarlyStopping(patience=5, mode="max")
 criterion = nn.BCEWithLogitsLoss()
epochs = 25
best_score = 0
for epoch in range(epochs):
model.train()
batch = 0
for train_data,label in train_dataset:
train_data = train_data.to(device)
 label = torch.tensor(label,dtype = torch.float32)
 label = label.to(device)
 optimizer.zero_grad()
out = model(train_data)
 loss = criterion(out,label.unsqueeze(1 ).type_as(out))
batch +=1
if batch%200==0 : print("EPOCH {} Loss {} batch {}".format(epoch,loss.item() ,batch))
loss.backward()
optimizer.step()
model.eval()
preds = []
batch = 0
for valid_data,valid_label in valid_dataset:
valid_data = valid_data.to(device)
 valid_label = torch.tensor(valid_label,dtype = torch.float32)
 valid_label = valid_label.to(device)
 batch +=1
with torch.no_grad() :
out = model(valid_data)
 loss = criterion(out,valid_label.unsqueeze(1 ).type_as(out))
preds.append(out.cpu())
 if batch%50==0 : print('Valid Loss {} batch {}'.format(loss.item() ,batch))
pred=np.vstack(( preds)).ravel()
auc_score = roc_auc_score(valid_y.astype(np.float32),pred)
 print("EPOCH {} AUC Score {}".format(epoch,auc_score))
schedular.step(auc_score)
 es(auc_score, model, model_path=f"model_fold_{fold}.bin")
 if es.early_stop:
print("Early stopping")
 break
del train_data,valid_data,label,valid_label
gc.collect()
class Net(nn.Module):
def __init__(self):
super(Net, self ).__init__()
self.features = nn.Sequential(
 nn.Conv2d(1, 32, kernel_size=3, stride=1, padding=1),
nn.BatchNorm2d(32),
nn.ReLU(inplace=True),
nn.Conv2d(32, 32, kernel_size=3, stride=1, padding=1),
nn.BatchNorm2d(32),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=2, stride=2),
nn.Conv2d(32, 64, kernel_size=3, padding=1),
nn.BatchNorm2d(64),
nn.ReLU(inplace=True),
nn.Conv2d(64, 64, kernel_size=3, padding=1),
nn.BatchNorm2d(64),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=2, stride=2)
)
 self.classifier = nn.Sequential(
 nn.Dropout(p = 0.5),
nn.Linear(64 * 7 * 7, 512),
nn.BatchNorm1d(512),
nn.ReLU(inplace=True),
nn.Dropout(p = 0.5),
nn.Linear(512, 512),
nn.BatchNorm1d(512),
nn.ReLU(inplace=True),
nn.Dropout(p = 0.5),
nn.Linear(512, 10), 
)
 for m in self.features.children() :
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2./ n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
 m.bias.data.zero_()
for m in self.classifier.children() :
if isinstance(m, nn.Linear):
nn.init.xavier_uniform_(m.weight)
 elif isinstance(m, nn.BatchNorm1d):
m.weight.data.fill_(1)
 m.bias.data.zero_()
def forward(self, x):
x = self.features(x)
 x = x.view(x.size(0), -1)
 x = self.classifier(x)
 return x<choose_model_class>
train(0)
 train(1)
 train(2)
 train(3)
 train(4 )
model = Net()
optimizer = optim.Adam(model.parameters() , lr=0.003)
 criterion = nn.CrossEntropyLoss()
exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)
 if torch.cuda.is_available() :
model = model.cuda()
criterion = criterion.cuda()<train_on_grid>
def predict(fold):
test_df = pd.read_csv('/kaggle/input/jpeg-melanoma-384x384/test.csv')
 im_path = '/kaggle/input/jpeg-melanoma-384x384/test/'
batch_t = 32
model_path = '/kaggle/input/tpu-model/model_fold_'+str(fold)+'.bin'
test_im = test_df.image_name.values.tolist()
test_y = np.ones(len(test_im))
test_dataset = Data_Loader(im_path,test_im,test_y,valid=False)
 test_dataset = DataLoader(test_dataset,batch_t,shuffle=False,num_workers=4)
 device = 'cuda'
model = EffNet()
model.load_state_dict(torch.load(model_path))
model.to(device)
 model.eval()
preds = []
batch = 0
for test_data,test_label in test_dataset:
test_data = test_data.to(device)
 batch +=1
with torch.no_grad() :
out = model(test_data)
 preds.append(out.cpu())
 if batch%50==0 : print('Batch {}'.format(batch))
pred=np.vstack(( preds)).ravel()
return pred
def train(epoch):
model.train()
optimizer.step()
exp_lr_scheduler.step()
for batch_idx,(data, target)in enumerate(train_loader):
data, target = Variable(data), Variable(target)
 if torch.cuda.is_available() :
data = data.cuda()
target = target.cuda()
optimizer.zero_grad()
output = model(data)
 loss = criterion(output, target)
 loss.backward()
optimizer.step()
if(batch_idx + 1)% 100 == 0:
print('Train Epoch: {} [{}/{}({:.0f}%)]\tLoss: {:.6f}'.format(
 epoch,(batch_idx + 1)* len(data), len(train_loader.dataset), 100.*(batch_idx + 1)/ len(train_loader), loss.item()))<compute_train_metric>
predict_1 = predict(0)
 predict_2 = predict(1)
 predict_3 = predict(2)
 predict_4 = predict(3)
 predict_5 = predict(4 )
def evaluate(data_loader):
model.eval()
loss = 0
correct = 0
with torch.no_grad() :
for data, target in data_loader:
data, target = Variable(data), Variable(target)
 if torch.cuda.is_available() :
data = data.cuda()
target = target.cuda()
output = model(data)
 loss += F.cross_entropy(output, target, reduction='sum' ).item()
pred = output.data.max(1, keepdim=True)[1]
correct += pred.eq(target.data.view_as(pred)).cpu().sum()
loss /= len(data_loader.dataset)
 print('
Average loss: {:.4f}, Accuracy: {}/{}({:.3f}%)
 '.format(
 loss, correct, len(data_loader.dataset), 100.* correct / len(data_loader.dataset)) )<train_model>
prediction =(predict_1+predict_2+predict_3+predict_4+predict_5)/5
submission = pd.read_csv(".. /input/jpeg-melanoma-384x384/sample_submission.csv")
 submission.loc[:,'target'] = prediction
submission.to_csv('submission.csv',index=False )
for epoch in range(N_EPOCHS):
train(epoch)
 evaluate(train_loader )<predict_on_test>
s1 = pd.read_csv('.. /working/submission.csv')
 s2 = pd.read_csv('.. /input/new-submit/submit_fl.csv')
 s3 = pd.read_csv('.. /input/new-submit/submit_bce.csv')
 target_res = s3.target.values
target_eff = s2.target.values
target_eff_1 = s1.target.values
result =(target_res + target_eff + target_eff_1)/3
submission = pd.read_csv(".. /input/jpeg-melanoma-384x384/sample_submission.csv")
 submission.loc[:,'target'] = result
submission.to_csv('submit_1.csv',index=False )
<compute_test_metric><EOS> 
sub = pd.read_csv('.. /working/submission.csv')
 sub.head()