Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<load_from_csv>
!pip install efficientnet_pytorch torchtoolbox
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<feature_engineering>
%matplotlib inline
X_train = X_train / 255.0
test = test / 255.0<categorify>
warnings.simplefilter('ignore')
 torch.manual_seed(47)
 np.random.seed(47 )
Y_train = to_categorical(Y_train, num_classes = 10 )<define_variables>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )
random_seed = 2<split>
class MelanomaDataset(Dataset):
def __init__(self, df: pd.DataFrame, imfolder: str, train: bool = True, transforms = None, meta_features = None):

 self.df = df
self.imfolder = imfolder
self.transforms = transforms
self.train = train
self.meta_features = meta_features
def __getitem__(self, index):
im_path = os.path.join(self.imfolder, self.df.iloc[index]['image_name'] + '.jpg')
 x = cv2.imread(im_path)
 meta = np.array(self.df.iloc[index][self.meta_features].values, dtype=np.float32)
 if self.transforms:
x = self.transforms(x)
 if self.train:
y = self.df.iloc[index]['target']
return(x, meta), y
else:
return(x, meta)
 def __len__(self):
return len(self.df)
 class Net(nn.Module):
def __init__(self, arch, n_meta_features: int):
super(Net, self ).__init__()
self.arch = arch
if 'ResNet' in str(arch.__class__):
self.arch.fc = nn.Linear(in_features=512, out_features=500, bias=True)
 if 'EfficientNet' in str(arch.__class__):
self.arch._fc = nn.Linear(in_features=1280, out_features=500, bias=True)
 self.meta = nn.Sequential(nn.Linear(n_meta_features, 500),
nn.BatchNorm1d(500),
nn.ReLU() ,
nn.Dropout(p=0.2),
nn.Linear(500, 250),
nn.BatchNorm1d(250),
nn.ReLU() ,
nn.Dropout(p=0.2))
self.ouput = nn.Linear(500 + 250, 1)
 def forward(self, inputs):

 x, meta = inputs
cnn_features = self.arch(x)
 meta_features = self.meta(meta)
 features = torch.cat(( cnn_features, meta_features), dim=1)
 output = self.ouput(features)
 return output
X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, random_state=random_seed )<choose_model_class>
class AdvancedHairAugmentation:

 def __init__(self, hairs: int = 5, hairs_folder: str = ""):
self.hairs = hairs
self.hairs_folder = hairs_folder
def __call__(self, img):

 n_hairs = random.randint(0, self.hairs)
 if not n_hairs:
return img
height, width, _ = img.shape
hair_images = [im for im in os.listdir(self.hairs_folder)if 'png' in im]
for _ in range(n_hairs):
hair = cv2.imread(os.path.join(self.hairs_folder, random.choice(hair_images)))
 hair = cv2.flip(hair, random.choice([-1, 0, 1]))
hair = cv2.rotate(hair, random.choice([0, 1, 2]))
h_height, h_width, _ = hair.shape
roi_ho = random.randint(0, img.shape[0] - hair.shape[0])
 roi_wo = random.randint(0, img.shape[1] - hair.shape[1])
 roi = img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]
img2gray = cv2.cvtColor(hair, cv2.COLOR_BGR2GRAY)
 ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)
 mask_inv = cv2.bitwise_not(mask)
 img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)
 hair_fg = cv2.bitwise_and(hair, hair, mask=mask)
 dst = cv2.add(img_bg, hair_fg)
 img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst
return img
def __repr__(self):
return f'{self.__class__.__name__}(hairs={self.hairs}, hairs_folder="{self.hairs_folder}")'
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))<choose_model_class>
class Microscope:

 def __init__(self, p: float = 0.5):
self.p = p
def __call__(self, img):

 if random.random() < self.p:
circle = cv2.circle(( np.ones(img.shape)* 255 ).astype(np.uint8), 
(img.shape[0]//2, img.shape[1]//2),
random.randint(img.shape[0]//2 - 3, img.shape[0]//2 + 15), 
(0, 0, 0),
-1)
 mask = circle - 255
img = np.multiply(img, mask)
 return img
def __repr__(self):
return f'{self.__class__.__name__}(p={self.p})'
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0 )<choose_model_class>
arch = EfficientNet.from_pretrained('efficientnet-b1')
 skf = GroupKFold(n_splits=5 )
model.compile(optimizer = optimizer , loss = "categorical_crossentropy", metrics=["accuracy"] )<choose_model_class>
train_df = pd.read_csv('/kaggle/input/melanoma-external-malignant-256/train_concat.csv')
 test_df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv' )
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )<define_variables>
concat = pd.concat([train_df['anatom_site_general_challenge'], test_df['anatom_site_general_challenge']], ignore_index=True)
 dummies = pd.get_dummies(concat, dummy_na=True, dtype=np.uint8, prefix='site')
 train_df = pd.concat([train_df, dummies.iloc[:train_df.shape[0]]], axis=1)
 test_df = pd.concat([test_df, dummies.iloc[train_df.shape[0]:].reset_index(drop=True)], axis=1)
 train_df['sex'] = train_df['sex'].map({'male': 1, 'female': 0})
 test_df['sex'] = test_df['sex'].map({'male': 1, 'female': 0})
 train_df['sex'] = train_df['sex'].fillna(-1)
 test_df['sex'] = test_df['sex'].fillna(-1)
 mean_age, std_age = train_df['age_approx'].mean() , train_df['age_approx'].std()
train_df['age_approx'] =(train_df['age_approx'] - mean_age)/(1e-7 + std_age)
 test_df['age_approx'] =(test_df['age_approx'] - mean_age)/(1e-7 + std_age)
 train_df['age_approx'] = train_df['age_approx'].fillna(0)
 test_df['age_approx'] = test_df['age_approx'].fillna(0)
 train_df['patient_id'] = train_df['patient_id'].fillna(0 )
epochs = 29
batch_size = 86<train_model>
meta_features = ['sex', 'age_approx'] + [col for col in train_df.columns if 'site_' in col]
meta_features.remove('anatom_site_general_challenge' )
<init_hyperparams>
test = MelanomaDataset(df=test_df,
imfolder='/kaggle/input/melanoma-external-malignant-256/test/test/',
train=False,
transforms=train_transform,
meta_features=meta_features )
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )<train_model>
epochs = 10
model_path = 'model.pth'
es_patience = 3
TTA = 3
oof = np.zeros(( len(train_df), 1))
preds = torch.zeros(( len(test), 1), dtype=torch.float32, device=device)
 for fold,(train_idx, val_idx)in enumerate(skf.split(X=np.zeros(len(train_df)) , y=train_df['target'], groups=train_df['patient_id'].tolist()), 1):
print('=' * 20, 'Fold', fold, '=' * 20)
 best_val = None
patience = es_patience
arch = EfficientNet.from_pretrained('efficientnet-b1')
 model = Net(arch=arch, n_meta_features=len(meta_features))
model = model.to(device)
 optim = torch.optim.Adam(model.parameters() , lr=0.001)
 scheduler = ReduceLROnPlateau(optimizer=optim, mode='max', patience=1, verbose=True, factor=0.2)
 criterion = nn.BCEWithLogitsLoss()
train = MelanomaDataset(df=train_df.iloc[train_idx].reset_index(drop=True),
imfolder='/kaggle/input/melanoma-external-malignant-256/train/train/',
train=True,
transforms=train_transform,
meta_features=meta_features)
 val = MelanomaDataset(df=train_df.iloc[val_idx].reset_index(drop=True),
imfolder='/kaggle/input/melanoma-external-malignant-256/train/train/',
train=True,
transforms=test_transform,
meta_features=meta_features)
 train_loader = DataLoader(dataset=train, batch_size=64, shuffle=True, num_workers=2)
 val_loader = DataLoader(dataset=val, batch_size=16, shuffle=False, num_workers=2)
 test_loader = DataLoader(dataset=test, batch_size=16, shuffle=False, num_workers=2)
 for epoch in range(epochs):
start_time = time.time()
correct = 0
epoch_loss = 0
model.train()
for x, y in train_loader:
x[0] = torch.tensor(x[0], device=device, dtype=torch.float32)
 x[1] = torch.tensor(x[1], device=device, dtype=torch.float32)
 y = torch.tensor(y, device=device, dtype=torch.float32)
 optim.zero_grad()
z = model(x)
 loss = criterion(z, y.unsqueeze(1))
loss.backward()
optim.step()
pred = torch.round(torch.sigmoid(z))
correct +=(pred.cpu() == y.cpu().unsqueeze(1)).sum().item()
epoch_loss += loss.item()
train_acc = correct / len(train_idx)
 model.eval()
val_preds = torch.zeros(( len(val_idx), 1), dtype=torch.float32, device=device)
 with torch.no_grad() :
for j,(x_val, y_val)in enumerate(val_loader):
x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)
 x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)
 y_val = torch.tensor(y_val, device=device, dtype=torch.float32)
 z_val = model(x_val)
 val_pred = torch.sigmoid(z_val)
 val_preds[j*x_val[0].shape[0]:j*x_val[0].shape[0] + x_val[0].shape[0]] = val_pred
val_acc = accuracy_score(train_df.iloc[val_idx]['target'].values, torch.round(val_preds.cpu()))
val_roc = roc_auc_score(train_df.iloc[val_idx]['target'].values, val_preds.cpu())
 print('Epoch {:03}: | Loss: {:.3f} | Train acc: {:.3f} | Val acc: {:.3f} | Val roc_auc: {:.3f} | Training time: {}'.format(
 epoch + 1,
epoch_loss,
train_acc,
val_acc,
val_roc,
str(datetime.timedelta(seconds=time.time() - start_time)) [:7]))
scheduler.step(val_roc)
 if not best_val:
best_val = val_roc
torch.save(model, model_path)
 continue
if val_roc >= best_val:
best_val = val_roc
patience = es_patience
torch.save(model, model_path)
 else:
patience -= 1
if patience == 0:
print('Early stopping.Best Val roc_auc: {:.3f}'.format(best_val))
break
model = torch.load(model_path)
 model.eval()
val_preds = torch.zeros(( len(val_idx), 1), dtype=torch.float32, device=device)
 with torch.no_grad() :
for j,(x_val, y_val)in enumerate(val_loader):
x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)
 x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)
 y_val = torch.tensor(y_val, device=device, dtype=torch.float32)
 z_val = model(x_val)
 val_pred = torch.sigmoid(z_val)
 val_preds[j*x_val[0].shape[0]:j*x_val[0].shape[0] + x_val[0].shape[0]] = val_pred
oof[val_idx] = val_preds.cpu().numpy()
for _ in range(TTA):
for i, x_test in enumerate(test_loader):
x_test[0] = torch.tensor(x_test[0], device=device, dtype=torch.float32)
 x_test[1] = torch.tensor(x_test[1], device=device, dtype=torch.float32)
 z_test = model(x_test)
 z_test = torch.sigmoid(z_test)
 preds[i*x_test[0].shape[0]:i*x_test[0].shape[0] + x_test[0].shape[0]] += z_test
preds /= TTA
del train, val, train_loader, val_loader, x, y, x_val, y_val
gc.collect()
preds /= skf.n_splits
history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_val,Y_val),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size
, callbacks=[learning_rate_reduction] )<predict_on_test>
print('OOF: {:.3f}'.format(roc_auc_score(train_df['target'], oof)) )
results = model.predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )<save_to_csv>
pd.Series(oof.reshape(-1,)).to_csv('oof.csv', index=False )
<import_modules><EOS> 
sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 sub['target'] = preds.cpu().numpy().reshape(-1,)
 sub.to_csv('submission.csv', index=False )