Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<save_to_csv>
!pip install wtfml==0.0.2
!pip install pretrainedmodels
predictions = model.predict_classes(X_test, verbose=0)
 submissions=pd.DataFrame({"ImageId": list(range(1,len(predictions)+1)) ,
"Label": predictions})
 submissions.to_csv("DR.csv", index=False, header=True)
<define_variables>
!pip install efficientnet_pytorch
batch_size=128
keras.__version__ 
<load_from_csv>
import os
import torch
import albumentations
import numpy as np
import pandas as pd
import torch.nn as nn
from sklearn import metrics
from sklearn import model_selection
from torch.nn import functional as F
from wtfml.utils import EarlyStopping
from wtfml.engine import Engine
from wtfml.data_loaders.image import ClassificationLoader
import efficientnet_pytorch
import pretrainedmodels
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 pred_df = pd.read_csv('.. /input/sample_submission.csv')
<define_variables>
class SEResnext50_32x4d(nn.Module):
def __init__(self, pretrained='imagenet'):
super(SEResnext50_32x4d, self ).__init__()
self.base_model = pretrainedmodels.__dict__[
"se_resnext50_32x4d"
](pretrained=None)
 if pretrained is not None:
self.base_model.load_state_dict(
 torch.load(
 ".. /input/pretrained-model-weights-pytorch/se_resnext50_32x4d-a260b3a4.pth" 
)
)
 self.l0 = nn.Linear(2048, 1)
 def forward(self, image, targets):
batch_size, _, _, _ = image.shape
x = self.base_model.features(image)
 x = F.adaptive_avg_pool2d(x, 1 ).reshape(batch_size, -1)
 out = self.l0(x)
 loss = nn.BCEWithLogitsLoss()(out, targets.view(-1, 1 ).type_as(x))
return out, loss
log_dir = '.. /work'<data_type_conversions>
class EfficientNet(nn.Module):
def __init__(self):
super(EfficientNet, self ).__init__()
self.base_model = efficientnet_pytorch.EfficientNet.from_pretrained(
 'efficientnet-b4' 
)
 self.base_model._fc = nn.Linear(
 in_features=1792,
out_features=1,
bias=True 
)
 def forward(self, image, targets):
out = self.base_model(image)
 loss = nn.BCEWithLogitsLoss()(out, targets.view(-1, 1 ).type_as(out))
return out, loss
train_df_n = train_df.astype('float32')/ 255
test_df_n = test_df.astype('float32')/ 255
train_np_n = train_df_n.values
test_np_n = test_df_n.values
print(train_np_n.shape)
 print(test_np_n.shape )<split>
df = pd.read_csv(".. /input/siim-isic-melanoma-classification/train.csv")
 df["kfold"] = -1
df = df.sample(frac=1 ).reset_index(drop=True)
 y = df.target.values
kf = model_selection.StratifiedKFold(n_splits=5)
 for f,(t_, v_)in enumerate(kf.split(X=df, y=y)) :
df.loc[v_, 'kfold'] = f
df.to_csv("train_folds.csv", index=False )
X_train, X_val, y_train, y_val = train_test_split(train_np_n, train_label_np, test_size=0.25, random_state=42)
 print(X_train.shape)
 print(y_train.shape)
 print(X_val.shape)
 print(y_val.shape )<categorify>
def train(fold):
training_data_path = ".. /input/siic-isic-224x224-images/train/"
df = pd.read_csv("/kaggle/working/train_folds.csv")
 device = "cuda"
epochs = 50
train_bs = 32
valid_bs = 16
df_train = df[df.kfold != fold].reset_index(drop=True)
 df_valid = df[df.kfold == fold].reset_index(drop=True)
 model = EfficientNet()
model.to(device)
 mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225)
 train_aug = albumentations.Compose(
 [
albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True),
albumentations.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=15),
albumentations.Flip(p=0.5)
 ] 
)
 valid_aug = albumentations.Compose(
 [
albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True)
 ] 
)
 train_images = df_train.image_name.values.tolist()
train_images = [os.path.join(training_data_path, i + ".png")for i in train_images]
train_targets = df_train.target.values
valid_images = df_valid.image_name.values.tolist()
valid_images = [os.path.join(training_data_path, i + ".png")for i in valid_images]
valid_targets = df_valid.target.values
train_dataset = ClassificationLoader(
 image_paths=train_images,
targets=train_targets,
resize=None,
augmentations=train_aug, 
)
 train_loader = torch.utils.data.DataLoader(
 train_dataset, batch_size=train_bs, shuffle=True, num_workers=4 
)
 valid_dataset = ClassificationLoader(
 image_paths=valid_images,
targets=valid_targets,
resize=None,
augmentations=valid_aug, 
)
 valid_loader = torch.utils.data.DataLoader(
 valid_dataset, batch_size=valid_bs, shuffle=False, num_workers=4 
)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-4)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
 optimizer,
patience=3,
threshold=0.001,
mode="max" 
)
 es = EarlyStopping(patience=5, mode="max")
 for epoch in range(epochs):
train_loss = Engine.train(train_loader, model, optimizer, device=device)
 predictions, valid_loss = Engine.evaluate(
 valid_loader, model, device=device 
)
 predictions = np.vstack(( predictions)).ravel()
auc = metrics.roc_auc_score(valid_targets, predictions)
 print(f"Epoch = {epoch}, AUC = {auc}")
 scheduler.step(auc)
 es(auc, model, model_path=f"model_fold_{fold}.bin")
 if es.early_stop:
print("Early stopping")
 break
print(y_train.shape)
 print(y_val.shape)
 Y_train = to_categorical(y_train)
 Y_val = to_categorical(y_val)
 print(Y_train.shape)
 print(Y_val.shape )<normalization>
def predict(fold):
test_data_path = ".. /input/siic-isic-224x224-images/test/"
df = pd.read_csv(".. /input/siim-isic-melanoma-classification/test.csv")
 device = "cuda"
model_path=f"model_fold_{fold}.bin"
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225)
 aug = albumentations.Compose(
 [
albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True)
 ] 
)
 images = df.image_name.values.tolist()
images = [os.path.join(test_data_path, i + ".png")for i in images]
targets = np.zeros(len(images))
test_dataset = ClassificationLoader(
 image_paths=images,
targets=targets,
resize=None,
augmentations=aug, 
)
 test_loader = torch.utils.data.DataLoader(
 test_dataset, batch_size=16, shuffle=False, num_workers=4 
)
 model = EfficientNet()
model.load_state_dict(torch.load(model_path))
model.to(device)
 predictions = Engine.predict(test_loader, model, device=device)
 predictions = np.vstack(( predictions)).ravel()
return predictions
def step_decay_for_conv2(epoch):
x = 0.0005
if epoch >= 20: x = 0.0001
if epoch >= 40: x = 0.00005
return x
lr_decay = LearningRateScheduler(step_decay_for_conv2,verbose=0 )<choose_model_class>
train(0)
 train(1)
 train(2)
 train(3)
 train(4 )
def create_model2() :
inputs_mnist = Input(shape=(28,28,1))
inputs = Conv2D(filters=64, kernel_size=(3,3), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs_mnist)
 inputs = Conv2D(filters=128, kernel_size=(3,3), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs)
 inputs = BatchNormalization()(inputs)
 inputs = MaxPooling2D(pool_size=(2,2))(inputs)
 inputs = Conv2D(filters=128, kernel_size=(3,3), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs)
 inputs = Conv2D(filters=128, kernel_size=(3,3), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs)
 inputs = BatchNormalization()(inputs)
 inputs = MaxPooling2D(pool_size=(2,2))(inputs)
 inputs = Conv2D(filters=128, kernel_size=(3,3), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs)
 inputs = Conv2D(filters=128, kernel_size=(3,3), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs)
 inputs = BatchNormalization()(inputs)
 inputs = MaxPooling2D(pool_size=(2,2))(inputs)
 inputs = Conv2D(filters=128, kernel_size=(3,3), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs)
 inputs = Conv2D(filters=128, kernel_size=(3,3), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs)
 inputs = BatchNormalization()(inputs)
 inputs_last = MaxPooling2D(pool_size=(2,2))(inputs)
 inputs2 = Conv2D(filters=64, kernel_size=(5,5), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs_mnist)
 inputs2 = Conv2D(filters=128, kernel_size=(5,5), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs2)
 inputs2 = BatchNormalization()(inputs2)
 inputs2 = MaxPooling2D(pool_size=(2,2))(inputs2)
 inputs2 = Conv2D(filters=128, kernel_size=(5,5), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs2)
 inputs2 = Conv2D(filters=128, kernel_size=(5,5), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs2)
 inputs2 = BatchNormalization()(inputs2)
 inputs2 = MaxPooling2D(pool_size=(2,2))(inputs2)
 inputs2 = Conv2D(filters=128, kernel_size=(5,5), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs2)
 inputs2 = Conv2D(filters=128, kernel_size=(5,5), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs2)
 inputs2 = BatchNormalization()(inputs2)
 inputs2 = MaxPooling2D(pool_size=(2,2))(inputs2)
 inputs2 = Conv2D(filters=128, kernel_size=(5,5), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs2)
 inputs2 = Conv2D(filters=128, kernel_size=(5,5), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs2)
 inputs2 = BatchNormalization()(inputs2)
 inputs2_last = MaxPooling2D(pool_size=(2,2))(inputs2)
 inputs_ad_ls = Flatten()(inputs_last)
 inputs_ad_ls = Dense(units=4096, activation='relu' )(inputs_ad_ls)
 inputs_ad_ls = Dropout(rate=0.5 )(inputs_ad_ls)
 inputs_ad_ls = Dense(units=4096, activation='relu' )(inputs_ad_ls)
 inputs_ad_ls = Dropout(rate=0.5 )(inputs_ad_ls)
 outputs_ad_ls = Dense(units=10, activation='softmax', name='1st_fc' )(inputs_ad_ls)
 inputs2_ad_ls = Flatten()(inputs2_last)
 inputs2_ad_ls = Dense(units=4096, activation='relu' )(inputs2_ad_ls)
 inputs2_ad_ls = Dropout(rate=0.5 )(inputs2_ad_ls)
 inputs2_ad_ls = Dense(units=4096, activation='relu' )(inputs2_ad_ls)
 inputs2_ad_ls = Dropout(rate=0.5 )(inputs2_ad_ls)
 outputs2_ad_ls = Dense(units=10, activation='softmax', name='2nd_fc' )(inputs2_ad_ls)
 inputs3 = keras.layers.concatenate([inputs_last, inputs2_last])
 inputs_2_fc = Flatten()(inputs3)
 inputs_2_fc = Dense(units=8192, activation='relu' )(inputs_2_fc)
 inputs_2_fc = Dropout(rate=0.5 )(inputs_2_fc)
 inputs_2_fc = Dense(units=4096, activation='relu' )(inputs_2_fc)
 inputs_2_fc = Dropout(rate=0.5 )(inputs_2_fc)
 inputs_2_fc = Dense(units=4096, activation='relu' )(inputs_2_fc)
 inputs_2_fc = Dropout(rate=0.5 )(inputs_2_fc)
 outputs = Dense(units=10, activation='softmax', name='last_fc' )(inputs_2_fc)
 model = Model(inputs=[inputs_mnist], outputs=[outputs, outputs_ad_ls, outputs2_ad_ls])
 model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'], loss_weights=[1.0, 0.2, 0.2])
 model.summary()
return model 
<choose_model_class>
p1 = predict(0)
 p2 = predict(1)
 p3 = predict(2)
 p4 = predict(3)
 p5 = predict(4 )
<train_model><EOS> 
predictions =(p1 + p2 + p3 + p4 + p5)/ 5
sample = pd.read_csv(".. /input/siim-isic-melanoma-classification/sample_submission.csv")
 sample.loc[:, "target"] = predictions
sample.to_csv("submission.csv", index=False )