Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<define_search_space>
%matplotlib inline
sns.set()
py.init_notebook_mode(connected=True)
 warnings.filterwarnings("ignore" )
threshold = [0.001, 0.0025, 0.005, 0.01, 0.025 ,0.05, 0.1]<compute_test_metric>
rcParams['figure.figsize'] = 12,8
print("Maximum accuracy score is :", np.max(np.array(scores_sfm)) )<compute_test_metric>
base = ".. /input/siim-isic-melanoma-classification/"
models = ".. /input/pytorch-pretrained-image-models/"
imagestats = ".. /input/siimisic-melanoma-classification-image-stats/"
print("Optimal threshold :", threshold[np.argmax(np.array(scores_sfm)) ] )<define_search_space>
train_data = pd.read_csv(base + "train.csv")
 train_data.head()
dt_params = {'criterion': ['gini'], 'min_samples_split': [
21, 22, 23], 'max_features': ['auto', 'log2', None]}<choose_model_class>
100*train_data.isnull().mean()
rs_dt = RandomizedSearchCV(dt, param_distributions= dt_params,
scoring='accuracy', cv= StratifiedKFold(7), refit=True, n_iter= 500 )<train_model>
train_data.nunique()
rs_dt.fit(x_train, y_train )<train_model>
test_data = pd.read_csv(base + "test.csv")
 test_data.head()
print('Best Parameters are:
', rs_dt.best_params_,
'
Training accuracy score is:
', rs_dt.best_score_ )<compute_test_metric>
!pip install chart_studio
print('Validation accuracy score is:
', rs_dt.score(x_valid, y_valid))<train_on_grid>
print("Total images in Train set: ",train_data['image_name'].count())
 print("Total images in Test set: ",test_data['image_name'].count() )
param_name = 'max_depth'
param_range = np.arange(1, 21)
 train_score, valid_score = [], []
for depth in param_range:
dt = DecisionTreeClassifier(
 criterion='gini', max_features=None, min_samples_split=22, max_depth= depth)
 dt.fit(x_train, y_train)
 train_score.append(dt.score(x_train, y_train))
valid_score.append(dt.score(x_valid, y_valid))<train_model>
groupped_data = train_data.groupby(['benign_malignant', 'sex'] ).agg({'target': 'count'})
 groupped_data['%'] = groupped_data.groupby(level=0 ).apply(lambda x: 100*x/x.sum())
 groupped_data
dt = DecisionTreeClassifier(criterion='gini', max_features=None, min_samples_split=22, max_depth= 3)
 dt.fit(features,target )<compute_train_metric>
groupped_data = train_data.groupby(['diagnosis'] ).agg({'age_approx': 'mean'})
 groupped_data
y_scores_dt = dt.predict_proba(x_test)[:, 1]
dt_fpr, dt_tpr, dt_thresholds = sklearn.metrics.roc_curve(y_test, y_scores_dt)
 dt_auc = sklearn.metrics.auc(x=dt_fpr, y=dt_tpr )<compute_test_metric>
groupped_data = train_data.groupby(['diagnosis', 'anatom_site_general_challenge'] ).agg({'target': 'count'})
 groupped_data['%'] = groupped_data.groupby(level=0 ).apply(lambda x: 100*x/x.sum())
 groupped_data = groupped_data.drop(['target'], axis=1)
 groupped_data = groupped_data.reset_index()
dt_acc = dt.score(x_test, y_test )<compute_test_metric>
ids = data['patient_id'].unique().tolist()
l = []
k = 0
for i, id in enumerate(ids):
d = data.loc[data['patient_id'] == id, ].copy()
d = d.reset_index(drop=True)
 index = d.loc[d['target'] == 1, ].index.values[0]
if index == 0:
k = k+1
continue
else:
d = d[d.index < index]
groupped_data = d.groupby(['benign_malignant', 'diagnosis'] ).agg({'target': 'count'})
 groupped_data['%'] = groupped_data.groupby(level=0 ).apply(lambda x: 100*x/x.sum())
 groupped_data = groupped_data.reset_index()
l.append(groupped_data)
 print(k)
 print(len(l))
print('Area Under Curve: {}, Accuracy: {}'.format(dt_auc, dt_acc))<choose_model_class>
groupped_data = train_data.groupby(['sex', 'diagnosis'] ).agg({'target': 'count'})
 groupped_data['%'] = groupped_data.groupby(level=0 ).apply(lambda x: 100*x/x.sum())
 groupped_data
rf = ens.RandomForestClassifier()<define_search_space>
groupped_data = train_data.groupby(['sex', 'diagnosis'] ).agg({'age_approx': 'mean'})
 groupped_data
threshold = np.arange(1, 10, 0.5)*1e-1<compute_train_metric>
columns = ['benign_malignant', 'sex', 'target']
train_data[columns].groupby(['benign_malignant', 'sex'] ).count()
scores = []
for i in threshold:
selector = sklearn.feature_selection.VarianceThreshold(threshold= i)
 selected_features = selector.fit_transform(features)
 rf.fit(selected_features, target)
 y_pred = rf.predict(features.loc[:, selector.get_support() ])
 scores.append(sklearn.metrics.accuracy_score(target, y_pred))
plt.figure()
plt.xlabel("variance threshold")
 plt.ylabel("Cross validated accuracy score")
 plt.plot(threshold, np.array(scores))<compute_test_metric>
basepath = ".. /input/siim-isic-melanoma-classification/"
modelspath = ".. /input/pytorch-pretrained-image-models/"
imagestatspath = ".. /input/siimisic-melanoma-classification-image-stats/"
print('The highest accuracy score is obtained after execluding features whose variance is less than: ',
np.round(threshold[np.argmax(np.array(scores)) ],3))<compute_test_metric>
example_files = os.listdir(basepath + "train/")[0:2]
example_files
print('The highest accuracy score is:', np.max(np.array(scores)) )<define_variables>
test_image_stats = pd.read_csv(imagestatspath + "test_image_stats.csv")
 test_image_stats.head(1 )
number_of_features = list(range(1,13))<compute_test_metric>
plot_test = True
print("Maximum accuracy score is :", max(scores_k))<define_variables>
!pip install efficientnet
print("Optimal number of features :", np.argmax(np.array(scores_k)) + 1 )<define_variables>
from sklearn import metrics
from sklearn.model_selection import train_test_split
import tensorflow as tf
import tensorflow.keras.layers as L
from kaggle_datasets import KaggleDatasets
from sklearn.model_selection import train_test_split
from sklearn.metrics import auc
from tensorflow.keras.callbacks import ReduceLROnPlateau
from tensorflow.keras.callbacks import CSVLogger
import numpy as np
import pandas as pd
import efficientnet.keras as efn
print("Optimal number of features : %d" % selector.n_features_ )<compute_test_metric>
def decode_image(image):
image = tf.image.decode_jpeg(image, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
print("Maximum accuracy score is :", np.max(selector.grid_scores_))<define_search_space>
def augmentation_pipeline(image, label):
image = tf.image.random_flip_left_right(image)
 return image, label
threshold = [0.001, 0.0025, 0.005, 0.01, 0.025 ,0.05, 0.1, 0.15]<compute_test_metric>
def read_tfrecord(example, labeled):
tfrecord_format = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64)
 } if labeled else {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string)
 }
example = tf.io.parse_single_example(example, tfrecord_format)
 image = decode_image(example['image'])
 if labeled:
label = tf.cast(example['target'], tf.int32)
 return image, label
idnum = example['image_name']
return image, idnum
print("Maximum accuracy score is :", np.max(np.array(scores_sfm)) )<compute_test_metric>
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img, augment=True, dim=256):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img,DIM=dim)
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 img = tf.reshape(img, [dim,dim, 3])
 return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )
print("Optimal threshold :", threshold[np.argmax(np.array(scores_sfm)) ] )<define_search_space>
class AdvancedHairAugmentation:

 def __init__(self, hairs: int = 5, hairs_folder: str = ""):
self.hairs = hairs
self.hairs_folder = hairs_folder
def __call__(self, img):

 n_hairs = random.randint(0, self.hairs)
 if not n_hairs:
return img
height, width, _ = img.shape
hair_images = [im for im in os.listdir(self.hairs_folder)if 'png' in im]
for _ in range(n_hairs):
hair = cv2.imread(os.path.join(self.hairs_folder, random.choice(hair_images)))
 hair = cv2.flip(hair, random.choice([-1, 0, 1]))
hair = cv2.rotate(hair, random.choice([0, 1, 2]))
h_height, h_width, _ = hair.shape
roi_ho = random.randint(0, img.shape[0] - hair.shape[0])
 roi_wo = random.randint(0, img.shape[1] - hair.shape[1])
 roi = img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]
img2gray = cv2.cvtColor(hair, cv2.COLOR_BGR2GRAY)
 ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)
 mask_inv = cv2.bitwise_not(mask)
 img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)
 hair_fg = cv2.bitwise_and(hair, hair, mask=mask)
 dst = cv2.add(img_bg, hair_fg)
 img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst
return img
def __repr__(self):
return f'{self.__class__.__name__}(hairs={self.hairs}, hairs_folder="{self.hairs_folder}")'
rf_params = {'n_estimators': [200, 300, 400], 'criterion': ['gini'], 'min_samples_split': [
22, 20, 25], 'max_features': ['auto', 'log2', None], 'class_weight': [{0: 0.6, 1: 0.4}, {0: 0.6, 1: 0.4}, {0: 0.5, 1: 0.5}]}<choose_model_class>
def get_ce_loss() :
criterion = torch.nn.CrossEntropyLoss()
return criterion
rs_rf = RandomizedSearchCV(rf, param_distributions= rf_params,
scoring='accuracy', cv= StratifiedKFold(7), refit=True, n_iter= 200 )<train_model>
def get_wce_loss(train_targets):
weights = compute_class_weight(y=train_targets,
class_weight="balanced",
classes=np.unique(train_targets))
class_weights = torch.FloatTensor(weights)
 if device.type=="cuda":
class_weights = class_weights.cuda()
criterion = torch.nn.CrossEntropyLoss(weight=class_weights)
 return criterion
rs_rf.fit(x_train, y_train )<train_model>
class MulticlassFocalLoss(torch.nn.Module):
def __init__(self, train_targets=None, gamma=2):
super(MulticlassFocalLoss, self ).__init__()
self.gamma = gamma
if train_targets is None:
self.class_weights = None
else:
weights = compute_class_weight(y=train_targets,
class_weight="balanced",
classes=np.unique(train_targets))
self.class_weights = torch.FloatTensor(weights)
 if device.type=="cuda":
self.class_weights = self.class_weights.cuda()
def forward(self, input, target):
if self.class_weights is None:
ce_loss = F.cross_entropy(input, target, reduction='none')
 else:
ce_loss = F.cross_entropy(input, target, reduction='none', weight=self.class_weights)
 pt = torch.exp(-ce_loss)
 loss =(1-pt)**self.gamma * ce_loss
return torch.mean(loss )
print('Best Parameters are:
', rs_rf.best_params_,
'
Training accuracy score is:
', rs_rf.best_score_ )<compute_test_metric>
VERBOSE = 0
DISPLAY_PLOT = True
skf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)
 oof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = []
preds = np.zeros(( count_data_items(files_test),1))
for fold,(idxT,idxV)in enumerate(skf.split(np.arange(15))):
if DEVICE=='TPU':
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 print('
print(' 
(IMG_SIZES[fold],EFF_NETS[fold],BATCH_SIZES[fold]*REPLICAS))
files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxT])
 if INC2019[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1])
 print('
if INC2018[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2])
 print('
np.random.shuffle(files_train); print('
files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxV])
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/test*.tfrec')))
 K.clear_session()
with strategy.scope() :
model = build_model(dim=IMG_SIZES[fold],ef=EFF_NETS[fold])
 sv = tf.keras.callbacks.ModelCheckpoint(
 'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min', save_freq='epoch')
 print('Training...')
 history = model.fit(
 get_dataset(files_train, augment=True, shuffle=True, repeat=True,
dim=IMG_SIZES[fold], batch_size = BATCH_SIZES[fold]),
epochs=EPOCHS[fold], callbacks = [sv,get_lr_callback(BATCH_SIZES[fold])],
steps_per_epoch=count_data_items(files_train)/BATCH_SIZES[fold]//REPLICAS,
validation_data=get_dataset(files_valid,augment=False,shuffle=False,
repeat=False,dim=IMG_SIZES[fold]),
verbose=VERBOSE 
)
 print('Loading best model...')
 model.load_weights('fold-%i.h5'%fold)
 print('Predicting OOF with TTA...')
 ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_valid = count_data_items(files_valid); STEPS = TTA * ct_valid/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,]
oof_pred.append(np.mean(pred.reshape(( ct_valid,TTA),order='F'),axis=1))
ds_valid = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=True, return_image_names=True)
 oof_tar.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]))
oof_folds.append(np.ones_like(oof_tar[-1],dtype='int8')*fold)
 ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 oof_names.append(np.array([img_name.numpy().decode("utf-8")for img, img_name in iter(ds.unbatch())]))
print('Predicting Test with TTA...')
 ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,]
preds[:,0] += np.mean(pred.reshape(( ct_test,TTA),order='F'),axis=1)* WGTS[fold]
auc = roc_auc_score(oof_tar[-1],oof_pred[-1])
 oof_val.append(np.max(history.history['val_auc']))
print('
if DISPLAY_PLOT:
plt.figure(figsize=(15,5))
plt.plot(np.arange(EPOCHS[fold]),history.history['auc'],'-o',label='Train AUC',color='
plt.plot(np.arange(EPOCHS[fold]),history.history['val_auc'],'-o',label='Val AUC',color='
x = np.argmax(history.history['val_auc']); y = np.max(history.history['val_auc'])
 xdist = plt.xlim() [1] - plt.xlim() [0]; ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
%.2f'%y,size=14)
 plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)
 plt.legend(loc=2)
 plt2 = plt.gca().twinx()
plt2.plot(np.arange(EPOCHS[fold]),history.history['loss'],'-o',label='Train Loss',color='
plt2.plot(np.arange(EPOCHS[fold]),history.history['val_loss'],'-o',label='Val Loss',color='
x = np.argmin(history.history['val_loss']); y = np.min(history.history['val_loss'])
 ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
plt.ylabel('Loss',size=14)
 plt.title('FOLD %i - Image Size %i, EfficientNet B%i, inc2019=%i, inc2018=%i'% 
(fold+1,IMG_SIZES[fold],EFF_NETS[fold],INC2019[fold],INC2018[fold]),size=18)
 plt.legend(loc=3)
 plt.show()
print('Validation accuracy score is:
', rs_rf.score(x_valid, y_valid))<train_model>
import numpy as np
import pandas as pd
import os
from matplotlib import pyplot as plt
<train_model><EOS> 
adias_submission = pd.DataFrame(dict(image_name=image_names, target=preds[:,0]))
adias_submission = submission.sort_values('image_name')
 adias_submission.to_csv('adias_submission.csv', index=False )