Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<train_on_grid>
!pip install -q efficientnet
warnings.filterwarnings('ignore' )
rf_model = RandomForestClassifier(random_state=0)
 rf_clf = GridSearchCV(rf_model, param_grid = {'n_estimators': [200, 300,400,500],
'max_depth' : [3,4,5,6,7],
'criterion' :['gini', 'entropy']},
cv=3)
 rf_clf.fit(x_train,y_train)
 print(rf_clf.best_params_)
 print(rf_clf.best_score_)
 rf_predict = rf_clf.predict(x_test)
<compute_test_metric>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
print(sm.classification_report(y_test,rf_predict))<compute_test_metric>
AUTO = tf.data.experimental.AUTOTUNE
EPOCHS = 25
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
AUG_BATCH = BATCH_SIZE
IMAGE_SIZE = [256, 256]
SEED = 123
LR = 0.0005
if(IMAGE_SIZE[0] == 256):
print('Using size 256')
 GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-256x256')
 elif(IMAGE_SIZE[0] == 384):
print('Using size 384')
 GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-384x384')
 elif(IMAGE_SIZE[0] == 512):
print('Using size 512')
 GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-512x512')
 elif(IMAGE_SIZE[0] == 768):
print('Using size 768')
 GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-768x768')
 elif(IMAGE_SIZE[0] == 1024):
print('Using size 1024')
 GCS_PATH = KaggleDatasets().get_gcs_path('siim-isic-melanoma-classification')
 TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')
 if(IMAGE_SIZE[0] == 1024):
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/tfrecords/train*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/tfrecords/test*.tfrec')
 SUB = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv' )
sm.r2_score(y_test,rf_predict )<train_model>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))
def transform(image, label):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
tmp = random.uniform(0, 1)
 if 0 < tmp <= 0.1:
rot = 15.0 * tf.random.normal([1],dtype='float32')
 elif 0.1 < tmp <= 0.2:
rot = 30.0 * tf.random.normal([1],dtype='float32')
 elif 0.2 < tmp <= 0.3:
rot = 45.0 * tf.random.normal([1],dtype='float32')
 elif 0.3 < tmp <= 0.4:
rot = 60.0 * tf.random.normal([1],dtype='float32')
 elif 0.4 < tmp <= 0.5:
rot = 75.0 * tf.random.normal([1],dtype='float32')
 elif 0.5 < tmp <= 0.6:
rot = 90.0 * tf.random.normal([1],dtype='float32')
 elif 0.6 < tmp <= 0.7:
rot = 110.0 * tf.random.normal([1],dtype='float32')
 elif 0.7 < tmp <= 0.8:
rot = 130.0 * tf.random.normal([1],dtype='float32')
 elif 0.8 < tmp <= 0.9:
rot = 150.0 * tf.random.normal([1],dtype='float32')
 elif 0.9 < tmp <= 1.0:
rot = 180.0 * tf.random.normal([1],dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image['inp1'],tf.transpose(idx3))
return {'inp1': tf.reshape(d,[DIM,DIM,3]), 'inp2': image['inp2']}, label
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
"age_approx": tf.io.FixedLenFeature([], tf.int64),
"sex": tf.io.FixedLenFeature([], tf.int64),
"anatom_site_general_challenge": tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['target'], tf.float32)
 data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, label, data
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
"age_approx": tf.io.FixedLenFeature([], tf.int64),
"sex": tf.io.FixedLenFeature([], tf.int64),
"anatom_site_general_challenge": tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 image_name = example['image_name']
data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, image_name, data
def load_dataset(filenames, labeled = True, ordered = False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO)
 return dataset
def setup_input1(image, label, data):
anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'inp1': image, 'inp2': tabular}, label
def setup_input2(image, image_name, data):
anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'inp1': image, 'inp2': tabular}, image_name
def setup_input3(image, image_name, target, data):
anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'inp1': image, 'inp2': tabular}, image_name, target
def data_augment(data, label):
data['inp1'] = tf.image.random_flip_left_right(data['inp1'])
 data['inp1'] = tf.image.random_flip_up_down(data['inp1'])
 data['inp1'] = tf.image.random_hue(data['inp1'], 0.01)
 data['inp1'] = tf.image.random_saturation(data['inp1'], 0.8, 1.2)
 data['inp1'] = tf.image.random_contrast(data['inp1'], 0.8, 1.2)
 data['inp1'] = tf.image.random_brightness(data['inp1'], 0.1)
 return data, label
def get_training_dataset(filenames, labeled = True, ordered = False):
dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(setup_input1, num_parallel_calls = AUTO)
 dataset = dataset.map(data_augment, num_parallel_calls = AUTO)
 dataset = dataset.map(transform, num_parallel_calls = AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(filenames, labeled = True, ordered = True):
dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(setup_input1, num_parallel_calls = AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(filenames, labeled = False, ordered = True):
dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(setup_input2, num_parallel_calls = AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 def read_tfrecord_full(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
"age_approx": tf.io.FixedLenFeature([], tf.int64),
"sex": tf.io.FixedLenFeature([], tf.int64),
"anatom_site_general_challenge": tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 image_name = example['image_name']
target = tf.cast(example['target'], tf.float32)
 data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, image_name, target, data
def load_dataset_full(filenames):
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.map(read_tfrecord_full, num_parallel_calls = AUTO)
 return dataset
def get_data_full(filenames):
dataset = load_dataset_full(filenames)
 dataset = dataset.map(setup_input3, num_parallel_calls = AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
NUM_TRAINING_IMAGES = int(count_data_items(TRAINING_FILENAMES)* 0.8)
 NUM_VALIDATION_IMAGES = int(count_data_items(TRAINING_FILENAMES)* 0.2)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))
model = RandomForestClassifier(random_state=0,
max_depth = 6,
n_estimators = 200)
 model.fit(x,y)
 pred =(model.predict(final_test))<create_dataframe>
%%time
def binary_focal_loss(gamma=2., alpha=.25):

 def binary_focal_loss_fixed(y_true, y_pred):

 pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
epsilon = K.epsilon()
pt_1 = K.clip(pt_1, epsilon, 1.- epsilon)
 pt_0 = K.clip(pt_0, epsilon, 1.- epsilon)
 return -K.sum(alpha * K.pow(1.- pt_1, gamma)* K.log(pt_1)) \
-K.sum(( 1 - alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0))
return binary_focal_loss_fixed
def get_model() :
with strategy.scope() :
inp1 = tf.keras.layers.Input(shape =(*IMAGE_SIZE, 3), name = 'inp1')
 inp2 = tf.keras.layers.Input(shape =(9), name = 'inp2')
 efnetb6 = efn.EfficientNetB6(weights = 'imagenet', include_top = False)
 x6 = efnetb6(inp1)
 x6 = tf.keras.layers.GlobalAveragePooling2D()(x6)
 x1 = tf.keras.layers.Dense(32 )(inp2)
 x1 = tf.keras.layers.BatchNormalization()(x1)
 x1 = tf.keras.layers.Activation('relu' )(x1)
 concat = tf.keras.layers.concatenate([x6, x1])
 concat = tf.keras.layers.Dense(256, activation = 'relu' )(concat)
 concat = tf.keras.layers.BatchNormalization()(concat)
 concat = tf.keras.layers.Dropout(0.2 )(concat)
 concat = tf.keras.layers.Dense(64, activation = 'relu' )(concat)
 concat = tf.keras.layers.BatchNormalization()(concat)
 concat = tf.keras.layers.Dropout(0.2 )(concat)
 output = tf.keras.layers.Dense(1, activation = 'sigmoid' )(concat)
 model = tf.keras.models.Model(inputs = [inp1, inp2], outputs = [output])
 opt = tf.keras.optimizers.Adam(learning_rate = LR)
 model.compile(
 optimizer = opt,
loss = [tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.05)],
metrics = [tf.keras.metrics.BinaryAccuracy() , tf.keras.metrics.AUC() ] 
)
 print(model.summary())
 return model
def train_and_predict(SUB, folds = 5):
models = []
oof_image_name = []
oof_target = []
oof_prediction = []
seed_everything(SEED)
 kfold = KFold(folds, shuffle = True, random_state = SEED)
 for fold,(trn_ind, val_ind)in enumerate(kfold.split(TRAINING_FILENAMES)) :
tic = time.time()
print('
')
 print('-'*50)
 print(f'Training fold {fold}')
 print('-'*50)
 print('
')
 train_dataset = get_training_dataset([TRAINING_FILENAMES[x] for x in trn_ind], labeled = True, ordered = False)
 val_dataset = get_validation_dataset([TRAINING_FILENAMES[x] for x in val_ind], labeled = True, ordered = True)
 K.clear_session()
model = get_model()
early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_auc', mode = 'max', patience = 10,
verbose = 1, min_delta = 0.0001, restore_best_weights = True)
 cb_lr_schedule = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_auc', factor = 0.5, patience = 2, verbose = 1, min_delta = 0.0001, mode = 'max')
 history = model.fit(train_dataset,
steps_per_epoch = STEPS_PER_EPOCH,
epochs = EPOCHS,
callbacks = [early_stopping, cb_lr_schedule],
validation_data = val_dataset,
verbose = 2)
 models.append(model)
 model.save_weights(f'ENet_fold_{fold}.h5')
 print('Weights saved!')
 number_of_files = count_data_items([TRAINING_FILENAMES[x] for x in val_ind])
 dataset = get_data_full([TRAINING_FILENAMES[x] for x in val_ind])
 image_name = dataset.map(lambda image, image_name, target: image_name ).unbatch()
image_name = next(iter(image_name.batch(number_of_files)) ).numpy().astype('U')
 target = dataset.map(lambda image, image_name, target: target ).unbatch()
target = next(iter(target.batch(number_of_files)) ).numpy()
image = dataset.map(lambda image, image_name, target: image)
 probabilities = model.predict(image)
 oof_image_name.extend(list(image_name))
oof_target.extend(list(target))
oof_prediction.extend(list(np.concatenate(probabilities)))
 toc = time.time()
print('
')
 print('Time taken:', toc-tic)
 print('
')
 print('-'*50)
 oof_df = pd.DataFrame({'image_name': oof_image_name, 'target': oof_target, 'predictions': oof_prediction})
 oof_df.to_csv('oof.csv', index = False)
 test_ds = get_test_dataset(TEST_FILENAMES, labeled = False, ordered = True)
 test_images_ds = test_ds.map(lambda image, image_name: image)
 print('Computing predictions...')
 probabilities = np.average([np.concatenate(models[i].predict(test_images_ds)) for i in range(folds)], axis = 0)
 print('Generating submission file...')
 test_ids_ds = test_ds.map(lambda image, image_name: image_name ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 pred_df = pd.DataFrame({'image_name': test_ids, 'target': probabilities})
 SUB.drop('target', inplace = True, axis = 1)
 SUB = SUB.merge(pred_df, on = 'image_name')
 SUB.to_csv(f'submission.csv', index = False)
 return oof_target, oof_prediction
oof_target, oof_prediction = train_and_predict(SUB )
<save_to_csv><EOS> 
roc_auc = metrics.roc_auc_score(oof_target, oof_prediction)
 print('Our out of folds roc auc score is: ', roc_auc )