Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<compute_train_metric>
!pip install -q efficientnet
lr = LogisticRegression(max_iter = 2000)
 cv = cross_val_score(lr,X_train,y_train,cv=5)
 print(cv)
 print(cv.mean() )<compute_train_metric>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
lr = LogisticRegression(max_iter = 2000)
 cv = cross_val_score(lr,X_train_scaled,y_train,cv=5)
 print(cv)
 print(cv.mean() )<compute_train_metric>
AUTO = tf.data.experimental.AUTOTUNE
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-384x384')
 EPOCHS = 40
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
AUG_BATCH = BATCH_SIZE
IMAGE_SIZE = [384, 384]
SEED = 123
LR = 0.0003
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')
 SUB = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv' )
dt = tree.DecisionTreeClassifier(random_state = 1)
 cv = cross_val_score(dt,X_train,y_train,cv=5)
 print(cv)
 print(cv.mean() )<compute_train_metric>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))
def transform(image, label):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
if 0.5 > tf.random.uniform([1], minval = 0, maxval = 1):
rot = 15.* tf.random.normal([1],dtype='float32')
 else:
rot = 180.* tf.random.normal([1],dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image['inp1'],tf.transpose(idx3))
return {'inp1': tf.reshape(d,[DIM,DIM,3]), 'inp2': image['inp2']}, label
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
"age_approx": tf.io.FixedLenFeature([], tf.int64),
"sex": tf.io.FixedLenFeature([], tf.int64),
"anatom_site_general_challenge": tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['target'], tf.int32)
 data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, label, data
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
"age_approx": tf.io.FixedLenFeature([], tf.int64),
"sex": tf.io.FixedLenFeature([], tf.int64),
"anatom_site_general_challenge": tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 image_name = example['image_name']
data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, image_name, data
def load_dataset(filenames, labeled = True, ordered = False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO)
 return dataset
def setup_input1(image, label, data):
anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'inp1': image, 'inp2': tabular}, label
def setup_input2(image, image_name, data):
anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'inp1': image, 'inp2': tabular}, image_name
def setup_input3(image, image_name, target, data):
anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'inp1': image, 'inp2': tabular}, image_name, target
def data_augment(data, label):
data['inp1'] = tf.image.random_flip_left_right(data['inp1'])
 data['inp1'] = tf.image.random_flip_up_down(data['inp1'])
 data['inp1'] = tf.image.random_hue(data['inp1'], 0.01)
 data['inp1'] = tf.image.random_saturation(data['inp1'], 0.7, 1.3)
 data['inp1'] = tf.image.random_contrast(data['inp1'], 0.8, 1.2)
 data['inp1'] = tf.image.random_brightness(data['inp1'], 0.1)
 return data, label
def get_training_dataset(filenames, labeled = True, ordered = False):
dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(setup_input1, num_parallel_calls = AUTO)
 dataset = dataset.map(data_augment, num_parallel_calls = AUTO)
 dataset = dataset.map(transform, num_parallel_calls = AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(filenames, labeled = True, ordered = True):
dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(setup_input1, num_parallel_calls = AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(filenames, labeled = False, ordered = True):
dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(setup_input2, num_parallel_calls = AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 def read_tfrecord_full(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
"age_approx": tf.io.FixedLenFeature([], tf.int64),
"sex": tf.io.FixedLenFeature([], tf.int64),
"anatom_site_general_challenge": tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 image_name = example['image_name']
target = tf.cast(example['target'], tf.int32)
 data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, image_name, target, data
def load_dataset_full(filenames):
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.map(read_tfrecord_full, num_parallel_calls = AUTO)
 return dataset
def get_data_full(filenames):
dataset = load_dataset_full(filenames)
 dataset = dataset.map(setup_input3, num_parallel_calls = AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
NUM_TRAINING_IMAGES = int(count_data_items(TRAINING_FILENAMES)* 0.8)
 NUM_VALIDATION_IMAGES = int(count_data_items(TRAINING_FILENAMES)* 0.2)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))
dt = tree.DecisionTreeClassifier(random_state = 1)
 cv = cross_val_score(dt,X_train_scaled,y_train,cv=5)
 print(cv)
 print(cv.mean() )<compute_train_metric>
def binary_focal_loss(gamma=2., alpha=.25):

 def binary_focal_loss_fixed(y_true, y_pred):

 pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
epsilon = K.epsilon()
pt_1 = K.clip(pt_1, epsilon, 1.- epsilon)
 pt_0 = K.clip(pt_0, epsilon, 1.- epsilon)
 return -K.sum(alpha * K.pow(1.- pt_1, gamma)* K.log(pt_1)) \
-K.sum(( 1 - alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0))
return binary_focal_loss_fixed
def get_model() :
with strategy.scope() :
inp1 = tf.keras.layers.Input(shape =(*IMAGE_SIZE, 3), name = 'inp1')
 inp2 = tf.keras.layers.Input(shape =(9), name = 'inp2')
 efnetb3 = efn.EfficientNetB3(weights = 'imagenet', include_top = False)
 x = efnetb3(inp1)
 x = tf.keras.layers.GlobalAveragePooling2D()(x)
 x1 = tf.keras.layers.Dense(64 )(inp2)
 x1 = tf.keras.layers.BatchNormalization()(x1)
 x1 = tf.keras.layers.Activation('relu' )(x1)
 concat = tf.keras.layers.concatenate([x, x1])
 concat = tf.keras.layers.Dense(512, activation = 'relu' )(concat)
 concat = tf.keras.layers.BatchNormalization()(concat)
 concat = tf.keras.layers.Dropout(0.2 )(concat)
 concat = tf.keras.layers.Dense(256, activation = 'relu' )(concat)
 concat = tf.keras.layers.BatchNormalization()(concat)
 concat = tf.keras.layers.Dropout(0.2 )(concat)
 output = tf.keras.layers.Dense(1, activation = 'sigmoid' )(concat)
 model = tf.keras.models.Model(inputs = [inp1, inp2], outputs = [output])
 opt = tf.keras.optimizers.Adam(learning_rate = LR)
 model.compile(
 optimizer = opt,
loss = [binary_focal_loss(gamma = 2.0, alpha = 0.80)],
metrics = [tf.keras.metrics.BinaryAccuracy() , tf.keras.metrics.AUC() ] 
)
 return model
def train_and_predict(SUB, folds = 5):
models = []
oof_image_name = []
oof_target = []
oof_prediction = []
seed_everything(SEED)
 kfold = KFold(folds, shuffle = True, random_state = SEED)
 for fold,(trn_ind, val_ind)in enumerate(kfold.split(TRAINING_FILENAMES)) :
print('
')
 print('-'*50)
 print(f'Training fold {fold + 1}')
 train_dataset = get_training_dataset([TRAINING_FILENAMES[x] for x in trn_ind], labeled = True, ordered = False)
 val_dataset = get_validation_dataset([TRAINING_FILENAMES[x] for x in val_ind], labeled = True, ordered = True)
 K.clear_session()
model = get_model()
early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_auc', mode = 'max', patience = 5,
verbose = 1, min_delta = 0.0001, restore_best_weights = True)
 cb_lr_schedule = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_auc', factor = 0.5, patience = 2, verbose = 1, min_delta = 0.0001, mode = 'max')
 history = model.fit(train_dataset,
steps_per_epoch = STEPS_PER_EPOCH,
epochs = EPOCHS,
callbacks = [early_stopping, cb_lr_schedule],
validation_data = val_dataset,
verbose = 2)
 models.append(model)
 number_of_files = count_data_items([TRAINING_FILENAMES[x] for x in val_ind])
 dataset = get_data_full([TRAINING_FILENAMES[x] for x in val_ind])
 image_name = dataset.map(lambda image, image_name, target: image_name ).unbatch()
image_name = next(iter(image_name.batch(number_of_files)) ).numpy().astype('U')
 target = dataset.map(lambda image, image_name, target: target ).unbatch()
target = next(iter(target.batch(number_of_files)) ).numpy()
image = dataset.map(lambda image, image_name, target: image)
 probabilities = model.predict(image)
 oof_image_name.extend(list(image_name))
oof_target.extend(list(target))
oof_prediction.extend(list(np.concatenate(probabilities)))
 print('
')
 print('-'*50)
 oof_df = pd.DataFrame({'image_name': oof_image_name, 'target': oof_target, 'predictions': oof_prediction})
 oof_df.to_csv('EfficientNetB3_384.csv', index = False)
 test_ds = get_test_dataset(TEST_FILENAMES, labeled = False, ordered = True)
 test_images_ds = test_ds.map(lambda image, image_name: image)
 print('Computing predictions...')
 probabilities = np.average([np.concatenate(models[i].predict(test_images_ds)) for i in range(folds)], axis = 0)
 print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, image_name: image_name ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 pred_df = pd.DataFrame({'image_name': test_ids, 'target': probabilities})
 SUB.drop('target', inplace = True, axis = 1)
 SUB = SUB.merge(pred_df, on = 'image_name')
 SUB.to_csv('sub_EfficientNetB3_384.csv', index = False)
 return oof_target, oof_prediction
oof_target, oof_prediction = train_and_predict(SUB )
<compute_train_metric><EOS> 
roc_auc = metrics.roc_auc_score(oof_target, oof_prediction)
 print('Our out of folds roc auc score is: ', roc_auc )