Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<prepare_x_and_y>
DEVICE = "TPU"
BASEPATH = ".. /input/siim-isic-melanoma-classification"
X = df.values<data_type_conversions>
!pip install -q efficientnet
class DatasetRetriever(Dataset):
def __init__(self, X, transforms=None):
super().__init__()
self.X = X.reshape(-1, 28, 28 ).astype(np.float32)
 self.transforms = transforms
def __getitem__(self, index):
image = self.X[index]
image = np.stack([image] * 3, axis=-1)
 image /= 255.
 if self.transforms:
image = self.transforms(image=image)['image']
return image
def __len__(self):
return self.X.shape[0]<create_dataframe>
random.seed(a=42)
test_dataset = DatasetRetriever(
 X = X,
transforms=get_valid_transforms() ,
)<load_pretrained>
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 print("REPLICAS: ", strategy.num_replicas_in_sync)
 AUTO = tf.data.experimental.AUTOTUNE
test_loader = DataLoader(
 test_dataset,
batch_size=DataLoaderConfig.batch_size,
shuffle=False,
num_workers=DataLoaderConfig.num_workers
)<categorify>
EPOCHS = 20
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
IMAGE_SIZE = [384, 384]
FOLDS = 5
AUG_BATCH = BATCH_SIZE
FIRST_FOLD_ONLY = False
SEED = 777
result = []
for step, images in enumerate(test_loader):
print(step, end='\r')
 y_pred = net(images.to(DEVICE)).detach().cpu().numpy().argmax(axis=1 ).astype(int)
 result.extend(y_pred )<load_from_csv>
df_train = pd.read_csv(os.path.join(BASEPATH, 'train.csv'))
df_test = pd.read_csv(os.path.join(BASEPATH, 'test.csv'))
sub = pd.read_csv(os.path.join(BASEPATH, 'sample_submission.csv'))
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-384x384')
 TRAINING_FILENAMES = np.array(tf.io.gfile.glob(GCS_PATH + '/train*.tfrec'))
TEST_FILENAMES = np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec'))
CLASSES = [0,1]
sub = pd.read_csv('.. /input/digit-recognizer/sample_submission.csv', index_col=0)
 sub.head()<prepare_output>
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['target'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['image_name']
return image, idnum
def load_dataset(filenames, labeled = True, ordered = False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO)
 return dataset
def data_augment(image, label):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 image = tf.image.random_hue(image, 0.01)
 image = tf.image.random_saturation(image, 0.7, 1.3)
 image = tf.image.random_contrast(image, 0.8, 1.2)
 image = tf.image.random_brightness(image, 0.1)
 return image, label
def get_training_dataset(dataset, do_aug=False):
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.batch(AUG_BATCH)
 if do_aug: dataset = dataset.map(transform, num_parallel_calls=AUTO)
 dataset = dataset.unbatch()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(dataset, do_onehot=False):
dataset = dataset.batch(BATCH_SIZE)
 if do_onehot: dataset = dataset.map(onehot, num_parallel_calls=AUTO)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = int(count_data_items(TRAINING_FILENAMES)*(FOLDS-1.) /FOLDS)
 NUM_VALIDATION_IMAGES = int(count_data_items(TRAINING_FILENAMES)*(1./FOLDS))
NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))
sub['Label'] = result<save_to_csv>
def onehot(image,label):
CLASSES = 104
return image,tf.one_hot(label,CLASSES )
sub.to_csv('submission.csv', index=True )<set_options>
def transform(image,label):
DIM = IMAGE_SIZE[0]
CLASSES = 104
SWITCH = 0.5
CUTMIX_PROB = 0.666
MIXUP_PROB = 0.666
imgs = []; labs = []
for j in range(AUG_BATCH):
P = tf.cast(tf.random.uniform([],0,1)<=SWITCH, tf.float32)
 imgs.append(P*image2[j,]+(1-P)*image3[j,])
 labs.append(P*label2[j,]+(1-P)*label3[j,])
 image4 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))
label4 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))
return image4,label4
%matplotlib inline
np.random.seed(0 )<load_from_csv>
def lrfn(epoch):
LR_START = 0.000005
LR_MAX = 0.000075 * strategy.num_replicas_in_sync
LR_MIN = 0.000001
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.8
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
rng = [i for i in range(25 if EPOCHS<25 else EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))
train = pd.read_csv(r'/kaggle/input/digit-recognizer/train.csv')
 test = pd.read_csv(r'/kaggle/input/digit-recognizer/test.csv' )<prepare_x_and_y>
reg_l2 = 0.0001
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1 )
Y = train['label']
X = train.drop('label',axis=1 )<feature_engineering>
def get_model() :
with strategy.scope() :
enet = efn.EfficientNetB3(
 input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights='imagenet',
include_top=False 
)
 enet.trainable = True
model = tf.keras.Sequential([
enet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(1, activation='sigmoid',dtype='float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'binary_crossentropy',
metrics=['binary_accuracy'] 
)
 return model
def train_cross_validate(folds = 5):
histories = []
models = []
early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 3)
 kfold = KFold(folds, shuffle = True, random_state = SEED)
 for f,(trn_ind, val_ind)in enumerate(kfold.split(TRAINING_FILENAMES)) :
print() ; print('
print('
print('
train_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES} ).loc[trn_ind]['TRAINING_FILENAMES']), labeled = True)
 val_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES} ).loc[val_ind]['TRAINING_FILENAMES']), labeled = True, ordered = True)
 model = get_model()
history = model.fit(
 get_training_dataset(train_dataset),
steps_per_epoch = STEPS_PER_EPOCH,
epochs = EPOCHS,
callbacks = [lr_callback],
validation_data = get_validation_dataset(val_dataset),
verbose=2 
)
 models.append(model)
 histories.append(history)
 if FIRST_FOLD_ONLY: break
return histories, models
def train_and_predict(folds = 5):
test_ds = get_test_dataset(ordered=True)
 test_images_ds = test_ds.map(lambda image, idnum: image)
 print('Start training %i folds'%folds)
 histories, models = train_cross_validate(folds = folds)
 print('Computing predictions...')
 if FIRST_FOLD_ONLY: probabilities = np.average([models[i].predict(test_images_ds)for i in range(1)], axis = 0)
 else: probabilities = np.average([models[i].predict(test_images_ds)for i in range(folds)], axis = 0)
 predictions = probabilities
print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 sub = pd.read_csv(os.path.join(BASEPATH, 'sample_submission.csv'))
pred_df = pd.DataFrame({'image_name': test_ids, 'target': list(predictions)})
 pred_df.target = pred_df.target.map(lambda x: x[0])
 del sub['target']
sub = sub.merge(pred_df, on='image_name')
 sub.head()
sub.to_csv('submission.csv', index=False)
 return histories, models
histories, models = train_and_predict(folds = FOLDS )
X = X / 255.0
test = test / 255.0<categorify>
"
Y = to_categorical(Y,10 )<split>
"
<choose_model_class><EOS> 
"