Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<choose_model_class>
DEVICE = "TPU"
CFG = dict(
 net_count = 7,
batch_size = 32,
read_size = 512,
crop_size = 512,
net_size = 512,
LR_START = 0.000005,
LR_MAX = 0.000020,
LR_MIN = 0.000001,
LR_RAMPUP_EPOCHS = 10,
LR_SUSTAIN_EPOCHS = 2,
LR_EXP_DECAY = 0.7,
epochs = 25,
rot = 180.0,
shr = 2.0,
hzoom = 8.0,
wzoom = 8.0,
hshift = 8.0,
wshift = 8.0,
optimizer = 'adam',
label_smooth_fac = 0.05,
tta_steps = 25
)
init = RandomNormal(stddev=0.02)
 model = Sequential([
Conv2D(32, 3, input_shape=(28, 28, 1), activation='relu', kernel_initializer=init),
BatchNormalization() ,
Conv2D(32, 3, activation='relu', kernel_initializer=init),
BatchNormalization() ,
Conv2D(32, 5, strides=2, padding='same', activation='relu', kernel_initializer=init),
BatchNormalization() ,
Dropout(0.4),
Conv2D(64, 3, activation='relu', kernel_initializer=init),
BatchNormalization() ,
Conv2D(64, 3, activation='relu', kernel_initializer=init),
BatchNormalization() ,
Conv2D(64, 5, strides=2, padding='same', activation='relu', kernel_initializer=init),
BatchNormalization() ,
Dropout(0.4),
Conv2D(128, 4, activation='relu', kernel_initializer=init),
BatchNormalization() ,
Flatten() ,
Dropout(0.4),
Dense(10, activation='softmax')
 ])
 model.summary()<choose_model_class>
!pip install -q efficientnet
model.compile(
 optimizer=Adam(lr=1e-3),
loss='sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy']
)<choose_model_class>
random.seed(a=42)
learning_rate_reduction = ReduceLROnPlateau(
 monitor='val_sparse_categorical_accuracy',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001
)<train_model>
BASEPATH = ".. /input/siim-isic-melanoma-classification"
df_train = pd.read_csv(os.path.join(BASEPATH, 'train.csv'))
df_test = pd.read_csv(os.path.join(BASEPATH, 'test.csv'))
df_sub = pd.read_csv(os.path.join(BASEPATH, 'sample_submission.csv'))
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-512x512')
 GCS_PATH_2019 = KaggleDatasets().get_gcs_path('isic2019-512x512')
 files_train_2019 = np.array(tf.io.gfile.glob(GCS_PATH_2019 + '/train*.tfrec'))
files_train_2020 = np.array(tf.io.gfile.glob(GCS_PATH + '/train*.tfrec'))
files_train = np.sort(np.concatenate(( files_train_2020, files_train_2019)))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')) )
history = model.fit(
 datagen.flow(X_train, Y_train, batch_size=64),
epochs=45,
validation_data=(X_val, Y_val),
callbacks=[learning_rate_reduction],
use_multiprocessing=True
)<save_to_csv>
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )
results = model.predict(test)
 results = np.argmax(results, axis=1)
 submission = pd.concat([
pd.Series(range(1,28001), name="ImageId"),
pd.Series(results, name="Label")
 ], axis=1)
 submission.to_csv("submission.csv", index=False )<set_options>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, cfg):
DIM = cfg["read_size"]
XDIM = DIM%2
rot = cfg['rot'] * tf.random.normal([1], dtype='float32')
 shr = cfg['shr'] * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['hzoom']
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['wzoom']
h_shift = cfg['hshift'] * tf.random.normal([1], dtype='float32')
 w_shift = cfg['wshift'] * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )
seed(42)
 tf.random.set_seed(42)
<load_from_csv>
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return(example['image'],(example['sex'], example['age_approx'], example['anatom_site_general_challenge']), example['target'])
 def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return(example['image'],(example['sex'], example['age_approx'], example['anatom_site_general_challenge']),(example['image_name'] if return_image_name else 0))
def prepare_data(data, cfg=None, augment=True):
img = data[0]
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.image.resize(img, [cfg['read_size'], cfg['read_size']])
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img, cfg)
 img = tf.image.random_crop(img, [cfg['crop_size'], cfg['crop_size'], 3])
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_hue(img, 0.01)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 else:
img = tf.image.central_crop(img, cfg['crop_size'] / cfg['read_size'])
 img = tf.image.resize(img, [cfg['net_size'], cfg['net_size']])
 img = tf.reshape(img, [cfg['net_size'], cfg['net_size'], 3])
 sex_oh = tf.one_hot(data[1][0], 2)
 age_aprox = tf.dtypes.cast(tf.reshape(data[1][1], [1]), tf.float32)
 anatom_site_general_challenge = tf.one_hot(data[1][2], 7)
 dense = tf.concat([sex_oh, age_aprox, anatom_site_general_challenge], axis=0)
 return(img, dense)
 def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )
train_df = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test_df = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 sub = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv" )<count_missing_values>
def get_dataset(files, cfg, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, dense, imgname_or_label:(prepare_data(( img, dense), augment=augment, cfg=cfg),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(cfg['batch_size'] * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds
train_df.isnull().sum()<count_missing_values>
def show_dataset(thumb_size, cols, rows, ds):
mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols +(cols-1),
thumb_size*rows +(rows-1)))
 for idx, data in enumerate(iter(ds)) :
img, target_or_imgid = data[0][0], data[1]
ix = idx % cols
iy = idx // cols
img = np.clip(img.numpy() * 255, 0, 255 ).astype(np.uint8)
 img = PIL.Image.fromarray(img)
 img = img.resize(( thumb_size, thumb_size), resample=PIL.Image.BILINEAR)
 mosaic.paste(img,(ix*thumb_size + ix,
iy*thumb_size + iy))
display(mosaic)
 ds = get_dataset(files_train, CFG ).unbatch().take(12*5)
 show_dataset(64, 12, 5, ds )
test_df.isnull().sum()<load_from_url>
len(df_train.diagnosis.unique() )
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py<import_modules>
ds = tf.data.TFRecordDataset(files_train, num_parallel_reads=AUTO)
 ds = ds.take(1 ).cache().repeat()
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 ds = ds.map(lambda img, dense, target:(prepare_data(( img, dense), cfg=CFG, augment=True), target),
num_parallel_calls=AUTO)
 ds = ds.take(12*5)
 ds = ds.prefetch(AUTO)
 show_dataset(64, 12, 5, ds )
import tokenization<categorify>
ds = get_dataset(files_test, CFG, augment=True, repeat=True,
labeled=False, return_image_names=False ).unbatch().take(12*5)
 show_dataset(64, 12, 5, ds )
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )<define_variables>
def get_lr_callback(cfg):
lr_start = cfg['LR_START']
lr_max = cfg['LR_MAX'] * strategy.num_replicas_in_sync
lr_min = cfg['LR_MIN']
lr_ramp_ep = cfg['LR_RAMPUP_EPOCHS']
lr_sus_ep = cfg['LR_SUSTAIN_EPOCHS']
lr_decay = cfg['LR_EXP_DECAY']
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback
Dropout_num = 0
learning_rate = 7e-6
valid = 0.2
epochs_num = 3
batch_size_num = 16
target_corrected = False
target_big_corrected = False<choose_model_class>
def get_model(cfg):
model_input = tf.keras.Input(shape=(cfg['net_size'], cfg['net_size'], 3), name='imgIn')
 model_input_data = tf.keras.Input(shape=(10,), name='dataIn')
 dense = tf.keras.layers.Dense(256, activation='relu' )(model_input_data)
 dense = tf.keras.layers.BatchNormalization()(dense)
 dense = tf.keras.layers.Dropout(0.5 )(dense)
 dense = tf.keras.layers.Dense(64, activation='relu' )(dense)
 dense = tf.keras.layers.BatchNormalization()(dense)
 dense = tf.keras.layers.Dropout(0.5 )(dense)
 constructor = getattr(efn, f'EfficientNetB6')
 x = constructor(include_top=False, weights='imagenet',
input_shape=(cfg['net_size'], cfg['net_size'], 3),
pooling='avg' )(model_input)
 x = tf.keras.layers.Concatenate()([x, dense])
 x = tf.keras.layers.Dense(256, activation='relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.5 )(x)
 x = tf.keras.layers.Dense(1, activation='sigmoid' )(x)
 model = tf.keras.Model([model_input, model_input_data], x, name='aNetwork')
 model.summary()
return model
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
if Dropout_num == 0:
out = Dense(1, activation='sigmoid' )(clf_output)
 else:
x = Dropout(Dropout_num )(clf_output)
 out = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=learning_rate), loss='binary_crossentropy', metrics=['accuracy'])
 return model<choose_model_class>
def compile_new_model(cfg, old_model=None):
with strategy.scope() :
if(old_model is None):
model = get_model(cfg)
 else:
model = old_model
model.compile(
 optimizer = tf.keras.optimizers.Adam(learning_rate=0.1),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = cfg['label_smooth_fac']),
metrics = [tf.keras.metrics.AUC(name='auc')])
 return model
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )<feature_engineering>
train = pd.read_csv('.. /input/siim-isic-melanoma-classification/train.csv')
 class_weights = class_weight.compute_class_weight('balanced',
np.unique(train.target),
train.target)
 class_weights = dict(enumerate(class_weights))
if target_corrected:
ids_with_target_error = [328,443,513,2619,3640,3900,4342,5781,6552,6554,6570,6701,6702,6729,6861,7226]
train_df.loc[train_df['id'].isin(ids_with_target_error),'target'] = 0
train_df[train_df['id'].isin(ids_with_target_error)]<feature_engineering>
file_path="/kaggle/working/effnet6_weights.best.hdf5"
checkpoint = ModelCheckpoint(file_path, monitor='auc', verbose=1, save_best_only=True, mode='max' )
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )<categorify>
ds_train = get_dataset(files_train, CFG, augment=True, shuffle=True, repeat=True)
 steps_train = count_data_items(files_train)/(CFG['batch_size'] * REPLICAS )
train_input = bert_encode(train_df.text.values, tokenizer, max_len=160)
 test_input = bert_encode(test_df.text.values, tokenizer, max_len=160)
 train_labels = train_df.target.values<train_model>
print("Buidling model...")
 model = compile_new_model(CFG )
train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=5,
batch_size=8 
)
 model.save('model.h5' )<predict_on_test>
history = model.fit(ds_train,
verbose = 1,
steps_per_epoch = steps_train,
epochs = CFG['epochs'],
callbacks = [checkpoint, get_lr_callback(CFG)],
class_weight=class_weights )
test_pred = model.predict(test_input )<save_to_csv>
model = compile_new_model(CFG)
 model.load_weights(file_path )
sub['target'] = test_pred.round().astype(int)
 sub.to_csv('submission.csv', index=False)
 sub.head()<set_options>
CFG['batch_size'] = 128
cnt_test = count_data_items(files_test)
 steps = cnt_test /(CFG['batch_size'] * REPLICAS)* CFG['tta_steps']
ds_testAug = get_dataset(files_test, CFG, augment=True, repeat=True,
labeled=False, return_image_names=False)
 probs = model.predict(ds_testAug, verbose=1, steps=steps)
 probs = np.stack(probs)
 probs = probs2 = probs[:cnt_test * CFG['tta_steps'],0]
probs = np.stack(np.split(probs, CFG['tta_steps'], axis=0), axis=1)
 probs = np.mean(probs, axis=1 )
%matplotlib inline 
<load_from_csv>
ds = get_dataset(files_test, CFG, augment=False, repeat=False,
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )
<split><EOS> 
submission = pd.DataFrame(dict(
 image_name = image_names,
target = probs[:]))
submission = submission.sort_values('image_name')
 submission.to_csv('submission_model.csv', index=False )