Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<load_pretrained>
DEVICE = "TPU"
CFG = dict(
 batch_size = 16,
read_size = 256,
crop_size = 250,
net_size = 248,
LR_START = 0.000003,
LR_MAX = 0.000020,
LR_MIN = 0.000001,
LR_RAMPUP_EPOCHS = 5,
LR_SUSTAIN_EPOCHS = 0,
LR_EXP_DECAY = 0.8,
epochs = 13,
rot = 180.0,
shr = 1.5,
hzoom = 6.0,
wzoom = 6.0,
hshift = 6.0,
wshift = 6.0,
optimizer = 'adam',
label_smooth_fac = 0.05,
tta_steps = 25
)
def evaluate(model, test_dataset):
test_dl = torchdata.DataLoader(test_dataset, batch_size=1, num_workers=1, sampler=torchdata.SequentialSampler(test_dataset))
predicts = []
model.eval()
with torch.no_grad() :
for x, attn in tqdm(test_dl):
x = x.to(device)
 attn = attn.to(device)
 logits, = model(x, attention_mask=attn)
 pred = torch.sigmoid(logits)> 0.5
pred = pred.cpu().squeeze().item()
predicts.append(pred)
 return np.array(predicts )<choose_model_class>
!pip install -q efficientnet
model = model.to(device)
 optimizer = tr.optimization.AdamW(params=model.parameters() , lr=cfg.lr)
 scheduler = tr.get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=n_warmup_steps, num_training_steps=total_steps )<train_model>
random.seed(a=42)
global_train_loss = []
global_val_acc = []
global_val_f1 = []
global_val_loss = []
for i in range(cfg.n_epochs):
model.train()
train_loss = []
for x, attn, y in tqdm(train_dl, unit='batch', desc=f"Train {i}"):
result = model_step(model, x, attn, y)
 loss = result['loss']
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , cfg.max_grad_norm)
 optimizer.step()
scheduler.step()
train_loss.append(loss.item())
 optimizer.zero_grad()
global_train_loss += train_loss
model.eval()
valid_loss = []
valid_f1 = []
valid_acc = []
with torch.no_grad() :
for x, attn, y in tqdm(valid_dl, unit='batch', desc=f"Valid {i}"):
result = model_step(model, x, attn, y, compute_metrics=True)
 valid_loss.append(result['loss'].item())
 valid_f1.append(result['f1'])
 valid_acc.append(result['accuracy'])
 global_val_acc += valid_acc
global_val_f1 += valid_f1
global_val_loss += valid_loss
print(f"Epoch {i}: Train Loss = {np.mean(train_loss):.6f}, Val Loss = {np.mean(valid_loss):.6f}, F1 = {np.mean(valid_f1):.4f}, Accuracy = {np.mean(valid_acc):.4f}" )<compute_test_metric>
BASEPATH = ".. /input/siim-isic-melanoma-classification"
df_train = pd.read_csv(os.path.join(BASEPATH, 'train.csv'))
df_test = pd.read_csv(os.path.join(BASEPATH, 'test.csv'))
df_sub = pd.read_csv(os.path.join(BASEPATH, 'sample_submission.csv'))
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-256x256')
 files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')) )
true, pred = [], []
for x, attn, y in tqdm(valid_dl):
with torch.no_grad() :
y_true, y_pred = model_step(model, x, attn, y, return_true_pred=True)['data']
true += list(y_true)
 pred += list(y_pred)
 true = np.vstack(true)
 pred = np.vstack(pred)
 f1 = ms.f1_score(y_true=true, y_pred=pred)
 acc = ms.accuracy_score(y_true=true, y_pred=pred )<compute_test_metric>
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )
print(f"F1 - {f1:.4f}")
 print(f"Acc - {acc:.4f}" )<compute_test_metric>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, cfg):
DIM = cfg["read_size"]
XDIM = DIM%2
rot = cfg['rot'] * tf.random.normal([1], dtype='float32')
 shr = cfg['shr'] * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['hzoom']
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['wzoom']
h_shift = cfg['hshift'] * tf.random.normal([1], dtype='float32')
 w_shift = cfg['wshift'] * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )
predicts = evaluate(model, test_dataset )<data_type_conversions>
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img, cfg=None, augment=True):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.image.resize(img, [cfg['read_size'], cfg['read_size']])
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img, cfg)
 img = tf.image.random_crop(img, [cfg['crop_size'], cfg['crop_size'], 3])
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_hue(img, 0.01)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 else:
img = tf.image.central_crop(img, cfg['crop_size'] / cfg['read_size'])
 img = tf.image.resize(img, [cfg['net_size'], cfg['net_size']])
 img = tf.reshape(img, [cfg['net_size'], cfg['net_size'], 3])
 return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )
submission = pd.DataFrame({
'id': test['id'],
'target': predicts.astype(np.int32)
 } )<save_to_csv>
def get_dataset(files, cfg, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, imgname_or_label:(prepare_image(img, augment=augment, cfg=cfg),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(cfg['batch_size'] * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds
submission.to_csv("./submission.csv", index=False, encoding='utf-8' )<set_options>
def show_dataset(thumb_size, cols, rows, ds):
mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols +(cols-1),
thumb_size*rows +(rows-1)))
 for idx, data in enumerate(iter(ds)) :
img, target_or_imgid = data
ix = idx % cols
iy = idx // cols
img = np.clip(img.numpy() * 255, 0, 255 ).astype(np.uint8)
 img = PIL.Image.fromarray(img)
 img = img.resize(( thumb_size, thumb_size), resample=PIL.Image.BILINEAR)
 mosaic.paste(img,(ix*thumb_size + ix,
iy*thumb_size + iy))
display(mosaic)
 ds = get_dataset(files_train, CFG ).unbatch().take(12*5)
 show_dataset(64, 12, 5, ds )
warnings.simplefilter(action='ignore', category=FutureWarning)
<load_from_csv>
ds = tf.data.TFRecordDataset(files_train, num_parallel_reads=AUTO)
 ds = ds.take(1 ).cache().repeat()
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 ds = ds.map(lambda img, target:(prepare_image(img, cfg=CFG, augment=True), target),
num_parallel_calls=AUTO)
 ds = ds.take(12*5)
 ds = ds.prefetch(AUTO)
 show_dataset(64, 12, 5, ds )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print(train.shape)
 print(test.shape )<drop_column>
ds = get_dataset(files_test, CFG, labeled=False ).unbatch().take(12*5)
 show_dataset(64, 12, 5, ds )
test.drop(['Ticket','Cabin'],axis=1, inplace=True)
 test.set_index('PassengerId',inplace=True)
 train.drop(['Ticket','Cabin'],axis=1, inplace=True)
 train.set_index('PassengerId',inplace=True)
 train.head()<categorify>
def get_lr_callback(cfg):
lr_start = cfg['LR_START']
lr_max = cfg['LR_MAX'] * strategy.num_replicas_in_sync
lr_min = cfg['LR_MIN']
lr_ramp_ep = cfg['LR_RAMPUP_EPOCHS']
lr_sus_ep = cfg['LR_SUSTAIN_EPOCHS']
lr_decay = cfg['LR_EXP_DECAY']
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback
test['title'] = test.Name.str.extract('([A-Za-z]+)\.')
 test['title'].replace(['Mlle','Mme','Ms','Major','Lady','Countess','Jonkheer','Col','Rev','Capt','Sir','Don','Dona','Dr'],['Miss','Miss','Miss','Mr','Mrs','Mrs','Other','Other','Other','Mr','Mr','Mr','Miss','Mr'],inplace=True)
 test = pd.concat([test.drop(['title','Name'],axis=1), pd.get_dummies(test[['title']], drop_first=True)], axis=1)
 train['title'] = train.Name.str.extract('([A-Za-z]+)\.')
 train['title'].replace(['Mlle','Mme','Ms','Major','Lady','Countess','Jonkheer','Col','Rev','Capt','Sir','Don','Dona','Dr'],['Miss','Miss','Miss','Mr','Mrs','Mrs','Other','Other','Other','Mr','Mr','Mr','Miss','Mr'],inplace=True)
 train = pd.concat([train.drop(['title','Name'],axis=1), pd.get_dummies(train[['title']], drop_first=True)], axis=1)
 train.head()<categorify>
def get_model(cfg, model):
model_input = tf.keras.Input(shape=(cfg['net_size'], cfg['net_size'], 3), name='imgIn')
 dummy = tf.keras.layers.Lambda(lambda x:x )(model_input)
 outputs = []
constructor = getattr(efn, model)
 x = constructor(include_top=False, weights='imagenet',
input_shape=(cfg['net_size'], cfg['net_size'], 3),
pooling='avg' )(dummy)
 x = tf.keras.layers.Dense(1, activation='sigmoid' )(x)
 outputs.append(x)
 model = tf.keras.Model(model_input, outputs, name='aNetwork')
 model.summary()
return model
features_numeric = ['Age','SibSp','Parch','Fare']
features_label = ['Pclass','Sex','Embarked']
test = test.fillna(test[features_numeric].mean())
 test = test.fillna(test[features_label].mode().iloc[0])
 train = train.fillna(train[features_numeric].mean())
 train = train.fillna(train[features_label].mode().iloc[0])
 train.info()<categorify>
def compile_new_model(cfg, model):
with strategy.scope() :
model = get_model(cfg, model)
 losses = tf.keras.losses.BinaryCrossentropy(label_smoothing = cfg['label_smooth_fac'])
 model.compile(
 optimizer = cfg['optimizer'],
loss = losses,
metrics = [tf.keras.metrics.AUC(name='auc')])
 return model
test = pd.concat([test.drop(['Sex'],axis=1), pd.get_dummies(test[['Sex']], drop_first=True)], axis=1)
 train = pd.concat([train.drop(['Sex'],axis=1), pd.get_dummies(train[['Sex']], drop_first=True)], axis=1)
 test = pd.concat([test.drop(['Embarked'],axis=1), pd.get_dummies(test[['Embarked']], drop_first=True)], axis=1)
 train = pd.concat([train.drop(['Embarked'],axis=1), pd.get_dummies(train[['Embarked']], drop_first=True)], axis=1)
 test['Family'] = test['SibSp'] + test['Parch']
test['Alone'] = 0
test.loc[test.Family==0,'Alone'] = 1
test.drop(['SibSp','Parch'],axis=1,inplace=True)
 train['Family'] = train['SibSp'] + train['Parch']
train['Alone'] = 0
train.loc[train.Family==0,'Alone'] = 1
train.drop(['SibSp','Parch'],axis=1,inplace=True)
 train.head()<normalization>
ds_train = get_dataset(files_train, CFG, augment=True, shuffle=True, repeat=True)
 ds_train = ds_train.map(lambda img, label:(img, tuple([label])))
 steps_train = count_data_items(files_train)/(CFG['batch_size'] * REPLICAS)
 model_B6 = compile_new_model(CFG, 'EfficientNetB6')
 model_B7 = compile_new_model(CFG, 'EfficientNetB7')
 print("
Begin Training Models")
 Done Training model_B5
")
 history_B6 = model_B6.fit(ds_train,
verbose = 1,
steps_per_epoch = steps_train,
epochs = CFG['epochs'],
callbacks = [get_lr_callback(CFG)])
 print("
Done Training model_B6
")
 history_B7 = model_B7.fit(ds_train,
verbose = 1,
steps_per_epoch = steps_train,
epochs = CFG['epochs'],
callbacks = [get_lr_callback(CFG)])
 print("
Done Training model_B7
" )
<split>
CFG['batch_size'] = 256
cnt_test = count_data_items(files_test)
 steps = cnt_test /(CFG['batch_size'] * REPLICAS)* CFG['tta_steps']
ds_testAug = get_dataset(files_test, CFG, augment=True, repeat=True,
labeled=False, return_image_names=False)
 preds_B6 = model_B6.predict(ds_testAug, verbose=1, steps=steps)
 preds_B7 = model_B7.predict(ds_testAug, verbose=1, steps=steps )
X,y = train.drop(['Survived'],axis=1), train['Survived']
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
 print(X.shape)
 print(X_train.shape)
 print(X_test.shape)
 print(y.shape)
 print(y_train.shape)
 print(y_test.shape )<choose_model_class>
preds = np.stack(preds_B6)
 preds = preds[:,:cnt_test* CFG['tta_steps']]
preds = preds[:df_test.shape[0]*CFG['tta_steps']]
preds = np.stack(np.split(preds, CFG['tta_steps']),axis=1)
 preds = np.mean(preds, axis=1)
 preds_B6 = preds.reshape(-1 )
classifiers = [
KNeighborsClassifier(3), SVC(probability=True), DecisionTreeClassifier() , RandomForestClassifier() , AdaBoostClassifier() ,
GradientBoostingClassifier() , GaussianNB() , LinearDiscriminantAnalysis() , QuadraticDiscriminantAnalysis() , LogisticRegression() ,
XGBClassifier()
]
log_cols = ["Classifier", "Accuracy"]
log = pd.DataFrame(columns=log_cols)
 acc_dict = {}
for clf in classifiers:
name = clf.__class__.__name__
clf.fit(X_train, y_train)
 train_predictions = clf.predict(X_test)
 acc = accuracy_score(y_test, train_predictions)
 if name in acc_dict:
acc_dict[name] += acc
else:
acc_dict[name] = acc
for clf in acc_dict:
acc_dict[clf] = acc_dict[clf]
log_entry = pd.DataFrame([[clf, acc_dict[clf]]], columns=log_cols)
 log = log.append(log_entry)
 plt.figure(figsize=(10,10))
plt.xlabel('Accuracy')
 plt.title('Classifier Accuracy')
 plt.grid(True)
 sns.barplot(x='Accuracy', y='Classifier', data=log, color="b");<train_model>
preds = np.stack(preds_B7)
 preds = preds[:,:cnt_test* CFG['tta_steps']]
preds = preds[:df_test.shape[0]*CFG['tta_steps']]
preds = np.stack(np.split(preds, CFG['tta_steps']),axis=1)
 preds = np.mean(preds, axis=1)
 preds_B7 = preds.reshape(-1 )
xgb = XGBClassifier()
xgb.fit(X_train, y_train, early_stopping_rounds=50, eval_metric='auc', eval_set=[(X_train, y_train),(X_test, y_test)] )<predict_on_test>
preds = preds_B6*0.6 + preds_B7*0.4
survivors = xgb.predict(test)
 submission = pd.DataFrame({'PassengerId':test.index,'Survived':survivors})
 submission.head()<save_to_csv>
ds = get_dataset(files_test, CFG, augment=False, repeat=False,
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )
<set_options><EOS> 
submission = pd.DataFrame(dict(
 image_name = image_names,
target = preds))
submission = submission.sort_values('image_name')
 submission.to_csv(f'submission_EfficientNet_B7_Blend_6.csv', index=False)
 submission.head()