Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<groupby>
!pip install -q efficientnet
!pip install -q git+https://github.com/AmedeoBiolatti/dsqol
train_test.groupby(['Initial','Pclass'])['Age'].aggregate(['mean','count'] )<count_values>
import os, re, time, tqdm
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import metrics, model_selection
import tensorflow as tf
import tensorflow_addons as tfa
from tensorflow import keras
from tensorflow.keras import backend as K
from efficientnet import tfkeras as efnet
from kaggle_datasets import KaggleDatasets
train_test.loc[(train_test.Age.isnull())]['Initial'].value_counts()<count_values>
from dsqol.tf import imgaug
from dsqol.tf.data import balance
from dsqol.tf.utils import average
from dsqol.tf import losses
train_test.loc[(train_test.Age.isnull())]['Pclass'].value_counts()<groupby>
SEED = 42
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 TIME_BUDGET = 2.5 * 3600
FOLDS = 6
INCLUDE_2019 = 1
INCLUDE_2018 = 1
INCLUDE_MALIGNANT = 1
IMG_READ_SIZE = 384
IMG_SIZE = 384
BALANCE_POS_RATIO = 0.08
EFF_NET = 1
LOSS_TYPE = 'BCE'
LOSS_PARAMS = dict(label_smoothing=0.09)
 BATCH_SIZE = 64
EPOCHS = 25
TBM = 6
TTA = 15
VALID_FREQ = 1
N_SWA = 3
SWA_DECAY = 0.9
train_test.loc[(train_test.Age.isnull())].groupby(['Initial','Pclass'])['Name'].aggregate('count' )<categorify>
DEVICE = "TPU"
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
GLOBAL_BATCH_SIZE = BATCH_SIZE * REPLICAS
print("REPLICAS: %d" % REPLICAS )
imp = Imputer(missing_values='NaN', strategy='mean', axis=0)
 temp_train_test = pd.get_dummies(train_test[['Initial','Pclass','Age']])
 train_test_age_filled = imp.fit_transform(temp_train_test )<create_dataframe>
GCS_PATH1 = KaggleDatasets().get_gcs_path('melanoma-%ix%i' %(IMG_READ_SIZE, IMG_READ_SIZE))
GCS_PATH2 = KaggleDatasets().get_gcs_path('isic2019-%ix%i' %(IMG_READ_SIZE, IMG_READ_SIZE))
GCS_PATH3 = KaggleDatasets().get_gcs_path('malignant-v2-%ix%i' %(IMG_READ_SIZE, IMG_READ_SIZE))
train_test_age_filled = pd.DataFrame(train_test_age_filled,columns=temp_train_test.columns.tolist() )<feature_engineering>
df_base_train = pd.read_csv(".. /input/siim-isic-melanoma-classification/train.csv")
 df_base_test = pd.read_csv(".. /input/siim-isic-melanoma-classification/test.csv" )
train_test.Age = train_test_age_filled.Age<filter>
train_files = tf.io.gfile.glob(os.path.join(GCS_PATH1, "train*.tfrec"))
if INCLUDE_2019:
train_files += tf.io.gfile.glob([os.path.join(GCS_PATH2, "train%.2i*.tfrec" % i)for i in range(1, 30, 2)])
 if INCLUDE_2018:
train_files += tf.io.gfile.glob([os.path.join(GCS_PATH2, "train%.2i*.tfrec" % i)for i in range(0, 30, 2)])
 if INCLUDE_MALIGNANT:
train_files += tf.io.gfile.glob([os.path.join(GCS_PATH3, "train%.2i*.tfrec" % i)for i in range(15, 30, 1)])
 print("%d train files found" % len(train_files))
train_test[np.isnan(train_test['Fare'])]<groupby>
test_files = tf.io.gfile.glob(os.path.join(GCS_PATH1, "test*.tfrec"))
print("%d test files found" % len(test_files))
train_test.groupby(['Pclass','Parch','SibSp'])['Fare'].mean()<feature_engineering>
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name=True):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.cast(img, tf.float32)/ 255.0
return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )
train_test.loc[(train_test.Fare.isnull()),'Fare']=9<filter>
def dropout(image, DIM=256, PROBABILITY = 0.75, CT = 8, SZ = 0.2):
P = tf.cast(tf.random.uniform([],0,1)<PROBABILITY, tf.int32)
 if(P==0)|(CT==0)|(SZ==0):
return image
for k in range(CT):
x = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 y = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 WIDTH = tf.cast(SZ*DIM,tf.int32)* P
ya = tf.math.maximum(0,y-WIDTH//2)
 yb = tf.math.minimum(DIM,y+WIDTH//2)
 xa = tf.math.maximum(0,x-WIDTH//2)
 xb = tf.math.minimum(DIM,x+WIDTH//2)
 one = image[ya:yb,0:xa,:]
two = tf.zeros([yb-ya,xb-xa,3])
 three = image[ya:yb,xb:DIM,:]
middle = tf.concat([one,two,three],axis=1)
 image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM,:,:]],axis=0)
 image = tf.reshape(image,[DIM,DIM,3])
 return image
train_test.loc[train_test.Embarked.isnull() ]<groupby>
def get_dataset(files, augment=False, repeat=False, shuffle=False, labeled=True, batch_size=16, drop_remainder=False,
dim=256, read_dim=None 
)-> tf.data.Dataset:
if read_dim is None:
read_dim = dim
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024 * 8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 ds = ds.map(lambda i, o:(prepare_image(i), o), num_parallel_calls=AUTO)
 if augment:
ds = basic_augmentation_pipeline(ds, batch_size=8 * batch_size, dim=read_dim)
 if isinstance(augment, list):
for a in augment:
ds = ds.map(a, num_parallel_calls=AUTO)
 ds = ds.map(lambda i, o:(tf.image.resize(i, [dim, dim]), o), num_parallel_calls=AUTO)
 ds = ds.batch(batch_size, drop_remainder=drop_remainder)
 ds = ds.prefetch(AUTO)
 return ds
def get_balanced_dataset(files, augment=False, repeat=False, shuffle=False, batch_size=16, drop_remainder=False,
dim=256, read_dim=None, pos_ratio=False)-> tf.data.Dataset:
if read_dim is None:
read_dim = dim
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 ds0, ds1 = balance.separate_by_target(ds)
 ds0 = ds.map(lambda i, o:(prepare_image(i), o), num_parallel_calls=AUTO)
 ds1 = ds.map(lambda i, o:(prepare_image(i), o), num_parallel_calls=AUTO)
 ds = balance.merge_ds(ds0, ds1, pos_ratio)
 if shuffle:
ds = ds.shuffle(1024)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if augment:
ds = basic_augmentation_pipeline(ds, batch_size=8 * batch_size, dim=read_dim)
 if isinstance(augment, list):
for a in augment:
ds = ds.map(a, num_parallel_calls=AUTO)
 ds = ds.map(lambda i, o:(tf.image.resize(i, [dim, dim]), o), num_parallel_calls=AUTO)
 ds = ds.batch(batch_size, drop_remainder=drop_remainder)
 ds = ds.prefetch(AUTO)
 return ds
train_test.groupby(['Embarked','Pclass'])['Fare'].mean()<feature_engineering>
def build_model(dim=128, ef=0):
inp = keras.layers.Input(shape=(dim,dim,3))
base = getattr(efnet, 'EfficientNetB%d' % ef )(input_shape=(dim, dim, 3), weights='imagenet', include_top=False)
 x = base(inp)
 x = keras.layers.GlobalAveragePooling2D()(x)
 x = keras.layers.Dropout(0.3 )(x)
 x = keras.layers.Dense(512 )(x)
 x = keras.layers.Dropout(0.4 )(x)
 x = keras.layers.Dense(256 )(x)
 x = keras.layers.Dropout(0.3 )(x)
 x = keras.layers.Dense(128 )(x)
 x = keras.layers.Dense(1 )(x)
 x = keras.layers.Activation('sigmoid', dtype='float32' )(x)
 model = keras.Model(inputs=inp,outputs=x)
 opt = keras.optimizers.Adam(learning_rate=1e-3)
 if LOSS_TYPE.upper() == 'BCE':
loss = keras.losses.BinaryCrossentropy(**LOSS_PARAMS)
 elif LOSS_TYPE.upper() == 'FOCAL':
loss = losses.BinaryFocalLoss(**LOSS_PARAMS)
 model.compile(optimizer=opt, loss=loss, metrics=['AUC'])
 return model
train_test["Embarked"] = train_test["Embarked"].fillna("S" )<categorify>
mult = 1
lr_start = 6e-6
lr_max = 1.45e-6 * GLOBAL_BATCH_SIZE
lr_min = 1e-6
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.85
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr * mult
plt.figure(figsize=(10, 5))
plt.subplot(1, 2, 1)
 plt.plot([lrfn(e)for e in range(EPOCHS)])
 plt.xlabel('Epoch'); plt.ylabel('LR')
 plt.subplot(1, 2, 2)
 plt.plot([lrfn(e)for e in range(EPOCHS)])
 plt.xlabel('Epoch'); plt.ylabel('Log LR')
 plt.yscale('log');
train_temp = pd.get_dummies(train['Cabin'])
 res = dict(zip(train_temp.columns.tolist() ,
mutual_info_classif(train_temp, train['Survived'], discrete_features=True)
))
print(res )<drop_column>
train_test.drop('Cabin',axis=1,inplace=True )<filter>
xxx = df_oof.groupby('image_name' ).mean().reset_index().merge(df_base_train, on='image_name')
 print("OOF AUC(TTA %d)= %.4f" %(TTA, metrics.roc_auc_score(xxx.target, xxx.pred)) )
<drop_column><EOS>