Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<save_to_csv>
train = pd.read_csv('.. /input/siim-isic-melanoma-classification/train.csv')
 print('Examples WITH Melanoma')
 imgs = train.loc[train.target==1].sample(10 ).image_name.values
plt.figure(figsize=(20,8))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()
print('Examples WITHOUT Melanoma')
 imgs = train.loc[train.target==0].sample(10 ).image_name.values
plt.figure(figsize=(20,8))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()
model = LogisticRegression(solver = 'liblinear')
 model.fit(X_train,y_train)
 predictions = model.predict(X_test)
 confusion_matrix(y_test,predictions)
 TestForPred = all_test.drop(['PassengerId', 'Survived'], axis = 1)
 t_pred = model.predict(TestForPred ).astype(int)
 PassengerId = all_test['PassengerId']
sub = pd.DataFrame({'PassengerId': PassengerId, 'Survived':t_pred })
 sub.to_csv("Submission.csv", index = False )<import_modules>
!pip install -q efficientnet >> /dev/null
warnings.filterwarnings('ignore' )<load_from_csv>
import pandas as pd, numpy as np
from kaggle_datasets import KaggleDatasets
import tensorflow as tf, re, math
import tensorflow.keras.backend as K
import efficientnet.tfkeras as efn
from sklearn.model_selection import KFold
from sklearn.metrics import roc_auc_score
import matplotlib.pyplot as plt
train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 print(train.shape)
 print(test.shape )<feature_engineering>
DEVICE = "TPU"
SEED = 42
FOLDS = 5
IMG_SIZES = [384,384,384,384,384]
INC2019 = [0]*FOLDS
INC2018 = [1]*FOLDS
BATCH_SIZES = [32]*FOLDS
EPOCHS = [12]*FOLDS
EFF_NETS = [6,6,6,6,6]
WGTS = [1/FOLDS]*FOLDS
TTA = 20
train = train[train.GrLivArea < 4500]
train.reset_index(drop = True, inplace = True)
 train['SalePrice'] = np.log1p(train['SalePrice'] )<prepare_x_and_y>
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )
train.drop('Id', axis=1, inplace=True)
 test.drop('Id', axis=1, inplace=True)
 y = train['SalePrice'].reset_index(drop=True )<create_dataframe>
GCS_PATH = [None]*FOLDS; GCS_PATH2 = [None]*FOLDS
for i,k in enumerate(IMG_SIZES):
GCS_PATH[i] = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(k,k))
GCS_PATH2[i] = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(k,k))
files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/train*.tfrec')))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec')) )
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data.drop('SalePrice', axis=1, inplace=True )<define_variables>
ROT_ = 180.0
SHR_ = 2.0
HZOOM_ = 8.0
WZOOM_ = 8.0
HSHIFT_ = 8.0
WSHIFT_ = 8.0
missing_val_col = ["Alley",
"PoolQC",
"MiscFeature",
"Fence",
"FireplaceQu",
"GarageType",
"GarageFinish",
"GarageQual",
"GarageCond",
'BsmtQual',
'BsmtCond',
'BsmtExposure',
'BsmtFinType1',
'BsmtFinType2',
'MasVnrType']
for c in missing_val_col:
all_data[c] = all_data[c].fillna('None' )<define_variables>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, DIM=256):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )
missing_val_col2 = ['BsmtFinSF1',
'BsmtFinSF2',
'BsmtUnfSF',
'TotalBsmtSF',
'BsmtFullBath',
'BsmtHalfBath',
'GarageYrBlt',
'GarageArea',
'GarageCars',
'MasVnrArea']
for i in missing_val_col2:
all_data[i] = all_data[i].fillna(0)
 all_data['LotFrontage'] = all_data.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.mean()))<data_type_conversions>
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img, augment=True, dim=256):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img,DIM=dim)
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 img = tf.reshape(img, [dim,dim, 3])
 return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )
all_data['MSSubClass'] = all_data['MSSubClass'].astype(str)
 all_data['MSZoning'] = all_data.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
all_data['Functional'] = all_data['Functional'].fillna('Typ')
 all_data['Utilities'] = all_data['Utilities'].fillna('AllPub')
 all_data['KitchenQual'] = all_data['KitchenQual'].fillna("TA")
 all_data['Electrical'] = all_data['Electrical'].fillna("SBrkr")
 all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0])
 all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0] )<feature_engineering>
def get_dataset(files, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True, batch_size=16, dim=256):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, imgname_or_label:(prepare_image(img, augment=augment, dim=dim),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(batch_size * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds
all_data = fixing_skewness(all_data)
 all_data = all_data.drop(['Utilities', 'Street', 'PoolQC',], axis=1)
 all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
all_data['YrBltAndRemod']=all_data['YearBuilt']+all_data['YearRemodAdd']
all_data['Total_sqr_footage'] =(all_data['BsmtFinSF1'] + all_data['BsmtFinSF2'] +
all_data['1stFlrSF'] + all_data['2ndFlrSF'])
 all_data['Total_Bathrooms'] =(all_data['FullBath'] +(0.5 * all_data['HalfBath'])+
all_data['BsmtFullBath'] +(0.5 * all_data['BsmtHalfBath']))
all_data['Total_porch_sf'] =(all_data['OpenPorchSF'] + all_data['3SsnPorch'] +
all_data['EnclosedPorch'] + all_data['ScreenPorch'] +
all_data['WoodDeckSF'])
 all_data['haspool'] = all_data['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 all_data['has2ndfloor'] = all_data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasgarage'] = all_data['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasbsmt'] = all_data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasfireplace'] = all_data['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )<categorify>
EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3,
efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6]
def build_model(dim=128, ef=0):
inp = tf.keras.layers.Input(shape=(dim,dim,3))
base = EFNS[ef](input_shape=(dim,dim,3),weights='imagenet',include_top=False)
 x = base(inp)
 x = tf.keras.layers.GlobalAveragePooling2D()(x)
 x = tf.keras.layers.Dense(1,activation='sigmoid' )(x)
 model = tf.keras.Model(inputs=inp,outputs=x)
 opt = tf.keras.optimizers.Adam(learning_rate=0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.01)
 model.compile(optimizer=opt,loss=loss,metrics=['AUC'])
 return model
all_data = pd.get_dummies(all_data ).reset_index(drop = True)
 all_data.shape<prepare_x_and_y>
def get_lr_callback(batch_size=8):
lr_start = 0.000005
lr_max = 0.00000125 * REPLICAS * batch_size
lr_min = 0.000001
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.8
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback
X = all_data.iloc[:len(y), :]
X_sub = all_data.iloc[len(y):, :]<drop_column>
VERBOSE = 0
DISPLAY_PLOT = True
skf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)
 oof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = []
preds = np.zeros(( count_data_items(files_test),1))
for fold,(idxT,idxV)in enumerate(skf.split(np.arange(15))):
if DEVICE=='TPU':
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 print('
print(' 
(IMG_SIZES[fold],EFF_NETS[fold],BATCH_SIZES[fold]*REPLICAS))
files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxT])
 if INC2019[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1])
 print('
if INC2018[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2])
 print('
np.random.shuffle(files_train); print('
files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxV])
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/test*.tfrec')))
 K.clear_session()
with strategy.scope() :
model = build_model(dim=IMG_SIZES[fold],ef=EFF_NETS[fold])
 sv = tf.keras.callbacks.ModelCheckpoint(
 'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min', save_freq='epoch')
 print('Training...')
 history = model.fit(
 get_dataset(files_train, augment=True, shuffle=True, repeat=True,
dim=IMG_SIZES[fold], batch_size = BATCH_SIZES[fold]),
epochs=EPOCHS[fold], callbacks = [sv,get_lr_callback(BATCH_SIZES[fold])],
steps_per_epoch=count_data_items(files_train)/BATCH_SIZES[fold]//REPLICAS,
validation_data=get_dataset(files_valid,augment=False,shuffle=False,
repeat=False,dim=IMG_SIZES[fold]),
verbose=VERBOSE 
)
 print('Loading best model...')
 model.load_weights('fold-%i.h5'%fold)
 print('Predicting OOF with TTA...')
 ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_valid = count_data_items(files_valid); STEPS = TTA * ct_valid/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,]
oof_pred.append(np.mean(pred.reshape(( ct_valid,TTA),order='F'),axis=1))
ds_valid = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=True, return_image_names=True)
 oof_tar.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]))
oof_folds.append(np.ones_like(oof_tar[-1],dtype='int8')*fold)
 ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 oof_names.append(np.array([img_name.numpy().decode("utf-8")for img, img_name in iter(ds.unbatch())]))
print('Predicting Test with TTA...')
 ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,]
preds[:,0] += np.mean(pred.reshape(( ct_test,TTA),order='F'),axis=1)* WGTS[fold]
auc = roc_auc_score(oof_tar[-1],oof_pred[-1])
 oof_val.append(np.max(history.history['val_auc']))
print('
if DISPLAY_PLOT:
plt.figure(figsize=(15,5))
plt.plot(np.arange(EPOCHS[fold]),history.history['auc'],'-o',label='Train AUC',color='
plt.plot(np.arange(EPOCHS[fold]),history.history['val_auc'],'-o',label='Val AUC',color='
x = np.argmax(history.history['val_auc']); y = np.max(history.history['val_auc'])
 xdist = plt.xlim() [1] - plt.xlim() [0]; ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
%.2f'%y,size=14)
 plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)
 plt.legend(loc=2)
 plt2 = plt.gca().twinx()
plt2.plot(np.arange(EPOCHS[fold]),history.history['loss'],'-o',label='Train Loss',color='
plt2.plot(np.arange(EPOCHS[fold]),history.history['val_loss'],'-o',label='Val Loss',color='
x = np.argmin(history.history['val_loss']); y = np.min(history.history['val_loss'])
 ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
plt.ylabel('Loss',size=14)
 plt.title('FOLD %i - Image Size %i, EfficientNet B%i, inc2019=%i, inc2018=%i'% 
(fold+1,IMG_SIZES[fold],EFF_NETS[fold],INC2019[fold],INC2018[fold]),size=18)
 plt.legend(loc=3)
 plt.show()
outliers = [30, 88, 462, 631, 1322]
X = X.drop(X.index[outliers])
 y = y.drop(y.index[outliers] )<drop_column>
oof = np.concatenate(oof_pred); true = np.concatenate(oof_tar);
names = np.concatenate(oof_names); folds = np.concatenate(oof_folds)
 auc = roc_auc_score(true,oof)
 print('Overall OOF AUC with TTA = %.3f'%auc)
 df_oof = pd.DataFrame(dict(
 image_name = names, target=true, pred = oof, fold=folds))
df_oof.to_csv('oof.csv',index=False)
 df_oof.head()
overfitted_features = overfit_reducer(X)
 X = X.drop(overfitted_features, axis=1)
 X_sub = X_sub.drop(overfitted_features, axis=1 )<compute_train_metric>
ds = get_dataset(files_test, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )
<define_search_space><EOS> 
submission = pd.DataFrame(dict(image_name=image_names, target=preds[:,0]))
submission = submission.sort_values('image_name')
 submission.to_csv('submission.csv', index=False)
 submission.head()