Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<normalization>
warnings.filterwarnings('ignore' )
scaler = StandardScaler()
scaler.fit(df_train_ml.drop(['Survived'],axis=1))
scaled_features = scaler.transform(df_train_ml.drop(['Survived'],axis=1))
df_train_ml_sc = pd.DataFrame(scaled_features)
 df_test_ml.fillna(df_test_ml.mean() , inplace=True)
 scaled_features = scaler.transform(df_test_ml)
 df_test_ml_sc = pd.DataFrame(scaled_features )<prepare_x_and_y>
train = pd.read_csv("/kaggle/input/siim-isic-melanoma-classification/train.csv")
 test = pd.read_csv("/kaggle/input/siim-isic-melanoma-classification/test.csv")
 sample = pd.read_csv("/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv" )
X = df_train_ml.drop('Survived', axis=1)
 y = df_train_ml['Survived']
X_test = df_test_ml
X_sc = df_train_ml_sc
y_sc = df_train_ml['Survived']
X_test_sc = df_test_ml_sc<import_modules>
!pip install -q efficientnet
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn import tree
from sklearn.metrics import accuracy_score 
<import_modules>
import tensorflow as tf
import tensorflow.keras.backend as K
import efficientnet.tfkeras as efn
from kaggle_datasets import KaggleDatasets
from sklearn.model_selection import cross_val_score<compute_train_metric>
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-512x512')
 GCS_PATH2 = KaggleDatasets().get_gcs_path('malignant-v2-512x512')
 GCS_PATH3 = KaggleDatasets().get_gcs_path('isic2019-512x512')
 filenames_train1 = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')
 filenames_train2 = tf.io.gfile.glob(GCS_PATH2 + '/train%.2i*.tfrec'%(2*x)for x in range(25))
filenames_train3 = tf.io.gfile.glob(GCS_PATH3 + '/train%.2i*.tfrec'%(2*x)for x in range(25))
filenames_test = np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec'))
svc = SVC(gamma = 0.01, C = 100)
 scores_svc = cross_val_score(svc, X, y, cv=10, scoring='accuracy')
 print(scores_svc)
 print(scores_svc.mean() )<compute_train_metric>
filenames_train = np.array(filenames_train1+filenames_train2+filenames_train3)
 np.random.shuffle(filenames_train)
 np.random.shuffle(filenames_train )
svc = SVC(gamma = 0.01, C = 100)
 scores_svc_sc = cross_val_score(svc, X_sc, y_sc, cv=10, scoring='accuracy')
 print(scores_svc_sc)
 print(scores_svc_sc.mean() )<compute_train_metric>
AUTO = tf.data.experimental.AUTOTUNE
rfc = RandomForestClassifier(max_depth=5, max_features=6)
 scores_rfc = cross_val_score(rfc, X, y, cv=10, scoring='accuracy')
 print(scores_rfc)
 print(scores_rfc.mean() )<import_modules>
cfg = dict(
 batch_size=32,
img_size=512,
lr_start=0.000005,
lr_max=0.00000125,
lr_min=0.000001,
lr_rampup=5,
lr_sustain=0,
lr_decay=0.8,
epochs=12,
transform_prob=1.0,
rot=180.0,
shr=2.0,
hzoom=8.0,
wzoom=8.0,
hshift=8.0,
wshift=8.0,
optimizer='adam',
label_smooth_fac=0.05,
tta_steps=20
)
from sklearn.model_selection import RandomizedSearchCV
from sklearn.model_selection import GridSearchCV
from scipy.stats import uniform<train_on_grid>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1], dtype='float32')
 zero = tf.constant([0], dtype='float32')
 rotation_matrix = tf.reshape(
 tf.concat([c1, s1, zero, -s1, c1, zero, zero, zero, one], axis=0),
[3, 3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(
 tf.concat([one, s2, zero, zero, c2, zero, zero, zero, one], axis=0),
[3, 3])
 zoom_matrix = tf.reshape(
 tf.concat([
one / height_zoom, zero, zero, zero, one / width_zoom, zero, zero,
zero, one
],axis=0), [3, 3])
 shift_matrix = tf.reshape(
 tf.concat(
 [one, zero, height_shift, zero, one, width_shift, zero, zero, one],
axis=0), [3, 3])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
model = SVC()
param_grid = {'C':uniform(0.1, 5000), 'gamma':uniform(0.0001, 1)}
rand_SVC = RandomizedSearchCV(model, param_distributions=param_grid, n_iter=100)
 rand_SVC.fit(X_sc,y_sc)
 score_rand_SVC = get_best_score(rand_SVC )<train_on_grid>
def transform(image, cfg):
DIM = cfg['img_size']
XDIM = DIM % 2
rot = cfg['rot'] * tf.random.normal([1], dtype='float32')
 shr = cfg['shr'] * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['hzoom']
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['wzoom']
h_shift = cfg['hshift'] * tf.random.normal([1], dtype='float32')
 w_shift = cfg['wshift'] * tf.random.normal([1], dtype='float32')
 m = get_mat(rot, shr, h_zoom, w_zoom, h_shift, w_shift)
 x = tf.repeat(tf.range(DIM // 2, -DIM // 2, -1), DIM)
 y = tf.tile(tf.range(-DIM // 2, DIM // 2), [DIM])
 z = tf.ones([DIM * DIM], dtype='int32')
 idx = tf.stack([x, y, z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM // 2 + XDIM + 1, DIM // 2)
 idx3 = tf.stack([DIM // 2 - idx2[0, ], DIM // 2 - 1 + idx2[1, ]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d, [DIM, DIM, 3] )
param_grid = {'C': [0.1,10, 100, 1000,5000], 'gamma': [1,0.1,0.01,0.001,0.0001], 'kernel': ['rbf']}
svc_grid = GridSearchCV(SVC() , param_grid, cv=10, refit=True, verbose=1)
 svc_grid.fit(X_sc,y_sc)
 sc_svc = get_best_score(svc_grid )<save_to_csv>
def dropout(image, DIM=512, PROBABILITY = 0.75, CT = 8, SZ = 0.2):
P = tf.cast(tf.random.uniform([],0,1)<PROBABILITY, tf.int32)
 if(P==0)|(CT==0)|(SZ==0): return image
for k in range(CT):
x = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 y = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 WIDTH = tf.cast(SZ*DIM,tf.int32)* P
ya = tf.math.maximum(0,y-WIDTH//2)
 yb = tf.math.minimum(DIM,y+WIDTH//2)
 xa = tf.math.maximum(0,x-WIDTH//2)
 xb = tf.math.minimum(DIM,x+WIDTH//2)
 one = image[ya:yb,0:xa,:]
two = tf.zeros([yb-ya,xb-xa,3])
 three = image[ya:yb,xb:DIM,:]
middle = tf.concat([one,two,three],axis=1)
 image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM,:,:]],axis=0)
 image = tf.reshape(image,[DIM,DIM,3])
 return image
pred_all_svc = svc_grid.predict(X_test_sc)
 sub_svc = pd.DataFrame()
sub_svc['PassengerId'] = df_test['PassengerId']
sub_svc['Survived'] = pred_all_svc
sub_svc.to_csv('svc.csv',index=False )<train_on_grid>
def prepare_image(img, cfg=None,droprate=0.5,dropct=8,dropsize=0.2):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.image.resize(img, [cfg['img_size'], cfg['img_size']],
antialias=True)
 img = tf.cast(img, tf.float32)/ 255.0
if cfg['transform_prob'] > tf.random.uniform([1], minval=0, maxval=1):
img = transform(img, cfg)
 if(tf.random.uniform([1], minval=0, maxval=1)> 0.5)&(droprate!=0)&(dropct!=0)&(dropsize!=0):
img = dropout(img, DIM=512, PROBABILITY=droprate, CT=dropct, SZ=dropsize)
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 return img
knn = KNeighborsClassifier()
leaf_range = list(range(3, 15, 1))
k_range = list(range(1, 15, 1))
weight_options = ['uniform', 'distance']
param_grid = dict(leaf_size=leaf_range, n_neighbors=k_range, weights=weight_options)
 print(param_grid)
 knn_grid = GridSearchCV(knn, param_grid, cv=10, verbose=1, scoring='accuracy')
 knn_grid.fit(X, y)
 sc_knn = get_best_score(knn_grid )<save_to_csv>
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
'image': tf.io.FixedLenFeature([], tf.string),
'image_name': tf.io.FixedLenFeature([], tf.string),
'target': tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 return example['image'], example['target']
pred_all_knn = knn_grid.predict(X_test)
 sub_knn = pd.DataFrame()
sub_knn['PassengerId'] = df_test['PassengerId']
sub_knn['Survived'] = pred_all_knn
sub_knn.to_csv('knn.csv',index=False )<train_on_grid>
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
'image': tf.io.FixedLenFeature([], tf.string),
'image_name': tf.io.FixedLenFeature([], tf.string)
 }
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 return example['image'], example['image_name']
dtree = DecisionTreeClassifier()
param_grid = {'min_samples_split': [4,7,10,12]}
dtree_grid = GridSearchCV(dtree, param_grid, cv=10, refit=True, verbose=1)
 dtree_grid.fit(X_sc,y_sc)
 print(dtree_grid.best_score_)
 print(dtree_grid.best_params_)
 print(dtree_grid.best_estimator_ )<save_to_csv>
def count_data_items(filenames):
n = [
int(re.compile(r'-([0-9]*)\.' ).search(filename ).group(1))
for filename in filenames
]
return np.sum(n )
pred_all_dtree = dtree_grid.predict(X_test_sc)
 sub_dtree = pd.DataFrame()
sub_dtree['PassengerId'] = df_test['PassengerId']
sub_dtree['Survived'] = pred_all_dtree
sub_dtree.to_csv('dtree.csv',index=False )<train_on_grid>
def getTrainDataset(files, cfg):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 ds = ds.repeat()
ds = ds.shuffle(2048)
 ds = ds.map(lambda img, label: 
(prepare_image(img, cfg=cfg), label),
num_parallel_calls=AUTO)
 ds = ds.batch(cfg['batch_size'] * strategy.num_replicas_in_sync)
 ds = ds.prefetch(AUTO)
 return ds
rfc = RandomForestClassifier()
param_grid = {'max_depth': [3, 5, 6, 7, 8], 'max_features': [6,7,8,9,10],
'min_samples_split': [5, 6, 7, 8]}
rf_grid = GridSearchCV(rfc, param_grid, cv=10, refit=True, verbose=1)
 rf_grid.fit(X_sc,y_sc)
 sc_rf = get_best_score(rf_grid )<save_to_csv>
def getTestDataset(files, cfg, augment=False, repeat=False):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
ds = ds.map(read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 ds = ds.map(lambda img, idnum: 
(prepare_image(img, cfg=cfg), idnum),
num_parallel_calls=AUTO)
 ds = ds.batch(cfg['batch_size'] * strategy.num_replicas_in_sync)
 ds = ds.prefetch(AUTO)
 return ds
pred_all_rf = rf_grid.predict(X_test_sc)
 sub_rf = pd.DataFrame()
sub_rf['PassengerId'] = df_test['PassengerId']
sub_rf['Survived'] = pred_all_rf
sub_rf.to_csv('rf.csv',index=False )<train_on_grid>
def getLearnRateCallback(cfg):
lr_start = cfg['lr_start']
lr_max = cfg['lr_max'] * strategy.num_replicas_in_sync * cfg['batch_size']
lr_min = cfg['lr_min']
lr_rampup = cfg['lr_rampup']
lr_sustain = cfg['lr_sustain']
lr_decay = cfg['lr_decay']
def lrfn(epoch):
if epoch < lr_rampup:
lr =(lr_max - lr_start)/ lr_rampup * epoch + lr_start
elif epoch < lr_rampup + lr_sustain:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_rampup -
lr_sustain)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback
extr = ExtraTreesClassifier()
param_grid = {'max_depth': [6,7,8,9], 'max_features': [7,8,9,10],
'n_estimators': [50, 100, 200]}
extr_grid = GridSearchCV(extr, param_grid, cv=10, refit=True, verbose=1)
 extr_grid.fit(X_sc,y_sc)
 sc_extr = get_best_score(extr_grid )<save_to_csv>
with strategy.scope() :
model_input = tf.keras.Input(shape=(cfg['img_size'], cfg['img_size'], 3),
name='img_input')
 dummy = tf.keras.layers.Lambda(lambda x: x )(model_input)
 outputs = []
x = efn.EfficientNetB3(include_top=False,
weights='imagenet',
input_shape=(cfg['img_size'], cfg['img_size'], 3),
pooling='avg' )(dummy)
 x = tf.keras.layers.Dense(1, activation='sigmoid' )(x)
 outputs.append(x)
 x = efn.EfficientNetB4(include_top=False,
weights='imagenet',
input_shape=(cfg['img_size'], cfg['img_size'], 3),
pooling='avg' )(dummy)
 x = tf.keras.layers.Dense(1, activation='sigmoid' )(x)
 outputs.append(x)
 x = efn.EfficientNetB5(include_top=False,
weights='imagenet',
input_shape=(cfg['img_size'], cfg['img_size'], 3),
pooling='avg' )(dummy)
 x = tf.keras.layers.Dense(1, activation='sigmoid' )(x)
 outputs.append(x)
 model = tf.keras.Model(model_input, outputs, name='aNetwork')
 model.compile(optimizer=cfg['optimizer'],
loss=[
tf.keras.losses.BinaryCrossentropy(
 label_smoothing=cfg['label_smooth_fac']),
tf.keras.losses.BinaryCrossentropy(
 label_smoothing=cfg['label_smooth_fac']),
tf.keras.losses.BinaryCrossentropy(
 label_smoothing=cfg['label_smooth_fac'])
 ],
metrics=[tf.keras.metrics.AUC(name='auc')])
pred_all_extr = extr_grid.predict(X_test_sc)
 sub_extr = pd.DataFrame()
sub_extr['PassengerId'] = df_test['PassengerId']
sub_extr['Survived'] = pred_all_extr
sub_extr.to_csv('extr.csv',index=False )<train_on_grid>
ds_train = getTrainDataset(filenames_train, cfg ).map(lambda img, label:(img,(label, label, label)))
 stepsTrain = count_data_items(filenames_train)/(cfg['batch_size'] * strategy.num_replicas_in_sync )
gbc = GradientBoostingClassifier()
param_grid = {'n_estimators': [50, 100],
'min_samples_split': [3, 4, 5, 6, 7],
'max_depth': [3, 4, 5, 6]}
gbc_grid = GridSearchCV(gbc, param_grid, cv=10, refit=True, verbose=1)
 gbc_grid.fit(X_sc,y_sc)
 sc_gbc = get_best_score(gbc_grid )<save_to_csv>
history = model.fit(ds_train,
validation_data = None,
verbose=1,
steps_per_epoch = stepsTrain,
validation_steps = 0,
epochs=14,
callbacks=callbacks )
pred_all_gbc = gbc_grid.predict(X_test_sc)
 sub_gbc = pd.DataFrame()
sub_gbc['PassengerId'] = df_test['PassengerId']
sub_gbc['Survived'] = pred_all_gbc
sub_gbc.to_csv('gbc.csv',index=False )<train_on_grid>
steps = count_data_items(filenames_test)/(cfg['batch_size'] * strategy.num_replicas_in_sync)
 z = np.zeros(( cfg['batch_size'] * strategy.num_replicas_in_sync))
ds_testAug = getTestDataset(filenames_test, cfg, augment=True,
repeat=True ).map(lambda img, label:(img,(z, z, z)))
xgb = XGBClassifier()
param_grid = {'max_depth': [5,6,7,8], 'gamma': [1, 2, 4], 'learning_rate': [0.1, 0.2, 0.3, 0.5]}
with ignore_warnings(category=DeprecationWarning):
xgb_grid = GridSearchCV(xgb, param_grid, cv=10, refit=True, verbose=1)
 xgb_grid.fit(X_sc,y_sc)
 sc_xgb = get_best_score(xgb_grid )<save_to_csv>
probs = model.predict(ds_testAug, verbose=1, steps=steps * cfg['tta_steps'])
 probs = np.stack(probs)
 probs = probs[:, :count_data_items(filenames_test)* cfg['tta_steps']]
probs = np.stack(np.split(probs, cfg['tta_steps'], axis=1), axis=1)
 probs = np.mean(probs, axis=1 )
with ignore_warnings(category=DeprecationWarning):
pred_all_xgb = xgb_grid.predict(X_test_sc)
 sub_xgb = pd.DataFrame()
sub_xgb['PassengerId'] = df_test['PassengerId']
sub_xgb['Survived'] = pred_all_xgb
sub_xgb.to_csv('xgb.csv',index=False )<train_on_grid>
y_test_sorted = np.zeros(( 3, probs.shape[1]))
test = test.reset_index()
test = test.set_index('image_name')
 i = 0
ds_test = getTestDataset(filenames_test, cfg)
 for img, imgid in tqdm(iter(ds_test.unbatch())) :
imgid = imgid.numpy().decode('utf-8')
 y_test_sorted[:, test.loc[imgid]['index']] = probs[:, i, 0]
i += 1
for i in range(y_test_sorted.shape[0]):
submission = sample
submission['target'] = y_test_sorted[i]
submission.to_csv('submission_model_%s.csv' % i, index=False)
 submission = sample
submission['target'] = np.mean(y_test_sorted, axis=0)
 submission.to_csv('blended_effnets.csv', index=False )
<save_to_csv><EOS> 
!gzip submission_model_0.csv
!gzip submission_model_1.csv
!gzip submission_model_2.csv
!gzip blended_effnets.csv