Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<categorify>
import numpy as np
import pandas as pd
import os
import re
import tensorflow.keras.backend as K
sex_mapping = {"male": 0, "female": 1}
for datatotal in datostotales:
datatotal['Sex'] = datatotal['Sex'].map(sex_mapping )<categorify>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )
train["Age"].fillna(train.groupby("Title")["Age"].transform("median"), inplace=True)
 test["Age"].fillna(test.groupby("Title")["Age"].transform("median"), inplace=True )<feature_engineering>
df_train = pd.read_csv("/kaggle/input/siim-isic-melanoma-classification/train.csv")
 df_test = pd.read_csv("/kaggle/input/siim-isic-melanoma-classification/test.csv")
 sample_sub = pd.read_csv("/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv" )
for datatotal in datostotales:
datatotal.loc[ datatotal['Age'] <= 15, 'Age'] = 0,
datatotal.loc[(datatotal['Age'] > 15)&(datatotal['Age'] <= 35), 'Age'] = 1,
datatotal.loc[(datatotal['Age'] > 35)&(datatotal['Age'] <= 55), 'Age'] = 2,
datatotal.loc[(datatotal['Age'] > 55)&(datatotal['Age'] <= 69), 'Age'] = 3,
datatotal.loc[ datatotal['Age'] > 69, 'Age'] = 4<count_values>
FOLDS = 5
IMG_SIZE = 512
BATCH_SIZE = 32
EPOCHS = 12
Pclass1 = train[train['Pclass']==1]['Embarked'].value_counts()
Pclass2 = train[train['Pclass']==2]['Embarked'].value_counts()
Pclass3 = train[train['Pclass']==3]['Embarked'].value_counts()
df = pd.DataFrame([Pclass1, Pclass2, Pclass3])
 df.index = ['1st class','2nd class', '3rd class']
df.plot(kind='bar',stacked=True, figsize=(10,5))<data_type_conversions>
GCS_PATH2 = KaggleDatasets().get_gcs_path("isic2019-512x512")
 GCS_PATH1 = KaggleDatasets().get_gcs_path("melanoma-512x512" )
for datatotal in datostotales:
datatotal['Embarked'] = datatotal['Embarked'].fillna('Q' )<categorify>
train_filenames1 = tf.io.gfile.glob(GCS_PATH1 + '/train*.tfrec')
 train_filenames2 = tf.io.gfile.glob(GCS_PATH2 + '/train*.tfrec')
 test_filenames = tf.io.gfile.glob(GCS_PATH1 + '/test*.tfrec')
 data_filenames = train_filenames1 + train_filenames2
embarked_mapping = {"S": 0, "C": 1, "Q": 2}
for datatotal in datostotales:
datatotal['Embarked'] = datatotal['Embarked'].map(embarked_mapping )<categorify>
train_filenames, valid_filenames = train_test_split(data_filenames, test_size=0.2, random_state=0, shuffle=True )
train["Fare"].fillna(train.groupby("Pclass")["Fare"].transform("median"), inplace=True)
 test["Fare"].fillna(test.groupby("Pclass")["Fare"].transform("median"), inplace=True )<feature_engineering>
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = example['image']
label = example['target']
return image, label
def read_unlabeled_tfrecord(example, return_image_name=True):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = example['image']
idnum = example['image_name']
return image, idnum if return_image_name else 0
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
for datatotal in datostotales:
datatotal.loc[(datatotal['Fare'] <= 30), 'Fare'] = 0,
datatotal.loc[(datatotal['Fare'] > 30)&(datatotal['Fare'] <= 100), 'Fare'] = 1,
datatotal.loc[(datatotal['Fare'] > 30)&(datatotal['Fare'] <= 100), 'Fare'] = 2,
datatotal.loc[(datatotal['Fare'] > 100), 'Fare'] = 3<count_values>
def decode_image(image_data, augment=False):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
if augment:
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 image = tf.image.random_saturation(image, 0, 2)
 image = tf.image.rot90(image)
 image = tf.reshape(image, [IMG_SIZE,IMG_SIZE, 3])
 return image
def get_training_dataset() :
dataset = tf.data.TFRecordDataset(train_filenames, num_parallel_reads=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(1024*8)
 dataset = dataset.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 dataset = dataset.map(lambda img, imgname_or_label:(decode_image(img, augment=True), imgname_or_label), num_parallel_calls=AUTO)
 dataset = dataset.batch(BATCH_SIZE * REPLICAS)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_val_dataset() :
dataset = tf.data.TFRecordDataset(valid_filenames, num_parallel_reads=AUTO)
 dataset = dataset.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 dataset = dataset.map(lambda img, imgname_or_label:(decode_image(img, augment=False), imgname_or_label), num_parallel_calls=AUTO)
 dataset = dataset.batch(BATCH_SIZE * REPLICAS)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = tf.data.TFRecordDataset(test_filenames, num_parallel_reads=AUTO)
 dataset = dataset.repeat()
dataset = dataset.map(lambda example: read_unlabeled_tfrecord(example, False), num_parallel_calls=AUTO)
 dataset = dataset.map(lambda img, imgname_or_label:(decode_image(img, augment=False), imgname_or_label), num_parallel_calls=AUTO)
 dataset = dataset.batch(BATCH_SIZE * REPLICAS)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n )
train.Cabin.value_counts()<feature_engineering>
def get_lr_callback(batch_size=8):
lr_start = 0.000005
lr_max = 0.00000125 * REPLICAS * batch_size
lr_min = 0.000001
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.8
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback
for datatotal in datostotales:
datatotal['Cabin'] = datatotal['Cabin'].str[:1]<categorify>
def build_model(dim=512):
inp = tf.keras.layers.Input(shape=(dim,dim,3))
base = efn.EfficientNetB0(input_shape=(dim,dim,3),weights='imagenet',include_top=False)
 x = base(inp)
 x = tf.keras.layers.GlobalAveragePooling2D()(x)
 x = tf.keras.layers.Dense(1,activation='sigmoid' )(x)
 model = tf.keras.Model(inputs=inp,outputs=x)
 opt = tf.keras.optimizers.Adam(learning_rate=0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05)
 model.compile(optimizer=opt,loss=loss,metrics=['AUC'])
 return model
cabin_mapping = {"A": 0, "B": 0.4, "C": 0.8, "D": 1.2, "E": 1.6, "F": 2, "G": 2.4, "T": 2.8}
for datatotal in datostotales:
datatotal['Cabin'] = datatotal['Cabin'].map(cabin_mapping )<categorify>
!pip install yapl==0.1.2 efficientnet > /dev/null
train["Cabin"].fillna(train.groupby("Pclass")["Cabin"].transform("median"), inplace=True)
 test["Cabin"].fillna(test.groupby("Pclass")["Cabin"].transform("median"), inplace=True )<feature_engineering>
input_shape =(512, 512, 3)
 def create_model() :
model = tf.keras.Sequential([
efn.EfficientNetB5(
 input_shape=input_shape,
weights='imagenet',
include_top=False 
),
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(1, activation='sigmoid')
 ])
 return model
train["FamilySize"] = train["SibSp"] + train["Parch"] + 1
test["FamilySize"] = test["SibSp"] + test["Parch"] + 1<categorify>
callbacks = [get_lr_callback(BATCH_SIZE)]
step_per_epoch = count_data_items(train_filenames)/BATCH_SIZE//REPLICAS
family_size = {1: 0, 2: 0.4, 3: 0.8, 4: 1.2, 5: 1.6, 6: 2, 7: 2.4, 8: 2.8, 9: 3.2, 10: 3.6, 11: 4}
for datatotal in datostotales:
datatotal['FamilySize'] = datatotal['FamilySize'].map(family_size )<drop_column>
test_dataset = get_test_dataset(ordered=True )
features_drop = ['Ticket', 'SibSp', 'Parch']
train = train.drop(features_drop, axis=1)
 test = test.drop(features_drop, axis=1)
 train = train.drop(['PassengerId'], axis=1 )<prepare_x_and_y>
def get_dataset(filename, shuffle=False, repeat=False, return_img_name=False, augment=False, labeled=False, batch_size=32, dim=512):
dataset = tf.data.TFRecordDataset(filename, num_parallel_reads=AUTO)
 if repeat:
dataset = dataset.repeat()
if shuffle:
dataset = dataset.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
dataset = dataset.with_options(opt)
 if labeled:
dataset = dataset.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
dataset = dataset.map(lambda example: read_unlabeled_tfrecord(example, return_img_name), num_parallel_calls=AUTO)
 dataset = dataset.map(lambda img, imgname_or_label:(decode_image(img, augment=augment), imgname_or_label), num_parallel_calls=AUTO)
 dataset = dataset.batch(batch_size * REPLICAS)
 dataset = dataset.prefetch(AUTO)
 return dataset
train_dfX = train.drop('Survived', axis=1)
 train_dfY = train['Survived']
submission = test[['PassengerId']].copy()
test_df = test.drop(['PassengerId'], axis=1 )<categorify>
FOLDS = 3
IMG_SIZE = 512
BATCH_SIZE = 32
EPOCHS = 12
weights = 1/6
categorical = ['Embarked', 'Title', 'Pclass', 'Fare']
for var in categorical:
train_dfX = pd.concat([train_dfX, pd.get_dummies(train_dfX[var], prefix=var)], axis=1)
 del train_dfX[var]<categorify>
kfold = KFold(n_splits=FOLDS, shuffle=True, random_state=0 )
categorical = ['Embarked', 'Title', 'Pclass', 'Fare']
for var in categorical:
test_df = pd.concat([test_df, pd.get_dummies(test_df[var], prefix=var)], axis=1)
 del test_df[var]<normalization>
TTA = 11
preds = np.zeros(( count_data_items(test_filenames),1))
for fold,(train_file, valid_file)in enumerate(kfold.split(data_filenames)) :
print(fold)
 training_dataset = get_dataset([data_filenames[x] for x in train_file], augment=True, shuffle=True, repeat=True, labeled=True)
 valid_dataset = get_dataset([data_filenames[x] for x in valid_file], augment=True, shuffle=False, repeat=False, labeled=False)
 K.clear_session()
with strategy.scope() :
model = create_model()
if fold == 3:
print("ok")
 optimizer = tf.keras.optimizers.Adam(lr=0.001)
 model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])
 callbacks = [get_lr_callback(BATCH_SIZE)]
steps_per_epoch = count_data_items([data_filenames[x] for x in train_file])/BATCH_SIZE//REPLICAS
sv = tf.keras.callbacks.ModelCheckpoint('fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True, save_weights_only=True, mode='min', save_freq='epoch')
 callbacks.append(sv)
 warmup_history = model.fit(training_dataset, steps_per_epoch=steps_per_epoch, validation_data=valid_dataset, epochs=12, verbose=1, callbacks=callbacks ).history
if fold == 3:
print("whatup")
 test_dataset = get_dataset(test_filenames, augment=True, repeat=True, batch_size=BATCH_SIZE*4)
 ct_valid = count_data_items(test_filenames)
 if fold == 3:
print("hello")
 STEPS = TTA * ct_valid/32/4/REPLICAS
pred = model.predict(test_dataset,steps=STEPS,verbose=1)[:TTA*ct_valid,]
preds[:, 0] += np.mean(pred.reshape(( ct_valid,TTA),order='F'),axis=1 )
sc = StandardScaler()
train_dfX = sc.fit_transform(train_dfX)
 test_df = sc.transform(test_df)
 print("Test shape : ",test_df.shape )<split>
preds = np.concatenate(preds )
train_dfX,val_dfX,train_dfY, val_dfY = train_test_split(train_dfX,train_dfY , test_size=0.10, stratify=train_dfY)
 print("Tamaño set de Entrenamiento: ",train_dfX.shape)
 print("Tamaño set de Validacion : ",val_dfX.shape )<choose_model_class>
preds = preds/6
def func_model() :
inp = Input(shape=(17,))
x=Dropout(0.1 )(inp)
 x=Dense(350, activation="relu", kernel_regularizer=regularizers.l2(0.01))(inp)
 x=Dropout(0.50 )(x)
 x=Dense(350, activation="relu", kernel_regularizer=regularizers.l2(0.01))(x)
 x=Dropout(0.50 )(x)
 x=Dense(350, activation="relu", kernel_regularizer=regularizers.l2(0.01))(x)
 x=Dropout(0.30 )(x)
 x=Dense(350, activation="relu", kernel_regularizer=regularizers.l2(0.01))(x)
 x=Dropout(0.30 )(x)
 x=Dense(350, activation="relu", kernel_regularizer=regularizers.l2(0.01))(x)
 x=Dropout(0 )(x)
 x=Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='Adam', metrics=['binary_accuracy'])
 return model<train_model>
ds = get_dataset(test_filenames, augment=False, repeat=False,
labeled=False, return_img_name=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )
train_history = model.fit(train_dfX, train_dfY, batch_size=64, epochs=epochs, validation_data=(val_dfX, val_dfY))<train_model>
submission = pd.DataFrame(dict(image_name=list(image_names), target=preds))
submission = submission.sort_values('image_name')
 submission = submission.set_index("image_name")
 submission.to_csv('submission.csv')
 submission.head()
print("Tiempo de ejecución %s segundos" %(time.time() - start_time))<save_to_csv>
<feature_engineering><EOS> 
np.zeros(( count_data_items(test_filenames),1))