Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<count_values>
base_dir = '/kaggle/input/siim-isic-melanoma-classification/'
external_base_dir = '/kaggle/input/melanoma-external-malignant-256/'
external_tfrec_base_dir = '.. /input/melanoma-256x256'
sns.set(style="darkgrid")
 random.seed(42)
df["titles"].replace(["Ms.","Jonkheer.","the","Don.","Capt.","Sir.","Lady.","Mme.","Col.","Major."],"sometitle", inplace = True)
 test_df["titles"].replace(["Ms.","Jonkheer.","the","Don.","Capt.","Sir.","Lady.","Mme.","Col.","Major."],"sometitle", inplace = True)
 df["titles"].replace("Mlle.","Miss.", inplace = True)
 test_df["titles"].replace("Mlle.","Miss.", inplace = True)
 test_df["titles"].value_counts()<categorify>
DEVICE = 'GPU'
df["titles"].replace(["Mr.", "Miss.", "Mrs", "Master.", "Dr.", "Rev.", "sometitle"],[0,1,2,3,4,5,6], inplace = True)
 df["titles"].astype("int64")
 test_df["titles"].replace(["Mr.", "Miss.", "Mrs", "Master.", "Dr.", "Rev.", "sometitle"],[0,1,2,3,4,5,6], inplace = True)
 test_df["titles"].astype("int64")
 df.drop(["Name"], axis = 1, inplace = True)
 test_df.drop(["Name"], axis = 1, inplace = True )<drop_column>
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )
df.drop(["SibSp","Parch","Fare","n_fam_mem","actual_fare"], axis = 1, inplace = True)
 test_df.drop(["SibSp","Parch","Fare","n_fam_mem","actual_fare"], axis = 1, inplace = True )<split>
IMG_HEIGHT = 256
IMG_WIDTH = 256
N_CHANNELS = 3
epochs = 20
BATCH_SIZE = 8 * REPLICAS
IMAGE_SIZE = [IMG_HEIGHT, IMG_WIDTH]
IMAGE_RESIZE = [IMG_HEIGHT, IMG_WIDTH]
input_shape =(IMG_HEIGHT, IMG_WIDTH, N_CHANNELS)
 BALANCE_DATA = True
aug_data = True
NETWORK_MODEL = 'DenseNet201'
labels = df["Survived"]
data = df.drop("Survived", axis = 1)
 X_train, X_test, Y_train, Y_test = train_test_split(data, labels, test_size = 0.1 )<define_variables>
if DEVICE == 'TPU':
files_dir = KaggleDatasets().get_gcs_path('melanoma-256x256')
 else:
files_dir = external_tfrec_base_dir
final_clf = None
clf_names = ["Logistic Regression", "KNN(3)", "KNN(5)", "Random forest classifier", "Decision Tree Classifier",
"Gradient Boosting Classifier", "Support Vector Machine"]
classifiers = []
scores = []<train_on_grid>
TRAINING_FILENAMES, VALID_FILENAMES = train_test_split(
 tf.io.gfile.glob(files_dir + '/train*.tfrec'),
test_size=0.1, random_state=42 
)
 TEST_FILENAMES = tf.io.gfile.glob(files_dir + '/test*.tfrec')
 print('Train TFRecord Files:', len(TRAINING_FILENAMES))
print('Validation TFRecord Files:', len(VALID_FILENAMES))
print('Test TFRecord Files:', len(TEST_FILENAMES))
submission_example = pd.read_csv(base_dir + 'sample_submission.csv' )
bestknn5 = None
bestknn3 = None
bestrf = None
bestgb = None
bestcvm = None
bestlr = None
bestdt = None
for i in range(10):
X_train, X_test, Y_train, Y_test = train_test_split(data, labels, test_size = 0.1)
 tempscores = []
lr_clf = LogisticRegression()
lr_clf.fit(X_train, Y_train)
 tempscores.append(( lr_clf.score(X_test, Y_test)) *100)
 if lr_clf.score(X_test, Y_test)*100 > 85:
bestlr = lr_clf
knn3_clf = KNeighborsClassifier()
knn3_clf.fit(X_train, Y_train)
 tempscores.append(( knn3_clf.score(X_test, Y_test)) *100)
 if knn3_clf.score(X_test, Y_test)*100 > 85:
bestknn3 = knn3_clf
knn5_clf = KNeighborsClassifier()
knn5_clf.fit(X_train, Y_train)
 tempscores.append(( knn5_clf.score(X_test, Y_test)) *100)
 if knn5_clf.score(X_test, Y_test)*100 > 85:
bestknn5 = knn5_clf
rf_clf = RandomForestClassifier(n_estimators = 100)
 rf_clf.fit(X_train, Y_train)
 tempscores.append(( rf_clf.score(X_test, Y_test)) *100)
 if rf_clf.score(X_test, Y_test)*100 > 85:
bestrf = rf_clf
dt_clf = DecisionTreeClassifier()
dt_clf.fit(X_train, Y_train)
 tempscores.append(( dt_clf.score(X_test, Y_test)) *100)
 if dt_clf.score(X_test, Y_test)*100 > 85:
bestdt = dt_clf
gb_clf = GradientBoostingClassifier()
gb_clf.fit(X_train, Y_train)
 tempscores.append(( gb_clf.score(X_test, Y_test)) *100)
 if gb_clf.score(X_test, Y_test)*100 > 85:
bestgb = lr_clf
svm_clf = SVC(gamma = "scale")
 svm_clf.fit(X_train, Y_train)
 tempscores.append(( svm_clf.score(X_test, Y_test)) *100)
 if svm_clf.score(X_test, Y_test)*100 > 85:
bestsvm = svm_clf
scores.append(tempscores )<feature_engineering>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))
scores = np.array(scores)
 clfs = pd.DataFrame({"Classifier":clf_names})
 clfs["iteration0"] = scores[0].T
clfs["iteration1"] = scores[1].T
clfs["iteration2"] = scores[2].T
clfs["iteration3"] = scores[3].T
clfs["iteration4"] = scores[4].T
clfs["iteration5"] = scores[5].T
clfs["iteration6"] = scores[6].T
clfs["iteration7"] = scores[7].T
clfs["iteration8"] = scores[8].T
clfs["iteration9"] = scores[9].T
means = clfs.mean(axis = 1)
 means = means.values.tolist()
clfs["Average"] = means<find_best_params>
def transform(image,label):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
rot = 180.* tf.random.normal([1],dtype='float32')
 shr = 2.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/8.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/8.
 h_shift = 8.* tf.random.normal([1],dtype='float32')
 w_shift = 8.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image,tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3]),label
final_clf = bestsvm<predict_on_test>
def decode_image(image):
image = tf.image.decode_jpeg(image, channels=3)
 image = tf.image.resize(image, [IMG_HEIGHT, IMG_WIDTH])
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [IMG_HEIGHT, IMG_WIDTH, 3])
 return image
def read_tfrecord(example, labeled):
tfrecord_format = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64)
 } if labeled else {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string)
 }
example = tf.io.parse_single_example(example, tfrecord_format)
 image = decode_image(example['image'])
 if labeled:
label = tf.cast(example['target'], tf.int32)
 return image, label
idnum = example['image_name']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTO)
 return dataset
def augmentation_pipeline(image, label):

 if aug_data:
image,_ = transform(image,label)
 image = tf.image.random_flip_left_right(image)
 image = tf.image.random_hue(image, 0.01)
 image = tf.image.random_saturation(image, 0.7, 1.3)
 image = tf.image.random_contrast(image, 0.8, 1.2)
 image = tf.image.random_brightness(image, 0.1)
 return image, label
def get_training_dataset() :
dataset = load_dataset(TRAINING_FILENAMES, labeled=True)
 dataset = dataset.map(augmentation_pipeline, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(ordered=False):
dataset = load_dataset(VALID_FILENAMES, labeled=True, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n )
test_data = test_df
predictions = final_clf.predict(test_data)
 print(len(predictions))<save_to_csv>
train_dataset = get_training_dataset()
valid_dataset = get_validation_dataset()
final_csv = []
csv_title = ['PassengerId', 'Survived']
final_csv.append(csv_title)
 for i in range(len(predictions)) :
passengerid = i + 892
survived = predictions[i]
temp = [passengerid, survived]
final_csv.append(temp)
 print(len(final_csv))
with open('submission_csv.csv', 'w')as file:
writer = csv.writer(file)
 writer.writerows(final_csv)
 file.close()<load_from_csv>
NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)
 NUM_VALIDATION_IMAGES = count_data_items(VALID_FILENAMES)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
VALID_STEPS = NUM_VALIDATION_IMAGES // BATCH_SIZE
print("Num of steps per epoch:", STEPS_PER_EPOCH)
 print("Num of steps per validation", VALID_STEPS)
 print(
 'Dataset: {} training images, {} validation images, {} unlabeled test images'.format(
 NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES 
)
 )
dataset=pd.read_csv('.. /input/train.csv' )<prepare_x_and_y>
def show_dataset(thumb_size, cols, rows, ds):
mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols +(cols-1),
thumb_size*rows +(rows-1)))
 for idx, data in enumerate(iter(ds)) :
img, target_or_imgid = data
ix = idx % cols
iy = idx // cols
img = np.clip(img.numpy() * 255, 0, 255 ).astype(np.uint8)
 img = PIL.Image.fromarray(img)
 img = img.resize(( thumb_size, thumb_size), resample=PIL.Image.BILINEAR)
 mosaic.paste(img,(ix*thumb_size + ix,
iy*thumb_size + iy))
display(mosaic)
 print("Training data examples...")
 ds = train_dataset.unbatch().take(12*5)
 show_dataset(64, 12, 5, ds)
 print("Validation data examples...")
 ds = valid_dataset.unbatch().take(12*5)
 show_dataset(64, 12, 5, ds )
y=dataset['label']<prepare_x_and_y>
train_df = pd.read_csv(base_dir + 'train.csv')
 y_train = train_df['target']
class_weights = class_weight.compute_class_weight('balanced',
classes=np.unique(y_train),
y=y_train)
 class_weights = {0: class_weights[0],1: class_weights[1]}
if not BALANCE_DATA:
class_weights = {0: 1,1: 2}
print(class_weights )
X_train=dataset.iloc[:,1:]<load_from_csv>
learning_rate_reduction =ReduceLROnPlateau(
 monitor='loss',
factor=0.25,
patience=2,
verbose=1,
mode="auto",
min_delta=0.0001,
cooldown=0,
min_lr=0.00001 
)
 es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=3, restore_best_weights=True )
X_test=pd.read_csv('.. /input/test.csv' )<feature_engineering>
def get_lr_callback() :
lr_start = 0.000003
lr_max = 0.000020 * REPLICAS
lr_min = 0.000001
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.8
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback
X_train=X_train/255.0
X_test=X_test/255.0<import_modules>
def get_model() :
model_input = tf.keras.Input(shape=input_shape, name='input_layer')
 dummy = tf.keras.layers.Lambda(lambda x:x )(model_input)
 outputs = []
x = DenseNet201(include_top=False, weights='imagenet',
input_shape=input_shape,
pooling='avg' )(dummy)
 x = tf.keras.layers.Dense(1, activation='sigmoid' )(x)
 outputs.append(x)
 model = tf.keras.Model(model_input, outputs, name='transfNetwork')
 model.summary()
return model
from keras.models import Sequential
from keras.layers import Dense,Dropout,Flatten,Conv2D,MaxPool2D
from keras.preprocessing.image import ImageDataGenerator
from keras.utils.np_utils import to_categorical
from sklearn.model_selection import train_test_split, cross_val_score
import itertools<categorify>
model = get_model()
model.compile(optimizer='adam', loss=BinaryCrossentropy(label_smoothing=0.05),
metrics=['AUC'])
 hist_train = model.fit(train_dataset,
validation_data=valid_dataset,
steps_per_epoch=STEPS_PER_EPOCH,
validation_steps=VALID_STEPS,
epochs=epochs,callbacks=[get_lr_callback() ], class_weight=class_weights )
target=to_categorical(y,10 )<split>
test_ds = get_test_dataset(ordered=True)
 test_images_ds = test_ds.map(lambda image, idnum: image )
X_t, X_v, Y_t, Y_v = train_test_split(X_train, target, test_size = 0.1 )<choose_model_class>
probabilities = model.predict(test_images_ds )
model=Sequential()
model.add(Conv2D(filters=64,kernel_size=(7,7),padding = 'Same',
activation ='relu', input_shape =(28,28,1)) )<choose_model_class>
test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U' )
model.add(Conv2D(filters=64,kernel_size=(7,7),padding = 'Same',
activation ='relu'))<choose_model_class>
pred_df = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(probabilities)})
 pred_df.head()
<choose_model_class><EOS> 
del submission_example['target']
submission_example = submission_example.merge(pred_df, on='image_name')
 submission_example.to_csv('submission_densenet.csv', index=False)
 submission_example.head()