Kaggle competition: APTOS 2019 Blindness Detection
sub.to_csv('submission.csv', index = False )<install_modules>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
%matplotlib inline
package_path = '.. /input/efficientnet-pytorch/efficientnet-pytorch/EfficientNet-PyTorch-master'
sys.path.append(package_path)
!pip install -q efficientnet<import_modules>
md_ef = EfficientNet.from_pretrained('efficientnet-b5', num_classes=1 )
import os
import re
import numpy as np
import pandas as pd
import math
from matplotlib import pyplot as plt
from sklearn import metrics
from sklearn.model_selection import train_test_split
import tensorflow as tf
import tensorflow.keras.layers as L
import efficientnet.tfkeras as efn
from kaggle_datasets import KaggleDatasets<define_variables>
class ImagePreprocessor(object):
def __init__(self, root_dir: str, save_dir: str, img_size: int, tolerance: int = 10, remove_outer_pixels: float = 0.0):

 if remove_outer_pixels > 0.50:
print("ERROR: eroding more than 50% of image")
 raise InterruptedError
self.root_dir = root_dir
self.img_size = img_size
self.tolerance = tolerance
self.remove_outer_pixels = remove_outer_pixels
self.images = glob.glob(f"{self.root_dir}/*.png")+ glob.glob(
 f"{self.root_dir}/*.jpeg")+ glob.glob(f"{self.root_dir}/*.jpg")
 self.save_dir = save_dir
os.makedirs(self.save_dir, exist_ok=True)
 self.total_count = len(self.images)
 @staticmethod
def _counter(func):
def wrapper(self, *args, **kwargs):
func(self, *args, **kwargs)
 return wrapper
@staticmethod
def light_sensitivity_reducer(img: np.ndarray, alpha: int = 4, beta: int = -4, gamma: int = 128):

 return cv2.addWeighted(img, alpha, cv2.GaussianBlur(img,(0, 0), 30), beta, gamma)
 @staticmethod
def outer_pixels_remover(img: np.ndarray, scale: float):

 scale_2 = scale / 2.0
miny = int(img.shape[0]*scale_2)
 maxy = int(img.shape[0]-miny)
 minx = int(img.shape[1]*scale_2)
 maxx = int(img.shape[1]-minx)
 return img[miny:maxy, minx:maxx]
@staticmethod
def scale_image(img: np.ndarray, img_size: int):

 return cv2.resize(img,(img_size, img_size))
@staticmethod
def pitch_black_remover(img: np.ndarray, tolerance: int = 10):

 if img.ndim == 2:
img_mask = img > tolerance
return img[np.ix_(img_mask.any(1), img_mask.any(0)) ]
elif img.ndim == 3:
greyed = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
 img_mask = greyed > tolerance
img_1 = img[:, :, 0][np.ix_(img_mask.any(1), img_mask.any(0)) ]
if img_1.shape[0] == 0:
return img
img_2 = img[:, :, 1][np.ix_(img_mask.any(1), img_mask.any(0)) ]
img_3 = img[:, :, 2][np.ix_(img_mask.any(1), img_mask.any(0)) ]
return np.stack([img_1, img_2, img_3], axis=-1)
 else:
print("Image has more than 3 dimensions")
 raise InterruptedError
def replace_existing(self):
shutil.rmtree(self.root_dir)
 os.makedirs(self.root_dir)
 processed_images = [i.path for i in os.scandir(
 self.save_dir)if i.is_file() ]
processor_pool = multiprocessing.Pool(64)
 for counter, _ in enumerate(processor_pool.imap_unordered(lambda x: shutil.move(x, self.root_dir), processed_images), 1):
sys.stdout.write(
 f"\rMoving : {(counter/self.total_count)*100:3.2f}% \t[ {counter}/{self.total_count} ]")
 os.rmdir(self.save_dir)
 sys.stdout.write("
")
 def forward(self, image: str):

 img = cv2.imread(image)
 img = self.pitch_black_remover(img, tolerance=self.tolerance)
 img = self.scale_image(img, img_size=self.img_size)
 if self.remove_outer_pixels > 0.0:
img = self.outer_pixels_remover(img, self.remove_outer_pixels)
 cv2.imwrite(os.path.join(self.save_dir, image.split('/')[-1]), img)
 def run(self, replace: bool = False):

 for image in tqdm(self.images):
self.forward(image)
 if replace:
self.replace_existing()
def parallel_run(self, workers: int = multiprocessing.cpu_count() , replace: bool = False):

 processor_pool = multiprocessing.Pool(workers)
 for counter, _ in enumerate(processor_pool.imap_unordered(self.forward, self.images), 1):
sys.stdout.write(
 f"\rProgress : {(counter/self.total_count)*100:3.2f}% \t[ {counter}/{self.total_count} ]")
 if replace:
self.replace_existing()
sys.stdout.write("
")
 preprocessor = ImagePreprocessor(root_dir='.. /input/aptos2019-blindness-detection/test_images/',
save_dir='test_processed',
img_size=256,
tolerance=10,
remove_outer_pixels=0 
)
 preprocessor.parallel_run(multiprocessing.cpu_count() ,False )
TRAINING = False<set_options>
def get_df() :
base_image_dir = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 train_dir = os.path.join(base_image_dir,'train_images/')
 df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 return df, test_df
df, test_df = get_df()
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
bs = 64
sz = 224
tfms = get_transforms(do_flip=True,flip_vert=True )
AUTO = tf.data.experimental.AUTOTUNE
GCS_PATH = KaggleDatasets().get_gcs_path('siim-isic-melanoma-classification')
 EPOCHS = 5
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
IMAGE_SIZE = [1024, 1024]<concatenate>
def qk(y_pred, y):
return torch.tensor(cohen_kappa_score(torch.round(y_pred), y, weights='quadratic'), device='cuda:0' )
def append_path(pre):
return np.vectorize(lambda file: os.path.join(GCS_DS_PATH, pre, file))<load_from_csv>
learn = Learner(data,
md_ef,
metrics = [qk],
model_dir="models")
 learn.data.add_test(ImageList.from_df(test_df,
'test_processed/',
folder='',
suffix='.png', 
))
learn.load('abcdef')
 pass
sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/tfrecords/train*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/tfrecords/test*.tfrec' )<split>
learn.to_fp32()
pass
VALIDATION_FILENAMES = TRAINING_FILENAMES[int(0.8*len(TRAINING_FILENAMES)) :]
TRAINING_FILENAMES = TRAINING_FILENAMES[:int(0.8*len(TRAINING_FILENAMES)) ]<categorify>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >=coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
y = y.cpu().apply_(lambda x: 0 if x==0 else 1 if x==0.25 else 2 if x==0.5 else 3 if x==0.75 else 4)
 loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.125,0.375,0.625,0.875]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
opt = OptimizedRounder()
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['target'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['image_name']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def data_augment(image, label):
image = tf.image.random_flip_left_right(image)
 return image, label
def get_training_dataset() :
dataset = load_dataset(TRAINING_FILENAMES, labeled=True)
 dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(ordered=False):
dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_TEST_IMAGES))<choose_model_class>
learn.dl(DatasetType.Test ).dataset.tfms=[]
op = _TTA(learn,ds_type=DatasetType.Test)
 op_1 = [2*p-4 for p in op]
def build_lrfn(lr_start=0.00001, lr_max=0.000075,
lr_min=0.000001, lr_rampup_epochs=20,
lr_sustain_epochs=0, lr_exp_decay=.8):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn<choose_model_class>
!cp.. /input/l1-aptos/L1.pkl export.pkl
learn = load_learner('',test=ImageList.from_df(test_df,
'test_processed/',
folder='',
suffix='.png', 
))
learn.dl(DatasetType.Test ).dataset.tfms=[]
op_2 = _TTA(learn,ds_type=DatasetType.Test )
if TRAINING:
with strategy.scope() :
model = tf.keras.Sequential([
efn.EfficientNetB7(
 input_shape=(*IMAGE_SIZE, 3),
weights='imagenet',
include_top=False 
),
L.GlobalAveragePooling2D() ,
L.Dense(1, activation='sigmoid')
 ])
 model.compile(
 optimizer='adam',
loss = 'binary_crossentropy',
metrics=['accuracy'] 
)
 model.summary()<choose_model_class>
p = [0] * len(op_1)
 for i in range(len(op_1)) :
p[i] =(op_1[i]+op_2[i])/2
if TRAINING:
with strategy.scope() :
model2 = tf.keras.Sequential([
efn.EfficientNetB0(
 input_shape=(*IMAGE_SIZE, 3),
weights='imagenet',
include_top=False 
),
L.GlobalAveragePooling2D() ,
L.Dense(1, activation='sigmoid')
 ])
 model2.compile(
 optimizer='adam',
loss = 'binary_crossentropy',
metrics=['accuracy'] 
)
 model2.summary()<choose_model_class>
opt = OptimizedRounder()
op = 0.3*p[0]+0.2*p[1]+0.3*p[2]+0.2*p[3]
op_preds = torch.Tensor(opt.predict(op,[-2.8031, -1.0877, 1.02 , 2.92]))
print(type(op_preds))
test_df.diagnosis = op_preds.numpy().astype(int)
 test_df.to_csv('submission.csv',index=False )
<find_best_params><EOS> 
test_df.groupby('diagnosis' ).size()