Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<define_variables>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import time as t
import cv2
import warnings
import shutil
import os
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from sklearn.utils import shuffle
from keras.applications.resnet50 import ResNet50
from keras.applications import DenseNet121
from keras.utils import Sequence
from keras.layers import Dense, Dropout, Flatten, Input, ZeroPadding2D, GlobalAveragePooling2D
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D
from keras.layers.normalization import BatchNormalization
from keras import regularizers
from keras.models import Model
from keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint
from keras import optimizers
from keras.utils import to_categorical
from keras.preprocessing.image import ImageDataGenerator
from keras import backend as K
import tensorflow as tf
all_data = all_data[all_data['date_block_num'] >= 12]
shift_range= [1, 2, 3, 4, 5, 6,9, 12]
fit_cols = [col for col in all_data.columns if col[-1] in [str(item)for item in shift_range]]
for col in all_data.columns:
if "mean" in col:
fit_cols.append(col)
 for col in all_data.columns:
if "tfidf" in col:
fit_cols.append(col)
 to_drop_cols = list(set(list(all_data.columns)) -(set(fit_cols)|set(index_cols)))+ ['date_block_num' , 'shop_id' , 'item_id' ]
gc.collect() ;<define_variables>
PATH = '.. /input/'
PATH_APTOS = PATH + 'aptos2019-blindness-detection/'
PATH_2015 = PATH + 'retinopathy-train-2015/rescaled_train_896/'
PATH_KERAS = PATH + 'keras-pretrained-models/'
densenet_weights_path = PATH + 'densenet121full/densenet121_weights_tf_dim_ordering_tf_kernels_notop.h5'
PATH_AUGM = '/data_augm/'
HEIGHT, WIDTH = 256, 256
RANDOM_STATE = 974
VERBOSE = True
BATCH_SIZE = 32
DATA_AUGM = True
DATA_AUGM_FACTOR = 1
warnings.filterwarnings('ignore')
 print(os.listdir(PATH_KERAS))
dates = all_data['date_block_num']
last_block = dates.max()
print('Test `date_block_num` is %d' % last_block )<prepare_x_and_y>
df_aptos = pd.read_csv(PATH_APTOS + 'train.csv')
 df_aptos.rename(index=str, columns={"id_code": "id_code_aptos"}, inplace=True )
dates = all_data['date_block_num']
print('Test `date_block_num` is %d' % test_block)
 dates_train = dates[dates < test_block-1]
dates_val = dates[dates == test_block-1]
dates_test = dates[dates == test_block]
X_train = all_data.loc[dates < test_block-1].drop(to_drop_cols, axis=1)
 X_test = all_data.loc[dates == test_block].drop(to_drop_cols, axis=1)
 X_val = all_data.loc[dates == test_block-1].drop(to_drop_cols, axis=1)
 y_train = all_data.loc[dates < test_block-1, 'target'].values.clip(0,40)
 y_test = all_data.loc[dates == test_block, 'target'].values.clip(0,40)
 y_val = all_data.loc[dates == test_block-1, 'target'].values.clip(0,40)
 target_range=(0,20)
 X_cv_train = all_data.loc[(( dates>28)&(dates<32)) ].drop(to_drop_cols, axis=1)
 y_cv_train = all_data.loc[(( dates>28)&(dates<32)) , 'target'].values.clip(0,40)
<count_missing_values>
df_2015 = pd.read_csv(PATH_2015 + 'trainLabels.csv')
 df_2015.rename(index=str, columns={"image": "id_code_2015", "level": "diagnosis"}, inplace=True)
 df_2015 = df_2015.drop(df_2015[df_2015['diagnosis'] == 0].sample(frac=0.8 ).index )
X_train.isna().sum().sum()<count_missing_values>
class RetinaGenerator() :
def __init__(self, image_df, batch_size, train=True):
self.image_df = image_df
self.batch_size = batch_size
self.train = train
self.step_per_epoch = len(self.image_df)// self.batch_size
self.step_per_pred = 1 +(len(self.image_df)- 1)// self.batch_size
def getGenerator(self):
while True:
for idx in range(self.step_per_epoch):
batch_x = np.array([imagePreprocessing(openImage(row)) for row in self.image_df[idx * self.batch_size:(idx + 1)* self.batch_size].itertuples() ])
 batch_y_cat= to_categorical([row.diagnosis for row in self.image_df[idx * self.batch_size:(idx + 1)* self.batch_size].itertuples() ], num_classes=5)
 batch_y = np.empty(batch_y_cat.shape, dtype=batch_y_cat.dtype)
 batch_y[:, 4] = batch_y_cat[:, 4]
for i in range(3, -1, -1):
batch_y[:, i] = np.logical_or(batch_y_cat[:, i], batch_y[:, i+1])
 yield batch_x, batch_y
def getInputGenerator(self):
for idx in range(self.step_per_pred + 1):
yield np.array([imagePreprocessing(openImage(row, self.train)) for row in self.image_df[idx * self.batch_size:min(( idx + 1)* self.batch_size, len(self.image_df)) ].itertuples() ] )
X_test.isna().sum().sum()<train_on_grid>
if DATA_AUGM:
shutil.rmtree(PATH_AUGM, ignore_errors=True, onerror=None)
 shutil.os.mkdir(PATH_AUGM)
 df = shuffle(df, random_state=RANDOM_STATE)
 max_size = int(df.diagnosis.value_counts().max() * DATA_AUGM_FACTOR)
 for diag in range(5):
shutil.rmtree('{}diag_{}'.format(PATH_AUGM, diag), ignore_errors=True, onerror=None)
 shutil.os.mkdir('{}diag_{}'.format(PATH_AUGM, diag))
diag_df = df[df.diagnosis == diag]
size = len(diag_df)
 to_create = max_size - size
augm_per_img = max(to_create // size, 1)
 while to_create > 0:
for row in diag_df.itertuples() :
if to_create < 0:
break
image = np.expand_dims(cv2.cvtColor(openImage(row, resize=True), cv2.COLOR_BGR2RGB), 0)
 data_generator = ImageDataGenerator(rotation_range=360, vertical_flip=True, horizontal_flip=True, zoom_range=0.1)
 data_generator.fit(image)
 id_code = row.id_code_aptos if not pd.isnull(row.id_code_aptos)else row.id_code_2015
for x, val in zip(data_generator.flow(image, save_to_dir='{}diag_{}'.format(PATH_AUGM, diag),
save_prefix=id_code, save_format='png'),
range(augm_per_img - 1)) :
pass
to_create -= augm_per_img
for diag in range(5):
images = np.array(os.listdir("{}diag_{}".format(PATH_AUGM, diag)))
 for image in images:
df = df.append(pd.DataFrame([[diag, np.nan, np.nan, "{}diag_{}/{}".format(PATH_AUGM, diag, image)]], columns=df.columns))
df = shuffle(df, random_state=RANDOM_STATE)
 display(df.head() )
print(' Training a plain linear regression model ')
 gc.collect()
target_range=(0,20)
 lr = LinearRegression()
lr.fit(X_train.values, y_train )<compute_test_metric>
train_df, val_df = train_test_split(df, test_size=0.2, random_state=RANDOM_STATE)
 train_df, test_df = train_test_split(train_df, test_size=0.2, random_state=RANDOM_STATE)
 if VERBOSE:
print(train_df.shape, val_df.shape, test_df.shape )
pred_lr_val = lr.predict(X_val)
 print('Val RMSE for lr is %f' % sqrt(mean_squared_error(y_val, pred_lr_val.clip(0,20))))
print('R^2 score for lr is %f' %(r2_score(y_val, pred_lr_val.clip(0,20))))<train_model>
LEARNING_RATE = 0.005
EPOCHS = 16
PATIENCE = 16
LR_PATIENCE = 8
VALIDATION_GENERATOR = RetinaGenerator(val_df, BATCH_SIZE)
 TRAINING_GENERATOR = RetinaGenerator(train_df, BATCH_SIZE)
 ALL_GENERATOR = RetinaGenerator(df, BATCH_SIZE)
 STEPS_PER_EPOCH = TRAINING_GENERATOR.step_per_epoch
VALIDATION_STEPS = VALIDATION_GENERATOR.step_per_epoch
over_underfit_plot(lr, X_train, y_train )<predict_on_test>
visible = Input(shape=(HEIGHT, WIDTH, 3))
densenet = DenseNet121(include_top=False,
weights=None,
input_tensor=visible)
 densenet.load_weights(densenet_weights_path)
 model = Sequential()
model.add(densenet)
 model.add(GlobalAveragePooling2D())
 model.add(Dropout(0.5))
model.add(Dense(5, activation='sigmoid'))
model.compile(
 loss='binary_crossentropy',
optimizer=optimizers.Adam(lr=0.00005),
metrics=['accuracy'] 
)
 print(model.summary())
 opt = optimizers.SGD(lr=LEARNING_RATE, decay=1e-6, momentum=0.9, nesterov=True)
 Callbacks=[EarlyStopping(patience=PATIENCE, restore_best_weights=True),
ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=LR_PATIENCE, verbose=VERBOSE)]
with tf.device('/device:GPU:0'):
H = model.fit_generator(generator=TRAINING_GENERATOR.getGenerator() ,
validation_data=VALIDATION_GENERATOR.getGenerator() ,
steps_per_epoch=STEPS_PER_EPOCH,
validation_steps=VALIDATION_STEPS,
epochs=EPOCHS,
callbacks=Callbacks,
verbose=VERBOSE )
pred_lr = lr.predict(X_test.values ).clip(*target_range)
 print(' pred_lr min %f , pred_lr max %f ' ,(pred_lr.min() ,pred_lr.max()))<train_on_grid>
if VERBOSE:
TEST_GENERATOR = RetinaGenerator(test_df, BATCH_SIZE)
 STEPS = TEST_GENERATOR.step_per_pred
Y_pred =(model.predict_generator(TEST_GENERATOR.getInputGenerator() , steps=STEPS)> 0.5 ).sum(axis=1)- 1
Y_test = np.array(test_df.diagnosis)
 print("Average absolute distance is: {:.2f}".format(np.abs(Y_pred - Y_test ).mean()))
display(confusion_matrix(Y_test, Y_pred))
print("Accuracy Score:" + str(accuracy_score(Y_test, Y_pred)))
 print("Cohen Kappa Score:" + str(cohen_kappa_score(Y_test, Y_pred, weights='quadratic')))
scaler = StandardScaler().fit(X_train)
 X_train_std = scaler.transform(X_train)
 X_valid_std = scaler.transform(X_val)
 model_lr_scaled = LinearRegression(fit_intercept=True, normalize=True)
 model_lr_scaled.fit(X_train_std, y_train)
 print(np.sqrt(mean_squared_error(y_val, model_lr_scaled.predict(X_valid_std))))
model_lr_scaled.predict(X_valid_std ).min()
over_underfit_plot(model_lr_scaled, X_train_std, y_train )<compute_test_metric>
STEPS = ALL_GENERATOR.step_per_pred
Y_pred = model.predict_generator(ALL_GENERATOR.getInputGenerator() , steps=STEPS)
 Y_true = to_categorical(np.array(df.diagnosis), 5 )
print('Val RMSE for lr is %f' % sqrt(mean_squared_error(y_val, model_lr_scaled.predict(X_val))))
print('R^2 score for lr is %f' %(r2_score(y_val, model_lr_scaled.predict(X_val))))<compute_test_metric>
post_model = Sequential()
post_model.add(Dense(5, input_dim=5, activation='relu'))
post_model.add(Dense(5, activation='softmax'))
post_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 opt = optimizers.SGD(lr=LEARNING_RATE, decay=1e-6, momentum=0.9, nesterov=True)
 Callbacks=[EarlyStopping(patience=PATIENCE, restore_best_weights=True),
ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=LR_PATIENCE, verbose=VERBOSE)]
with tf.device('/device:GPU:0'):
H = post_model.fit(Y_pred, Y_true, batch_size=10, epochs=10, verbose=VERBOSE)
print('Val RMSE for lr is %f' % sqrt(mean_squared_error(y_val, model_lr_scaled.predict(X_val ).clip(0,20))))
print('R^2 score for lr is %f' %(r2_score(y_val, model_lr_scaled.predict(X_val ).clip(0,20))))<predict_on_test>
if DATA_AUGM:
shutil.rmtree(PATH_AUGM, ignore_errors=True, onerror=None )
pred_lr_scaled = model_lr_scaled.predict(X_test.values ).clip(*target_range )<train_model>
sub_df = pd.read_csv(PATH_APTOS + 'test.csv')
 if VERBOSE:
print(sub_df.shape)
 display(sub_df.head() )
<init_hyperparams>
SUB_GENERATOR = RetinaGenerator(sub_df, BATCH_SIZE, False)
 STEPS = SUB_GENERATOR.step_per_pred
Y_sub_preview = model.predict_generator(SUB_GENERATOR.getInputGenerator() , steps=STEPS)
 Y_sub_cat = post_model.predict(Y_sub_preview)
 Y_sub = np.argmax(Y_sub_cat, axis=1)
 if VERBOSE:
print(Y_sub.shape )
<init_hyperparams><EOS> 
sub_df.to_csv('submission.csv', index=False )