Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<predict_on_test>
from random import randint
import tensorflow as tf
import json
import os
from PIL import Image
from glob import glob
from zipfile import ZipFile
import pandas as pd
from keras.optimizers import Adam
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ModelCheckpoint, Callback, EarlyStopping
from keras.models import Model, Input, load_model
from keras.layers import Dense, GlobalAveragePooling2D, Dropout
from keras.applications.nasnet import NASNetLarge
from keras import backend as K
from matplotlib import pyplot as plt
from matplotlib.image import imread
import os
import sys
import cv2
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.metrics import cohen_kappa_score, accuracy_score
import scipy
from tqdm import tqdm
from skimage.io import imsave
submission['target'] = 1 - lgb_base.predict_proba(test.drop('ID_code', axis = 1))<save_to_csv>
def process_csv(dataframe: pd.DataFrame, image_column_name: str,
label_column_name: str,
folder_with_images: str)-> pd.DataFrame:

 dataframe[image_column_name] = dataframe[image_column_name].apply(
 lambda x: f"{folder_with_images}{x}.png")
 dataframe[label_column_name] = dataframe[label_column_name].astype('str')
 return dataframe
submission.to_csv("submission_hyperparams_borrowed.csv", index = False )<set_options>
train_datagen = ImageDataGenerator(rescale=1./ 255,
rotation_range=15,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=0.01,
zoom_range=[0.9, 1.25],
horizontal_flip=True,
vertical_flip=True,
fill_mode='reflect',
data_format='channels_last',
brightness_range=[0.5, 1.5],
validation_split=0.3 )
print(os.listdir(".. /input"))
warnings.simplefilter(action='ignore', category=FutureWarning)
 warnings.filterwarnings('ignore')
 plt.style.use('seaborn')
 sns.set(font_scale=1 )<load_from_csv>
train_csv = pd.read_csv("/kaggle/input/aptos2019-blindness-detection/train.csv")
 train_csv = process_csv(
 dataframe=train_csv,
image_column_name="id_code",
label_column_name="diagnosis",
folder_with_images="/kaggle/dataset_with_ben/" )
random_state = 42
np.random.seed(random_state)
 df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 features = [c for c in df_train.columns if c not in ['ID_code', 'target']]
specific_features = ['var_68', 'var_91', 'var_108', 'var_103', 'var_12']<feature_engineering>
train_generator = train_datagen.flow_from_dataframe(
 dataframe=train_csv, x_col="id_code", y_col="diagnosis", subset="training",
batch_size=32, target_size=(299, 299))
val_generator = train_datagen.flow_from_dataframe(
 dataframe=train_csv, x_col="id_code", y_col="diagnosis",
subset="validation", batch_size=32, target_size=(299, 299))
a, b = np.zeros(len(df_train['var_0'])) , np.zeros(len(df_test['var_0']))
for f in specific_features:
col = collections.Counter(df_train[f])
 for i in range(len(df_train[f])) :
a[i] = col[df_train[f][i]]
for j in range(len(df_test[f])) :
b[j] = col[df_test[f][j]]
a_avg, b_avg = np.average(a), np.average(b)
 a_std, b_std = np.std(a), np.std(b)
 a =(a - a_avg)/ a_std
b =(b - b_avg)/ b_std
temp1 = pd.DataFrame({'freq_' + f: a})
 temp2 = pd.DataFrame({'freq_' + f: b})
 df_train = pd.merge(df_train, temp1, left_index=True, right_index=True)
 df_test = pd.merge(df_test, temp2, left_index=True, right_index=True )<categorify>
class RAdam(Optimizer):
def __init__(self, lr, beta1=0.9, beta2=0.99, decay=0, **kwargs):
super(RAdam, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.lr = K.variable(lr)
 self._beta1 = K.variable(beta1, dtype="float32")
 self._beta2 = K.variable(beta2, dtype="float32")
 self._max_sma_length = 2 /(1 - self._beta2)
 self._iterations = K.variable(0)
 self._decay = K.variable(decay)
 def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self._iterations, 1)]
first_moments = [K.zeros(K.int_shape(p), dtype=K.dtype(p))
for(i, p)in enumerate(params)]
second_moments = [K.zeros(K.int_shape(p), dtype=K.dtype(p))
for(i, p)in enumerate(params)]
self.weights = [self._iterations] + first_moments + second_moments
bias_corrected_beta1 = K.pow(self._beta1, self._iterations)
 bias_corrected_beta2 = K.pow(self._beta2, self._iterations)
 for i,(curr_params, curr_grads)in enumerate(zip(params, grads)) :
new_first_moment = self._beta1 * first_moments[i] +(
 1 - self._beta1)* curr_grads
new_second_moment = self._beta2 * second_moments[i] +(
 1 - self._beta2)* K.square(curr_grads)
 self.updates.append(K.update(first_moments[i],
new_first_moment))
self.updates.append(K.update(second_moments[i],
new_second_moment))
bias_corrected_moving_average = new_first_moment /(
 1 - bias_corrected_beta1)
 sma_length = self._max_sma_length - 2 *(
 self._iterations * bias_corrected_beta2)/(
 1 - bias_corrected_beta2)
 variance_rectification_term = K.sqrt(
 self._max_sma_length *(sma_length - 4)*(sma_length - 2)/(
 sma_length *(self._max_sma_length - 4)* 
(self._max_sma_length - 2)+ K.epsilon()))
resulting_parameters = K.switch(
 sma_length > 5, variance_rectification_term *
bias_corrected_moving_average / K.sqrt(
 K.epsilon() + new_second_moment /(1 -
bias_corrected_beta2)) ,
bias_corrected_moving_average)
 resulting_parameters = curr_params - self.lr * resulting_parameters
self.updates.append(K.update(curr_params, resulting_parameters))
if self._decay != 0:
new_lr = self.lr *(1./(1.+ self._decay * K.cast(
 self._iterations, K.dtype(self._decay))))
self.updates.append(K.update(self.lr, new_lr))
return self.updates
def get_config(self):
config = {
"lr": float(K.get_value(self.lr)) ,
"beta1": float(K.get_value(self._beta1)) ,
"beta2": float(K.get_value(self._beta2)) ,
}
base_config = super(RAdam, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
def augment(x,y,t=2):
xs,xn = [],[]
for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xs.append(x1)
 for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xn.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x,xs,xn])
 y = np.concatenate([y,ys,yn])
 return x,y<init_hyperparams>
sys.path.append(os.path.abspath('.. /input/kerasefficientnetsmaster/keras-efficientnets-master/keras-efficientnets-master/'))
def create_model() :
input_tensor = Input(( 299, 299, 3))
outputs = []
effnet = EfficientNetB7(input_shape=(299,299,3),
weights=sys.path.append(os.path.abspath('/kaggle/input/efficientnetb0b7-keras-weights/efficientnet-b7_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5')) ,
include_top=False)
 xception_model = Xception(weights=None,include_top=False,input_shape=(299,299,3))
xception_model.load_weights("/kaggle/input/xception/xception_weights_tf_dim_ordering_tf_kernels_notop.h5")
 pretrained_models = [
effnet,xception_model
]
for i, model in enumerate(pretrained_models):
curr_output = model(input_tensor)
 curr_output = GlobalAveragePooling2D()(curr_output)
 curr_output = Dense(1024, activation="relu" )(curr_output)
 outputs.append(curr_output)
 output_tensor = Average()(outputs)
 output_tensor = Dense(5, activation="softmax" )(output_tensor)
 model = Model(input_tensor, output_tensor)
 return model
lgb_params = {
"objective" : "binary",
"metric" : "auc",
"boosting": 'gbdt',
"max_depth" : -1,
"num_leaves" : 22,
"learning_rate" : 0.0083,
"bagging_freq": 5,
"bagging_fraction" : 0.93,
"feature_fraction" : 0.042,
"min_data_in_leaf": 76,
"min_sum_heassian_in_leaf": 15,
"tree_learner": "serial",
"boost_from_average": "false",
"bagging_seed" : random_state,
"verbosity" : 1,
"seed": random_state
}<create_dataframe>
start_lr = 1e-10
end_lr = 1
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=random_state)
 oof = df_train[['ID_code', 'target']]
oof['predict'] = 0
predictions = df_test[['ID_code']]
val_aucs = []
feature_importance_df = pd.DataFrame()<prepare_x_and_y>
def kappa_loss(y_pred, y_true, y_pow=2, eps=1e-10, N=5, bsize=256, name='kappa'):

 with tf.name_scope(name):
y_true = tf.to_float(y_true)
 repeat_op = tf.to_float(tf.tile(tf.reshape(tf.range(0, N), [N, 1]), [1, N]))
repeat_op_sq = tf.square(( repeat_op - tf.transpose(repeat_op)))
 weights = repeat_op_sq / tf.to_float(( N - 1)** 2)
 pred_ = y_pred ** y_pow
try:
pred_norm = pred_ /(eps + tf.reshape(tf.reduce_sum(pred_, 1), [-1, 1]))
except Exception:
pred_norm = pred_ /(eps + tf.reshape(tf.reduce_sum(pred_, 1), [bsize, 1]))
hist_rater_a = tf.reduce_sum(pred_norm, 0)
 hist_rater_b = tf.reduce_sum(y_true, 0)
 conf_mat = tf.matmul(tf.transpose(pred_norm), y_true)
 nom = tf.reduce_sum(weights * conf_mat)
 denom = tf.reduce_sum(weights * tf.matmul(
 tf.reshape(hist_rater_a, [N, 1]), tf.reshape(hist_rater_b, [1, N])) /
tf.to_float(bsize))
return nom /(denom + eps )
features = [col for col in df_train.columns if col not in ['target', 'ID_code']]
X_test = df_test[features].values<split>
reduce_lr = ReduceLROnPlateau(monitor='val_acc', factor=0.2,
patience=5, min_lr=1e-5 )
for fold,(trn_idx, val_idx)in enumerate(skf.split(df_train, df_train['target'])) :
X_train, y_train = df_train.iloc[trn_idx][features], df_train.iloc[trn_idx]['target']
X_valid, y_valid = df_train.iloc[val_idx][features], df_train.iloc[val_idx]['target']
N = 5
p_valid,yp = 0,0
for i in range(N):
X_t, y_t = augment(X_train.values, y_train.values)
 X_t = pd.DataFrame(X_t)
 X_t = X_t.add_prefix('var_')
 trn_data = lgb.Dataset(X_t, label=y_t)
 val_data = lgb.Dataset(X_valid, label=y_valid)
 evals_result = {}
lgb_clf = lgb.train(lgb_params,
trn_data,
100000,
valid_sets = [trn_data, val_data],
early_stopping_rounds=3000,
verbose_eval = 1000,
evals_result=evals_result 
)
 p_valid += lgb_clf.predict(X_valid)
 yp += lgb_clf.predict(X_test)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = lgb_clf.feature_importance()
fold_importance_df["fold"] = fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 oof['predict'][val_idx] = p_valid/N
val_score = roc_auc_score(y_valid, p_valid)
 val_aucs.append(val_score)
 predictions['fold{}'.format(fold+1)] = yp/N<compute_test_metric>
callbacks = [
ModelCheckpoint(
 "best_weights.hdf5",
monitor='val_acc',
verbose=1, save_best_only=True,
save_weights_only=True),
EarlyStopping(monitor='val_acc', patience=5),
reduce_lr
]
mean_auc = np.mean(val_aucs)
 std_auc = np.std(val_aucs)
 all_auc = roc_auc_score(oof['target'], oof['predict'])
 print("Mean auc: %.9f, std: %.9f.All auc: %.9f." %(mean_auc, std_auc, all_auc))<save_to_csv>
model = create_model()
model.compile(optimizer=Adam(1e-4),
loss="categorical_crossentropy", metrics=["accuracy"])
 model.fit_generator(generator=train_generator,
steps_per_epoch=len(train_generator),
validation_data=val_generator,
validation_steps=len(val_generator),
epochs=10,
callbacks=callbacks )
predictions['target'] = np.mean(predictions[[col for col in predictions.columns if col not in ['ID_code', 'target']]].values, axis=1)
 predictions.to_csv('lgb_all_predictions.csv', index=None)
 sub_df = pd.DataFrame({"ID_code":df_test["ID_code"].values})
 sub_df["target"] = predictions['target']
sub_df.to_csv("lgb_submission.csv", index=False)
 oof.to_csv('lgb_oof.csv', index=False )<install_modules>
def test_time_augmentation(image, network_model):
datagen = ImageDataGenerator()
all_images = np.expand_dims(image, axis=0)
 flip_horizontal_image = np.expand_dims(datagen.apply_transform(
 x=image, transform_parameters={"flip_horizontal": True}), axis=0)
 all_images = np.append(all_images, flip_horizontal_image, axis=0)
 flip_vertical_image = np.expand_dims(datagen.apply_transform(
 x=image, transform_parameters={"flip_vertical": True}), axis=0)
 all_images = np.append(all_images, flip_vertical_image, axis=0)
 rotated_image = np.expand_dims(datagen.apply_transform(
 x=image, transform_parameters={"theta": randint(0, 15)}), axis=0)
 all_images = np.append(all_images, rotated_image, axis=0)
 prediction = int(np.argmax(np.mean(network_model.predict(all_images), axis=0)))
 return prediction
!pip install bayesian-optimization<set_options>
model.load_weights("best_weights.hdf5" )
<load_from_csv><EOS> 
test_csv = pd.read_csv("/kaggle/input/aptos2019-blindness-detection/test.csv")
 predicted_csv = pd.DataFrame(columns=["id_code", "diagnosis"])
 for id_code in test_csv["id_code"]:
filename = f"/kaggle/input/aptos2019-blindness-detection/test_images/{id_code}.png"
img = imread(filename)
 img = cv2.resize(img, dsize=(299, 299), interpolation=cv2.INTER_CUBIC)
 prediction = test_time_augmentation(img, model)
 predicted_csv = predicted_csv.append(
 {'id_code':id_code ,"diagnosis": prediction}, ignore_index=True)
 with open("submission.csv", "w")as f:
f.write(predicted_csv.to_csv(index=False))