Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<define_variables>
import os
import sys
import cv2
import time
import scipy as sp
import numpy as np
import pandas as pd
from tqdm import tqdm
from PIL import Image
from functools import partial
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import set_random_seed
import keras
from keras import initializers
from keras import regularizers
from keras import constraints
from keras import backend as K
from keras.activations import elu
from keras.optimizers import Adam
from keras.models import Sequential
from keras.engine import Layer, InputSpec
from keras.utils.generic_utils import get_custom_objects
from keras.callbacks import Callback, EarlyStopping, ReduceLROnPlateau
from keras.layers import Dense, Conv2D, Flatten, GlobalAveragePooling2D, Dropout
from keras.preprocessing.image import ImageDataGenerator
from sklearn.metrics import cohen_kappa_score
from keras.applications.resnet50 import ResNet50
for month in range(21, 33):
items_temp = items_trans['item_id'][items_trans.loc[:,str(month): '33'].sum(axis = 1)== 0]
print('From month: ', month, ' until month: 33, ', np.round(100 * len(items_temp)/ len(items_trans), 2), '% of the items have made no sales' )<set_options>
SEED = 7
np.random.seed(SEED)
 set_random_seed(SEED)
 INPUT_PATH = '.. /input/aptos2019-blindness-detection/'
DIM = 224
BATCH_SIZE = 4
CHANNEL_SIZE = 3
NUM_EPOCHS = 30
LR = 1e-3
CLASS= {0: "No DR", 1: "Mild", 2: "Moderate", 3: "Severe", 4: "Proliferative DR"}
NUM_CLASSES = len(CLASS.keys())
 SAVED_MODEL_NAME = 'model.h5'
del items_trans
del items_temp
gc.collect()<prepare_x_and_y>
train = pd.read_csv(INPUT_PATH + 'train.csv')
 test = pd.read_csv(INPUT_PATH + 'test.csv' )
start = time.time()
shops_trans = create_record_for_features(sales, ['shop_id'], 'item_cnt_day', 'date_block_num', aggfunc = np.count_nonzero, fill = 0)
 print('It tooks: ', round(time.time() - start), 'seconds' )<define_variables>
train['images'] = train['id_code'].apply(lambda x: INPUT_PATH + "train_images/" + str(x)+ ".png")
 test['images'] = test['id_code'].apply(lambda x: INPUT_PATH + "test_images/" + str(x)+ ".png")
 train.drop(['id_code'],axis = 1, inplace =True)
 train = train[['images','diagnosis']]
for month in range(1, 12):
shops_temp = shops_trans['shop_id'][shops_trans.loc[:,'0': str(month)].sum(axis = 1)== 0]
print('From month: 0 until ', month,', ', np.round(100 * len(shops_temp)/ len(shops_trans), 2), '% of the shops have made no sales' )<define_variables>
def get_preds_and_labels(model, generator):

 preds = []
labels = []
for _ in range(int(np.ceil(generator.samples / BATCH_SIZE))):
x, y = next(generator)
 preds.append(model.predict(x))
labels.append(y)
 return np.concatenate(preds ).ravel() , np.concatenate(labels ).ravel()
for month in range(21, 33):
shops_temp = shops_trans['shop_id'][shops_trans.loc[:, str(month): '33'].sum(axis = 1)== 0]
print('From month: ', month, ' until month: 33, ', np.round(100 * len(shops_temp)/ len(shops_trans), 2), '% of the shops have made no sales' )<set_options>
class Metrics(Callback):

 def on_train_begin(self, logs={}):

 self.val_kappas = []
def on_epoch_end(self, epoch, logs={}):

 y_pred, labels = get_preds_and_labels(model, val_generator)
 y_pred = np.rint(y_pred ).astype(np.uint8 ).clip(0, 4)
 _val_kappa = cohen_kappa_score(labels, y_pred, weights='quadratic')
 self.val_kappas.append(_val_kappa)
 print(f"val_kappa: {round(_val_kappa, 4)}")
 if _val_kappa == max(self.val_kappas):
print("Validation Kappa has improved.Saving model.")
 self.model.save(SAVED_MODEL_NAME)
 return
del shops_trans
del shops_temp
gc.collect() 
<count_unique_values>
train_datagen = ImageDataGenerator(rotation_range=360,
horizontal_flip=True,
vertical_flip=True,
validation_split=0.15,
preprocessing_function=preprocess_image,
rescale=1 / 128.)
train_generator = train_datagen.flow_from_dataframe(train,
x_col='images',
y_col='diagnosis',
target_size=(DIM, DIM),
batch_size=BATCH_SIZE,
class_mode='other',
subset='training')
 val_generator = train_datagen.flow_from_dataframe(train,
x_col='images',
y_col='diagnosis',
target_size=(DIM, DIM),
batch_size=BATCH_SIZE,
class_mode='other',
subset='validation' )
print('test shape: ', test.shape)
 print('number of items in test set: ', test['item_id'].nunique())
 print('number of shops in test set: ', test['shop_id'].nunique())
<count_unique_values>
class RAdam(keras.optimizers.Optimizer):

 def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., weight_decay=0., amsgrad=False,
total_steps=0, warmup_proportion=0.1, min_lr=0., **kwargs):
super(RAdam, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.lr = K.variable(lr, name='lr')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.weight_decay = K.variable(weight_decay, name='weight_decay')
 self.total_steps = K.variable(total_steps, name='total_steps')
 self.warmup_proportion = K.variable(warmup_proportion, name='warmup_proportion')
 self.min_lr = K.variable(lr, name='min_lr')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.initial_weight_decay = weight_decay
self.initial_total_steps = total_steps
self.amsgrad = amsgrad
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * K.cast(self.iterations, K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx())+ 1
if self.initial_total_steps > 0:
warmup_steps = self.total_steps * self.warmup_proportion
decay_steps = self.total_steps - warmup_steps
lr = K.switch(
 t <= warmup_steps,
lr *(t / warmup_steps),
lr *(1.0 - K.minimum(t, decay_steps)/ decay_steps), 
)
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for(i, p)in enumerate(params)]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for(i, p)in enumerate(params)]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for(i, p)in enumerate(params)]
else:
vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params)) ]
self.weights = [self.iterations] + ms + vs + vhats
beta_1_t = K.pow(self.beta_1, t)
 beta_2_t = K.pow(self.beta_2, t)
 sma_inf = 2.0 /(1.0 - self.beta_2)- 1.0
sma_t = sma_inf - 2.0 * t * beta_2_t /(1.0 - beta_2_t)
 for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 m_corr_t = m_t /(1.0 - beta_1_t)
 if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 v_corr_t = K.sqrt(vhat_t /(1.0 - beta_2_t)+ self.epsilon)
 self.updates.append(K.update(vhat, vhat_t))
else:
v_corr_t = K.sqrt(v_t /(1.0 - beta_2_t)+ self.epsilon)
 r_t = K.sqrt(( sma_t - 4.0)/(sma_inf - 4.0)* 
(sma_t - 2.0)/(sma_inf - 2.0)*
sma_inf / sma_t)
 p_t = K.switch(sma_t > 5, r_t * m_corr_t / v_corr_t, m_corr_t)
 if self.initial_weight_decay > 0:
p_t += self.weight_decay * p
p_t = p - lr * p_t
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
def get_config(self):
config = {
'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.weight_decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad,
'total_steps': float(K.get_value(self.total_steps)) ,
'warmup_proportion': float(K.get_value(self.warmup_proportion)) ,
'min_lr': float(K.get_value(self.min_lr)) ,
}
base_config = super(RAdam, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
test['item_id'].nunique() * test['shop_id'].nunique() == len(test )<categorify>
class GroupNormalization(Layer):

 def __init__(self,
groups=32,
axis=-1,
epsilon=1e-5,
center=True,
scale=True,
beta_initializer='zeros',
gamma_initializer='ones',
beta_regularizer=None,
gamma_regularizer=None,
beta_constraint=None,
gamma_constraint=None,
**kwargs):
super(GroupNormalization, self ).__init__(**kwargs)
 self.supports_masking = True
self.groups = groups
self.axis = axis
self.epsilon = epsilon
self.center = center
self.scale = scale
self.beta_initializer = initializers.get(beta_initializer)
 self.gamma_initializer = initializers.get(gamma_initializer)
 self.beta_regularizer = regularizers.get(beta_regularizer)
 self.gamma_regularizer = regularizers.get(gamma_regularizer)
 self.beta_constraint = constraints.get(beta_constraint)
 self.gamma_constraint = constraints.get(gamma_constraint)
 def build(self, input_shape):
dim = input_shape[self.axis]
if dim is None:
raise ValueError('Axis ' + str(self.axis)+ ' of '
'input tensor should have a defined dimension '
'but the layer received an input with shape ' +
str(input_shape)+ '.')
 if dim < self.groups:
raise ValueError('Number of groups(' + str(self.groups)+ ')cannot be '
'more than the number of channels(' +
str(dim)+ ' ).')
 if dim % self.groups != 0:
raise ValueError('Number of groups(' + str(self.groups)+ ')must be a '
'multiple of the number of channels(' +
str(dim)+ ' ).')
 self.input_spec = InputSpec(ndim=len(input_shape),
axes={self.axis: dim})
 shape =(dim,)
 if self.scale:
self.gamma = self.add_weight(shape=shape,
name='gamma',
initializer=self.gamma_initializer,
regularizer=self.gamma_regularizer,
constraint=self.gamma_constraint)
 else:
self.gamma = None
if self.center:
self.beta = self.add_weight(shape=shape,
name='beta',
initializer=self.beta_initializer,
regularizer=self.beta_regularizer,
constraint=self.beta_constraint)
 else:
self.beta = None
self.built = True
def call(self, inputs, **kwargs):
input_shape = K.int_shape(inputs)
 tensor_input_shape = K.shape(inputs)
 reduction_axes = list(range(len(input_shape)))
 del reduction_axes[self.axis]
broadcast_shape = [1] * len(input_shape)
 broadcast_shape[self.axis] = input_shape[self.axis] // self.groups
broadcast_shape.insert(1, self.groups)
 reshape_group_shape = K.shape(inputs)
 group_axes = [reshape_group_shape[i] for i in range(len(input_shape)) ]
group_axes[self.axis] = input_shape[self.axis] // self.groups
group_axes.insert(1, self.groups)
 group_shape = [group_axes[0], self.groups] + group_axes[2:]
group_shape = K.stack(group_shape)
 inputs = K.reshape(inputs, group_shape)
 group_reduction_axes = list(range(len(group_axes)))
 group_reduction_axes = group_reduction_axes[2:]
mean = K.mean(inputs, axis=group_reduction_axes, keepdims=True)
 variance = K.var(inputs, axis=group_reduction_axes, keepdims=True)
 inputs =(inputs - mean)/(K.sqrt(variance + self.epsilon))
inputs = K.reshape(inputs, group_shape)
 outputs = inputs
if self.scale:
broadcast_gamma = K.reshape(self.gamma, broadcast_shape)
 outputs = outputs * broadcast_gamma
if self.center:
broadcast_beta = K.reshape(self.beta, broadcast_shape)
 outputs = outputs + broadcast_beta
outputs = K.reshape(outputs, tensor_input_shape)
 return outputs
def get_config(self):
config = {
'groups': self.groups,
'axis': self.axis,
'epsilon': self.epsilon,
'center': self.center,
'scale': self.scale,
'beta_initializer': initializers.serialize(self.beta_initializer),
'gamma_initializer': initializers.serialize(self.gamma_initializer),
'beta_regularizer': regularizers.serialize(self.beta_regularizer),
'gamma_regularizer': regularizers.serialize(self.gamma_regularizer),
'beta_constraint': constraints.serialize(self.beta_constraint),
'gamma_constraint': constraints.serialize(self.gamma_constraint)
 }
base_config = super(GroupNormalization, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
def compute_output_shape(self, input_shape):
return input_shape
item_shop_test = set(test.item_id.astype(str)+ '_' + test.shop_id.astype(str))
item_shop_sales = set(sales.item_id.astype(str)+ '_' + sales.shop_id.astype(str))
pairs_with_history = len(item_shop_test.intersection(item_shop_sales))
pairs_with_no_history = len(shops_in_test_and_sales)* len(item_in_test_not_sales)
 just_item_with_history = test.shape[0] -(pairs_with_history + pairs_with_no_history)
 print('There is sales history for:', np.round(100 * pairs_with_history / len(test), 2), '% items in the same shops')
 print('There is No sales history for:', np.round(100 * pairs_with_no_history / len(test), 2), '% shop/item pairs')
 print('There is sales history for:', np.round(100 * just_item_with_history / len(test), 2), '% items but in different shops' )<drop_column>
resnet = ResNet50(weights=None,
include_top=False,
input_shape=(DIM, DIM, CHANNEL_SIZE))
resnet.load_weights('.. /input/resnet50-weights-file/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5' )
del items_test
del items_sales
del item_in_test_and_sales
del item_in_test_not_sales
del shops_test
del shops_sales
del shops_in_test_and_sales
del shops_in_test_not_sales
del item_shop_test
del item_shop_sales
del pairs_with_history
del pairs_with_no_history
del just_item_with_history
gc.collect()<feature_engineering>
for i, layer in enumerate(resnet.layers):
if "batch_normalization" in layer.name:
effnet.layers[i] = GroupNormalization(groups=32, axis=-1, epsilon=0.00001 )
shops['shop_name'] = shops['shop_name'].apply(lambda x: x.lower() ).str.replace('[^\w\s]', '' ).str.replace('\d+','' ).str.strip()
shops['shop_city'], shops['shop_name'] = shops['shop_name'].str.split(' ', 1 ).str
shops['shop_type'] = shops['shop_name'].apply(lambda x: 'мтрц' if 'мтрц' in x else 'трц' if 'трц' in x else 'трк' if 'трк' in x else 'тц' if 'тц' in x else 'тк' if 'тк' in x else 'unkown')
 shops.head()<count_unique_values>
def build_model() :

 model = Sequential()
model.add(resnet)
 model.add(GlobalAveragePooling2D())
 model.add(Dropout(0.5))
model.add(Dense(5, activation=elu))
model.add(Dense(1, activation="linear"))
model.compile(loss='mse',
optimizer=RAdam(lr=0.00005),
metrics=['mse', 'acc'])
 print(model.summary())
 return model
model = build_model()
print('Shops number:', shops['shop_id'].nunique())
 print('Shop names number:', shops['shop_name'].nunique())
 print('Shop cities number:', shops['shop_city'].nunique())
 print('Shop types number:', shops['shop_type'].nunique() )<feature_engineering>
kappa_metrics = Metrics()
es = EarlyStopping(monitor='val_loss', mode='auto', verbose=1, patience=12)
 rlr = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=4,
verbose=1,
mode='auto',
epsilon=0.0001)
 with tf.device('/gpu:0'):
history = model.fit_generator(train_generator,
steps_per_epoch=train_generator.samples // BATCH_SIZE,
epochs=35,
validation_data=val_generator,
validation_steps = val_generator.samples // BATCH_SIZE,
callbacks=[kappa_metrics, es, rlr],
verbose = 1 )
sales.loc[sales['shop_id'] == 11, 'shop_id'] = 10
shops.loc[shops['shop_id'] == 11, 'shop_id'] = 10
sales.loc[sales['shop_id'] == 23, 'shop_id'] = 24
shops.loc[shops['shop_id'] == 23, 'shop_id'] = 24
sales.loc[sales['shop_id'] == 0, 'shop_id'] = 57
shops.loc[shops['shop_id'] == 0, 'shop_id'] = 57
sales.loc[sales['shop_id'] == 1, 'shop_id'] = 58
shops.loc[shops['shop_id'] == 1, 'shop_id'] = 58
sales.loc[sales['shop_id'] == 40, 'shop_id'] = 39
shops.loc[shops['shop_id'] == 40, 'shop_id'] = 39
shops = shops.drop_duplicates(subset = 'shop_id' )<count_unique_values>
model.load_weights(SAVED_MODEL_NAME )
print('Shops number:', shops['shop_id'].nunique())
 print('Shop names number:', shops['shop_name'].nunique())
 print('Shop cities number:', shops['shop_city'].nunique())
 print('Shop types number:', shops['shop_type'].nunique() )<feature_engineering>
y_train_preds, train_labels = get_preds_and_labels(model, train_generator)
 y_train_preds = np.rint(y_train_preds ).astype(np.uint8 ).clip(0, 4)
 train_score = cohen_kappa_score(train_labels, y_train_preds, weights="quadratic")
 y_val_preds, val_labels = get_preds_and_labels(model, val_generator)
 y_val_preds = np.rint(y_val_preds ).astype(np.uint8 ).clip(0, 4)
 val_score = cohen_kappa_score(val_labels, y_val_preds, weights="quadratic" )
def clean_names(df, cols):
for col in cols:
df[col] = df[col].str.replace('[^A-Za-z0-9А-Яа-я]+', ' ' ).str.lower()
df[col] = df[col].str.strip()
df.loc[df[col] == '', col] = 'unknown'<feature_engineering>
print(f"The Training Cohen Kappa Score is: {round(train_score, 5)}")
 print(f"The Validation Cohen Kappa Score is: {round(val_score, 5)}" )
items['item_name'], items['item_type'] = items['item_name'].str.split('[', 1 ).str
items['item_name'], items['item_subtype'] = items['item_name'].str.split('(', 1 ).str
clean_names(items, ['item_name', 'item_type', 'item_subtype'])
 items = items.fillna('unkown')
 items.head() 
<count_unique_values>
class OptimizedRounder(object):

 def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):

 X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):

 loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):

 X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
print('Number of items:', items['item_id'].nunique())
 print('Number of item_name:', items['item_name'].nunique())
 print('Number of item_type:', items['item_type'].nunique())
 print('Number of item_subtype:', items['item_subtype'].nunique() )<merge>
y_val_preds, val_labels = get_preds_and_labels(model, val_generator)
 optR = OptimizedRounder()
optR.fit(y_val_preds, val_labels)
 coefficients = optR.coefficients()
opt_val_predictions = optR.predict(y_val_preds, coefficients)
 new_val_score = cohen_kappa_score(val_labels, opt_val_predictions, weights="quadratic" )
sales = sales.merge(items[['item_id', 'item_category_id']], on = 'item_id', how = 'left' )<define_variables>
print(f"Optimized Thresholds:
{coefficients}
")
 print(f"The Validation Quadratic Weighted Kappa(QWK)
 \
with optimized rounding thresholds is: {round(new_val_score, 5)}
")
 print(f"This is an improvement of {round(new_val_score - val_score, 5)}
\
over the unoptimized rounding" )
time_shift = 14<create_dataframe>
test['diagnosis'] = np.zeros(test.shape[0])
 test_generator = ImageDataGenerator(preprocessing_function=preprocess_image,
rescale=1 / 128.).flow_from_dataframe(test,
x_col='images',
y_col='diagnosis',
target_size=(DIM, DIM),
batch_size=BATCH_SIZE,
class_mode= 'other',
shuffle=False )
pairs_sales = create_record_for_features(sales, ['shop_id', 'item_id'], 'item_cnt_day', 'date_block_num', aggfunc = np.sum, fill = 0 )<feature_engineering>
y_test,_ = get_preds_and_labels(model, test_generator)
 y_test = optR.predict(y_test, coefficients ).astype(np.uint8)
 test['diagnosis'] = y_test
test['id_code'] = test['id_code']
test.drop(['images'], axis = 1, inplace = True)
 test.to_csv('submission.csv', index=False )