Kaggle competition: APTOS 2019 Blindness Detection
embed_size = 300
max_features = 120000
maxlen = 70
batch_size = 512
n_epochs = 4
n_splits = 5
char_features = 130
char_max_len = 130
char_out = 50
k = 6
pca_dim = 16
SEED = 1029<set_options>
TRAIN_PATH_NEW = '.. /input/upload3/train_images/train_images/'
TRAIN_PATH_OLD = '.. /input/upload3/train_2015/train_2015/'
TEST_PATH = '.. /input/aptos2019-blindness-detection/test_images/'
df_old = pd.read_csv('.. /input/upload2/trainlabels_2015.csv')
 df_old.columns = ['id_code', 'diagnosis']
df_old['id_code'] = TRAIN_PATH_OLD + df_old['id_code'] + '.jpeg'
df_new = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 df_new['id_code'] = TRAIN_PATH_NEW + df_new['id_code'] + ".png"
df = pd.concat([df_old, df_new], axis = 0)
 df_old['diagnosis'] = df_old['diagnosis'].astype(str)
 df_new['diagnosis'] = df_new['diagnosis'].astype(str)
 train, test = train_test_split(df_new, test_size =.2, random_state = 123, stratify = df_new['diagnosis'])
 batch_size = 8
image_size =(456,456)
 figax, ax = plt.subplots(1,1,figsize =(10,5))
sns.barplot(x = df['diagnosis'].value_counts().index, y = df['diagnosis'].value_counts() ).set_title("Diagnosis Distribution")
 sns.despine(bottom = True, left = True)
 sns.set_style('dark')
 ax.set_ylabel("")
 ax.get_yaxis().set_visible(False)
 plt.show()
def seed_everything(seed=1029):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<features_selection>
pretrain_generator = ImgGenerator(df = df_old,
x_col = "id_code",
y_col = 'diagnosis',
batch_size = batch_size,
img_size = image_size,
postprocessing_function = postprocess_input,
should_aug = True,
should_mix = False,
iaa_seq = seq)
 pretrain_validation_generator = ImgGenerator(df = df_new,
x_col = "id_code",
y_col = 'diagnosis',
batch_size = batch_size,
img_size = image_size,
postprocessing_function = postprocess_input,
should_aug = False,
should_mix = False)
 train_generator = ImgGenerator(df = train,
x_col = "id_code",
y_col = 'diagnosis',
batch_size = batch_size,
img_size = image_size,
postprocessing_function = postprocess_input,
should_aug = True,
should_mix = False,
iaa_seq = seq)
 validation_generator = ImgGenerator(df = test,
x_col = "id_code",
y_col = 'diagnosis',
batch_size = batch_size,
img_size = image_size,
postprocessing_function = postprocess_input,
should_aug = False,
should_mix = False )
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')[:300]
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.005838499,0.48782197
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.0053247833,0.49346462
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<load_from_csv>
class AdamAccumulate(Optimizer):
def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., amsgrad=False, accum_iters=1, **kwargs):
if accum_iters < 1:
raise ValueError('accum_iters must be >= 1')
 super(AdamAccumulate, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.lr = K.variable(lr, name='lr')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.amsgrad = amsgrad
self.accum_iters = K.variable(accum_iters, K.dtype(self.iterations))
self.accum_iters_float = K.cast(self.accum_iters, K.floatx())
 @interfaces.legacy_get_updates_support
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
completed_updates = K.cast(K.tf.floordiv(self.iterations, self.accum_iters), K.floatx())
 if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * completed_updates))
t = completed_updates + 1
lr_t = lr *(K.sqrt(1.- K.pow(self.beta_2, t)) /(1.- K.pow(self.beta_1, t)))
 update_switch = K.equal(( self.iterations + 1)% self.accum_iters, 0)
 update_switch = K.cast(update_switch, K.floatx())
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
else:
vhats = [K.zeros(1)for _ in params]
self.weights = [self.iterations] + ms + vs + vhats
for p, g, m, v, vhat, tg in zip(params, grads, ms, vs, vhats, gs):
sum_grad = tg + g
avg_grad = sum_grad / self.accum_iters_float
m_t =(self.beta_1 * m)+(1.- self.beta_1)* avg_grad
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(avg_grad)
 if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 p_t = p - lr_t * m_t /(K.sqrt(vhat_t)+ self.epsilon)
 self.updates.append(K.update(vhat,(1 - update_switch)* vhat + update_switch * vhat_t))
else:
p_t = p - lr_t * m_t /(K.sqrt(v_t)+ self.epsilon)
 self.updates.append(K.update(m,(1 - update_switch)* m + update_switch * m_t))
self.updates.append(K.update(v,(1 - update_switch)* v + update_switch * v_t))
self.updates.append(K.update(tg,(1 - update_switch)* sum_grad))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p,(1 - update_switch)* p + update_switch * new_p))
return self.updates
def get_config(self):
config = {'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad}
base_config = super(AdamAccumulate, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
df_train = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv")
 df = pd.concat([df_train ,df_test],sort=True )<feature_engineering>
class EvaluationMetrics(Callback):
def __init__(self, validation_data = None, batch_size = 32):
super(Callback, self ).__init__()
self.validation_generator = validation_data
self.batch_size = batch_size
def on_train_begin(self, logs={}):
self._val_qwk = []
self._val_pred_counts = pd.DataFrame(index = [0,1,2,3,4])
 self._val_confusion_matrices = []
self._epoch = 1
def on_epoch_end(self, batch, logs={}):
y_val = np.array([])
 for i in range(len(validation_generator)) :
y_val_batch = validation_generator[i][1]
y_val = np.append(y_val, y_val_batch)
 y_val.astype(int)
 y_predict = model.predict_generator(generator=validation_generator,
steps=np.ceil(float(len(y_val)) / float(batch_size)) ,
verbose = 1 ).tolist()
for i in range(len(y_predict)) :
if y_predict[i][0] <= 0.5:
y_predict[i][0] = 0
elif y_predict[i][0] <= 1.5:
y_predict[i][0] = 1
elif y_predict[i][0] <= 2.5:
y_predict[i][0] = 2
elif y_predict[i][0] <= 3.5:
y_predict[i][0] = 3
else:
y_predict[i][0] = 4
y_predict = np.array(y_predict)
 self.qwk_hx(y_val, y_predict)
 self.value_counts_hx(y_predict)
 self.confusion_matrix_hx(y_val, y_predict)
 self._epoch += 1
return
def qwk_hx(self, y_val, y_predict):
qwk = cohen_kappa_score(y_val,
y_predict,
labels = [0,1,2,3,4],
weights='quadratic')
 print("
quadratic weighted kappa score: {0}
".format(qwk))
self._val_qwk.append(qwk)
 return
def value_counts_hx(self, y_predict):
self._val_pred_counts['Epoch {0}'.format(self._epoch)] = pd.DataFrame(y_predict)[0].value_counts()
self._val_pred_counts.fillna(0,inplace=True)
 self._val_pred_counts = self._val_pred_counts.astype(int)
 print("Value counts at Epoch {0}:
".format(self._epoch))
print(self._val_pred_counts.head())
 return
def confusion_matrix_hx(self, y_val, y_predict):
self._val_confusion_matrices.append(confusion_matrix(y_val, y_predict))
return
def get_qwk_hx(self):
return self._val_qwk
def get_value_counts_hx(self):
return self._val_pred_counts
def get_confusion_matrix_hx(self):
return self.val_confusion_matrices
eval_metrics = EvaluationMetrics(validation_data = validation_generator, batch_size = batch_size )
def build_vocab(texts):
sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
vocab = build_vocab(df['question_text'] )<define_variables>
def mse(y_true, y_pred):
return K.mean(K.square(y_pred -y_true), axis=-1)
 inputs = Input(( 456, 456, 3))
base_model = EfficientNetB5(include_top=False, weights=None, input_tensor = inputs)
 base_model.load_weights('.. /input/efficientnet-keras-weights-b0b5/efficientnet-b5_imagenet_1000_notop.h5')
 out = GlobalAveragePooling2D()(base_model.output)
 out = Dropout(0.5 )(out)
 out = Dense(1024,activation="relu", kernel_regularizer=l2(0.01))(out)
 out = Dropout(0.5 )(out)
 out = Dense(1,activation="linear", kernel_regularizer=l2(0.01))(out)
 model = Model(inputs, out)
 learning_rate_reduction = ReduceLROnPlateau(monitor='val_loss',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.0000001)
 early_stopping = EarlyStopping(monitor = 'val_loss',
patience = 10,
restore_best_weights = True)
 for layer in model.layers[:-5]:
layer.trainable = False
model.compile(loss='mse', optimizer=AdamAccumulate(lr=0.001, accum_iters = 2), metrics=[mse])
 print("
Warming up model: " + "
"*2)
 epochs = 5
history = model.fit_generator(pretrain_generator,
epochs=epochs,
steps_per_epoch =(df_old.shape[0] // batch_size)+ 1,
validation_data = pretrain_validation_generator,
validation_steps =(df_new.shape[0] // batch_size)+ 1,
workers = 2,
use_multiprocessing = True,
verbose=1)
 for layer in model.layers:
layer.trainable = True
print('
Begining training session: ' + '
'*2)
 model.compile(loss='mse', optimizer=AdamAccumulate(lr=0.0005, accum_iters = 3), metrics=[mse])
 epochs = 10
history2 = model.fit_generator(pretrain_generator,
epochs=epochs,
steps_per_epoch =(df_old.shape[0] // batch_size)+ 1,
validation_data = pretrain_validation_generator,
validation_steps =(df_new.shape[0] // batch_size)+ 1,
callbacks=[learning_rate_reduction, early_stopping],
workers = 2,
use_multiprocessing = True,
verbose=1)
 epochs = 40
history3 = model.fit_generator(train_generator,
epochs=epochs,
steps_per_epoch =(train.shape[0] // batch_size)+ 1,
validation_data = validation_generator,
validation_steps =(test.shape[0] // batch_size)+ 1,
callbacks=[learning_rate_reduction, early_stopping],
workers = 2,
use_multiprocessing = True,
verbose=1)
 model.save('model.h5' )
sin = len(df_train[df_train["target"]==0])
 insin = len(df_train[df_train["target"]==1])
 persin =(sin/(sin+insin)) *100
perinsin =(insin/(sin+insin)) *100
print("
print("<feature_engineering>
y_val = np.array([])
 for i in range(len(validation_generator)) :
v_val_batch = validation_generator[i][1]
y_val = np.append(y_val, v_val_batch)
 y_val.astype(int)
 y_pred = model.predict_generator(generator=validation_generator,
steps=np.ceil(float(len(y_val)) / float(batch_size)) ,
verbose=1)
 y_pred[y_pred <= 0.5] = 0
y_pred[(y_pred <= 1.5)&(y_pred > 0.5)] = 1
y_pred[(y_pred <= 2.5)&(y_pred > 1.5)] = 2
y_pred[(y_pred <= 3.5)&(y_pred > 2.5)] = 3
y_pred[y_pred > 3.5] = 4
score = cohen_kappa_score(y_val,
y_pred.astype(int),
labels=[0,1,2,3,4],
weights='quadratic')
 print("Quadratic weighted kappa: {0}".format(score))
<define_variables><EOS> 
sub_df = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 sub_df['id_code'] = sub_df['id_code']+'.png'
preds = []
for ix, idx in enumerate(sub_df['id_code']):
path = '.. /input/aptos2019-blindness-detection/test_images/' + idx
image = cv2.imread(path)
 image = cv2.resize(image,(456,456))
score_predict = [model.predict(( postprocess_input(seq_tta.augment_image(image)) [np.newaxis])/255)for i in range(5)]
score_predict.append(model.predict(( postprocess_input(image)[np.newaxis])/255))
score_predict = sum(score_predict)/ float(len(score_predict))
if score_predict <= 0.5:
score_predict = 0
elif score_predict <= 1.5:
score_predict = 1
elif score_predict <= 2.5:
score_predict = 2
elif score_predict <= 3.5:
score_predict = 3
else:
score_predict = 4
preds.append(int(score_predict))
sub_df['diagnosis'] = np.array(preds, np.int)
 sub_df['id_code'] = sub_df['id_code'].str.split(".png", expand = True)[0]
sub_df.to_csv('submission.csv', index = False)
 sub_df.head()