Kaggle competition: APTOS 2019 Blindness Detection
def bestThresshold(y_train,train_preds):
tmp = [0,0,0]
delta = 0
for tmp[0] in tqdm(np.arange(0.1, 0.501, 0.01)) :
tmp[1] = f1_score(y_train, np.array(train_preds)>tmp[0])
 if tmp[1] > tmp[2]:
delta = tmp[0]
tmp[2] = tmp[1]
print('best threshold is {:.4f} with F1 score: {:.4f}'.format(delta, tmp[2]))
return delta , tmp[2]
delta, _ = bestThresshold(y_train,train_preds )<save_to_csv>
BatchNormalization, Input, Conv2D, GlobalAveragePooling2D,concatenate,Concatenate)
if debug:
df_test = pd.read_csv(".. /input/test.csv")[:20000]
else:
df_test = pd.read_csv(".. /input/test.csv")
 submission = df_test[['qid']].copy()
submission['prediction'] =(test_preds > delta ).astype(int)
 submission.to_csv('submission.csv', index=False )<define_variables>
WORKERS = 2
CHANNEL = 3
warnings.filterwarnings("ignore")
 SIZE = 300
NUM_CLASSES = 5
max_features = 90000
maxlen = 50 
<set_options>
df_train = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 df_test = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv' )
os.environ['PYTHONHASHSEED'] = '11'
np.random.seed(22)
 random.seed(33)
 tf.set_random_seed(44)
<feature_engineering>
y = to_categorical(y, num_classes=NUM_CLASSES)
 train_x, valid_x, train_y, valid_y = train_test_split(x, y, test_size=0.15,
stratify=y, random_state=8)
 print(train_x.shape)
 print(train_y.shape)
 print(valid_x.shape)
 print(valid_y.shape )
t0 = time.time()<load_from_csv>
class My_Generator(Sequence):
def __init__(self, image_filenames, labels,
batch_size, is_train=True,
mix=False, augment=False):
self.image_filenames, self.labels = image_filenames, labels
self.batch_size = batch_size
self.is_train = is_train
self.is_augment = augment
if(self.is_train):
self.on_epoch_end()
self.is_mix = mix
def __len__(self):
return int(np.ceil(len(self.image_filenames)/ float(self.batch_size)))
 def __getitem__(self, idx):
batch_x = self.image_filenames[idx * self.batch_size:(idx + 1)* self.batch_size]
batch_y = self.labels[idx * self.batch_size:(idx + 1)* self.batch_size]
if(self.is_train):
return self.train_generate(batch_x, batch_y)
 return self.valid_generate(batch_x, batch_y)
 def on_epoch_end(self):
if(self.is_train):
self.image_filenames, self.labels = shuffle(self.image_filenames, self.labels)
 else:
pass
def mix_up(self, x, y):
lam = np.random.beta(0.2, 0.4)
 ori_index = np.arange(int(len(x)))
 index_array = np.arange(int(len(x)))
 np.random.shuffle(index_array)
 mixed_x = lam * x[ori_index] +(1 - lam)* x[index_array]
mixed_y = lam * y[ori_index] +(1 - lam)* y[index_array]
return mixed_x, mixed_y
def train_generate(self, batch_x, batch_y):
batch_images = []
for(sample, label)in zip(batch_x, batch_y):
img = cv2.imread('.. /input/aptos2019-blindness-detection/train_images/'+sample+'.png')
 img = cv2.resize(img,(SIZE, SIZE))
if(self.is_augment):
img = seq.augment_image(img)
 batch_images.append(img)
 batch_images = np.array(batch_images, np.float32)/ 255
batch_y = np.array(batch_y, np.float32)
 if(self.is_mix):
batch_images, batch_y = self.mix_up(batch_images, batch_y)
 return batch_images, batch_y
def valid_generate(self, batch_x, batch_y):
batch_images = []
for(sample, label)in zip(batch_x, batch_y):
img = cv2.imread('.. /input/aptos2019-blindness-detection/train_images/'+sample+'.png')
 img = cv2.resize(img,(SIZE, SIZE))
batch_images.append(img)
 batch_images = np.array(batch_images, np.float32)/ 255
batch_y = np.array(batch_y, np.float32)
 return batch_images, batch_y
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )<define_variables>
def create_model(input_shape, n_out):
input_tensor = Input(shape=input_shape)
 base_model = DenseNet121(include_top=False,
weights=None,
input_tensor=input_tensor)
 base_model.load_weights(".. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5")
 x = GlobalAveragePooling2D()(base_model.output)
 x = Dropout(0.5 )(x)
 x = Dense(1024, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 final_output = Dense(n_out, activation='softmax', name='final_output' )(x)
 model = Model(input_tensor, final_output)
 return model
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x<data_type_conversions>
EarlyStopping, ReduceLROnPlateau,CSVLogger)
 epochs = 30; batch_size = 32
checkpoint = ModelCheckpoint('.. /working/densenet_.h5', monitor='val_loss', verbose=1,
save_best_only=True, mode='min', save_weights_only = True)
 reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=4,
verbose=1, mode='auto', epsilon=0.0001)
 early = EarlyStopping(monitor="val_loss",
mode="min",
patience=9)
 csv_logger = CSVLogger(filename='.. /working/training_log.csv',
separator=',',
append=True)
 train_generator = My_Generator(train_x, train_y, 128, is_train=True)
 train_mixup = My_Generator(train_x, train_y, batch_size, is_train=True, mix=False, augment=True)
 valid_generator = My_Generator(valid_x, valid_y, batch_size, is_train=False)
 model = create_model(
 input_shape=(SIZE,SIZE,3),
n_out=NUM_CLASSES )
train_df["question_text"] = train_df["question_text"].str.lower()
test_df["question_text"] = test_df["question_text"].str.lower()
train_df["question_text"] = train_df["question_text"].apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_text(x))<split>
def kappa_loss(y_true, y_pred, y_pow=2, eps=1e-12, N=5, bsize=32, name='kappa'):

 with tf.name_scope(name):
y_true = tf.to_float(y_true)
 repeat_op = tf.to_float(tf.tile(tf.reshape(tf.range(0, N), [N, 1]), [1, N]))
repeat_op_sq = tf.square(( repeat_op - tf.transpose(repeat_op)))
 weights = repeat_op_sq / tf.to_float(( N - 1)** 2)
 pred_ = y_pred ** y_pow
try:
pred_norm = pred_ /(eps + tf.reshape(tf.reduce_sum(pred_, 1), [-1, 1]))
except Exception:
pred_norm = pred_ /(eps + tf.reshape(tf.reduce_sum(pred_, 1), [bsize, 1]))
hist_rater_a = tf.reduce_sum(pred_norm, 0)
 hist_rater_b = tf.reduce_sum(y_true, 0)
 conf_mat = tf.matmul(tf.transpose(pred_norm), y_true)
 nom = tf.reduce_sum(weights * conf_mat)
 denom = tf.reduce_sum(weights * tf.matmul(
 tf.reshape(hist_rater_a, [N, 1]), tf.reshape(hist_rater_b, [1, N])) /
tf.to_float(bsize))
return nom*0.5 /(denom + eps)+ categorical_crossentropy(y_true, y_pred)*0.5
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)
 train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 trunc = 'pre'
train_X = pad_sequences(train_X, maxlen=maxlen, truncating=trunc)
 val_X = pad_sequences(val_X, maxlen=maxlen, truncating=trunc)
 test_X = pad_sequences(test_X, maxlen=maxlen, truncating=trunc)
 train_y = train_df['target'].values
val_y = val_df['target'].values<statistical_test>
class QWKEvaluation(Callback):
def __init__(self, validation_data=() , batch_size=64, interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.batch_size = batch_size
self.valid_generator, self.y_val = validation_data
self.history = []
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict_generator(generator=self.valid_generator,
steps=np.ceil(float(len(self.y_val)) / float(self.batch_size)) ,
workers=1, use_multiprocessing=False,
verbose=1)
 def flatten(y):
return np.argmax(y, axis=1 ).reshape(-1)
 score = cohen_kappa_score(flatten(self.y_val),
flatten(y_pred),
labels=[0,1,2,3,4],
weights='quadratic')
 print("
epoch: %d - QWK_score: %.6f
" %(epoch+1, score))
self.history.append(score)
 if score >= max(self.history):
print('saving checkpoint: ', score)
 self.model.save('.. /working/densenet_bestqwk.h5')
 qwk = QWKEvaluation(validation_data=(valid_generator, valid_y),
batch_size=batch_size, interval=1 )
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_1 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix_1[i] = embedding_vector
del embeddings_index; gc.collect()<statistical_test>
for layer in model.layers:
layer.trainable = False
for i in range(-3,0):
model.layers[i].trainable = True
model.compile(
 loss='categorical_crossentropy',
optimizer=Adam(1e-3))
model.fit_generator(
 train_generator,
steps_per_epoch=np.ceil(float(len(train_y)) / float(128)) ,
epochs=2,
workers=WORKERS, use_multiprocessing=True,
verbose=1,
callbacks=[qwk] )
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_2 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix_2[i] = embedding_vector
del embeddings_index; gc.collect()<statistical_test>
for layer in model.layers:
layer.trainable = True
callbacks_list = [checkpoint, csv_logger, reduceLROnPlat, early, qwk]
model.compile(loss='categorical_crossentropy',
optimizer=Adam(lr=1e-4))
model.fit_generator(
 train_mixup,
steps_per_epoch=np.ceil(float(len(train_x)) / float(batch_size)) ,
validation_data=valid_generator,
validation_steps=np.ceil(float(len(valid_x)) / float(batch_size)) ,
epochs=epochs,
verbose=1,
workers=1, use_multiprocessing=False,
callbacks=callbacks_list )
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_3 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix_3[i] = embedding_vector
del embeddings_index; gc.collect()<features_selection>
submit = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 model.load_weights('.. /working/densenet_bestqwk.h5')
 predicted = []
EMBEDDING_FILE = '.. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'
embeddings_index = KeyedVectors.load_word2vec_format(EMBEDDING_FILE, binary=True)
 word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_4 =(np.random.rand(nb_words, embed_size)- 0.5)/ 5.0
for word, i in word_index.items() :
if i >= max_features: continue
if word in embeddings_index:
embedding_vector = embeddings_index.get_vector(word)
 embedding_matrix_4[i] = embedding_vector
del embeddings_index; gc.collect()<concatenate>
for i, name in tqdm(enumerate(submit['id_code'])) :
path = os.path.join('.. /input/aptos2019-blindness-detection/test_images/', name+'.png')
 image = cv2.imread(path)
 image = cv2.resize(image,(SIZE, SIZE))
X = np.array(( image[np.newaxis])/255)
 score_predict=(( model.predict(X ).ravel() *model.predict(X[:, ::-1, :, :] ).ravel() *model.predict(X[:, ::-1, ::-1, :] ).ravel() *model.predict(X[:, :, ::-1, :] ).ravel())**0.25 ).tolist()
label_predict = np.argmax(score_predict)
 predicted.append(str(label_predict))
<train_model><EOS> 
submit['diagnosis'] = predicted
submit.to_csv('submission.csv', index=False)
 submit.head()