Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<predict_on_test>
def seed_everything(seed=0):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 set_random_seed(seed)
 seed = 0
seed_everything(seed)
 %matplotlib inline
sns.set(style="whitegrid")
 warnings.filterwarnings("ignore" )
pred_val_y, pred_test_y, best_score = train_pred(model_lstm_atten(embedding_matrix_3), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, best_score, '2 LSTM w/ attention Para'] )<predict_on_test>
train = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 test = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 print('Number of train samples: ', train.shape[0])
 print('Number of test samples: ', test.shape[0])
 train["id_code"] = train["id_code"].apply(lambda x: x + ".png")
 test["id_code"] = test["id_code"].apply(lambda x: x + ".png")
 train['diagnosis'] = train['diagnosis'].astype('str')
 display(train.head() )
pred_val_y, pred_test_y, best_score = train_pred(capsule(embedding_matrix), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, best_score, 'capsule'] )<define_variables>
BATCH_SIZE = 8
EPOCHS = 40
WARMUP_EPOCHS = 2
LEARNING_RATE = 1e-4
WARMUP_LEARNING_RATE = 1e-3
HEIGHT = 320
WIDTH = 320
CANAL = 3
N_CLASSES = train['diagnosis'].nunique()
ES_PATIENCE = 5
RLROP_PATIENCE = 3
DECAY_DROP = 0.5
<set_options>
X_train, X_val = train_test_split(train, test_size=0.2, random_state=seed )
color = sns.color_palette()
sns.set_style('dark')
 plt.figure(figsize=(10, 8))
mask = np.zeros_like(dd, dtype=np.bool)
 mask[np.triu_indices_from(mask)] = True
cmap = sns.diverging_palette(220, 10, as_cmap=True)
 g = sns.heatmap(dd, mask=mask, cmap=cmap, square=True, annot=True, fmt='0.2f')
 plt.show()<compute_test_metric>
train_datagen=ImageDataGenerator(rescale=1./255,
rotation_range=360,
horizontal_flip=True,
vertical_flip=True)
 train_generator=train_datagen.flow_from_dataframe(
 dataframe=X_train,
directory=".. /input/aptos2019-blindness-detection/train_images/",
x_col="id_code",
y_col="diagnosis",
class_mode="categorical",
batch_size=BATCH_SIZE,
target_size=(HEIGHT, WIDTH),
seed=0)
 validation_datagen = ImageDataGenerator(rescale=1./255)
 valid_generator=validation_datagen.flow_from_dataframe(
 dataframe=X_val,
directory=".. /input/aptos2019-blindness-detection/train_images/",
x_col="id_code",
y_col="diagnosis",
class_mode="categorical",
batch_size=BATCH_SIZE,
target_size=(HEIGHT, WIDTH),
seed=0)
 test_datagen = ImageDataGenerator(rescale=1./255)
 test_generator = test_datagen.flow_from_dataframe(
 dataframe=test,
directory = ".. /input/aptos2019-blindness-detection/test_images/",
x_col="id_code",
batch_size=1,
class_mode=None,
shuffle=False,
target_size=(HEIGHT, WIDTH),
seed=0 )
outputs.sort(key=lambda x: x[2])
 weights = [i for i in range(1, len(outputs)+ 1)]
weights = [float(i)/ sum(weights)for i in weights]
print(weights)
 for output in outputs:
print(output[2], output[3])
 X = np.asarray([outputs[i][0] for i in range(len(outputs)) ])
 X = X[...,0]
reg = LinearRegression().fit(X.T, val_y)
 print(reg.score(X.T, val_y),reg.coef_)
 pred_val_y = np.sum([outputs[i][0] * reg.coef_[i] for i in range(len(outputs)) ], axis = 0)
 thresholds = []
for thresh in np.arange(0.2, 0.601, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(val_y,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh)
 pred_test_y = np.sum([outputs[i][1] * reg.coef_[i] for i in range(len(outputs)) ], axis = 0)
 pred_test_y =(pred_test_y > best_thresh ).astype(int)
 test_df = pd.read_csv(".. /input/test.csv", usecols=["qid"])
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<define_variables>
def create_model(input_shape, n_out):
input_tensor = Input(shape=input_shape)
 base_model = applications.ResNet50(weights=None,
include_top=False,
input_tensor=input_tensor)
 base_model.load_weights('.. /input/resnet50/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5')
 x = GlobalAveragePooling2D()(base_model.output)
 x = Dropout(0.5 )(x)
 x = Dense(2048, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 final_output = Dense(n_out, activation='softmax', name='final_output' )(x)
 model = Model(input_tensor, final_output)
 return model
seed = 123<load_from_csv>
STEP_SIZE_TRAIN = train_generator.n//train_generator.batch_size
STEP_SIZE_VALID = valid_generator.n//valid_generator.batch_size
history_warmup = model.fit_generator(generator=train_generator,
steps_per_epoch=STEP_SIZE_TRAIN,
validation_data=valid_generator,
validation_steps=STEP_SIZE_VALID,
epochs=WARMUP_EPOCHS,
class_weight=class_weights,
verbose=1 ).history
class DataLoader:
def load(self, file_names):
self.train_df = pd.read_csv(file_names['train'])
 self.test_df = pd.read_csv(file_names['test'])
 self.train_df['question_text'].fillna('_na_', inplace=True)
 self.test_df['question_text'].fillna('_na_', inplace=True)
 print('Train shape : ', self.train_df.shape)
 print('Test shape : ', self.test_df.shape )<feature_engineering>
for layer in model.layers:
layer.trainable = True
es = EarlyStopping(monitor='val_loss', mode='min', patience=ES_PATIENCE, restore_best_weights=True, verbose=1)
 rlrop = ReduceLROnPlateau(monitor='val_loss', mode='min', patience=RLROP_PATIENCE, factor=DECAY_DROP, min_lr=1e-6, verbose=1)
 callback_list = [es, rlrop]
optimizer = optimizers.Adam(lr=LEARNING_RATE)
 model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=metric_list)
 model.summary()
class Embeddings:
def __init__(self):
self.embed_len = 300
self.punctuation = set('!
def load(self, embedding_file_name):
self.embeddings_index = {}
f = open(embedding_file_name)
 for line in tqdm(f):
values = line.split(' ');
word = values[0]
coef = np.asarray(values[1:], dtype='float32')
 self.embeddings_index[word] = coef
self.embed_len = len(coef)
 f.close()
def text_to_vec(self, text, max_text_len=30):
if text[-1] in self.punctuation:
text = text[:-1].split() + [text[-1]]
else:
text = text.split()
text = text[:max_text_len]
empyt_emb = np.zeros(self.embed_len)
 embeds = [self.embeddings_index.get(word, empyt_emb)for word in text]
embeds+= [empyt_emb] *(max_text_len - len(embeds))
return np.array(embeds)
 def sequences_to_vec(self, text_sequences, max_text_len=30):
vectors = [self.text_to_vec(text, max_text_len)for text in text_sequences]
return np.array(vectors )<split>
history_finetunning = model.fit_generator(generator=train_generator,
steps_per_epoch=STEP_SIZE_TRAIN,
validation_data=valid_generator,
validation_steps=STEP_SIZE_VALID,
epochs=EPOCHS,
callbacks=callback_list,
class_weight=class_weights,
verbose=1 ).history
def batch_gen(train_df, batch_size, emb, max_text_len=30):
n_batches = math.ceil(len(train_df)/ batch_size)
 while True:
train_df = train_df.sample(frac=1.)
for i in range(n_batches):
batch_df = train_df.iloc[i*batch_size:(i+1)*batch_size]
X_ = emb.sequences_to_vec(batch_df["question_text"], max_text_len)
 y_ = batch_df["target"].values
yield X_, y_
def batch_gen_test(test_df, emb, batch_size_test=256, max_text_len=30):
n_batches = math.ceil(len(test_df)/ batch_size)
 for i in range(n_batches):
batch_df = test_df.iloc[i*batch_size:(i+1)*batch_size]
X_ = emb.sequences_to_vec(batch_df["question_text"], max_text_len)
 yield X_<load_from_csv>
lastFullTrainPred = np.empty(( 0, N_CLASSES))
lastFullTrainLabels = np.empty(( 0, N_CLASSES))
lastFullValPred = np.empty(( 0, N_CLASSES))
lastFullValLabels = np.empty(( 0, N_CLASSES))
for i in range(STEP_SIZE_TRAIN+1):
im, lbl = next(train_generator)
 scores = model.predict(im, batch_size=train_generator.batch_size)
 lastFullTrainPred = np.append(lastFullTrainPred, scores, axis=0)
 lastFullTrainLabels = np.append(lastFullTrainLabels, lbl, axis=0)
 for i in range(STEP_SIZE_VALID+1):
im, lbl = next(valid_generator)
 scores = model.predict(im, batch_size=valid_generator.batch_size)
 lastFullValPred = np.append(lastFullValPred, scores, axis=0)
 lastFullValLabels = np.append(lastFullValLabels, lbl, axis=0)
 lastFullComPred = np.concatenate(( lastFullTrainPred, lastFullValPred))
lastFullComLabels = np.concatenate(( lastFullTrainLabels, lastFullValLabels))
complete_labels = [np.argmax(label)for label in lastFullComLabels]
train_preds = [np.argmax(pred)for pred in lastFullTrainPred]
train_labels = [np.argmax(label)for label in lastFullTrainLabels]
validation_preds = [np.argmax(pred)for pred in lastFullValPred]
validation_labels = [np.argmax(label)for label in lastFullValLabels]
file_names = {'train': '.. /input/train.csv', 'test': '.. /input/test.csv'}
dl = DataLoader()
dl.load(file_names )<load_pretrained>
fig,(ax1, ax2)= plt.subplots(1, 2, sharex='col', figsize=(24, 7))
labels = ['0 - No DR', '1 - Mild', '2 - Moderate', '3 - Severe', '4 - Proliferative DR']
train_cnf_matrix = confusion_matrix(train_labels, train_preds)
 validation_cnf_matrix = confusion_matrix(validation_labels, validation_preds)
 train_cnf_matrix_norm = train_cnf_matrix.astype('float')/ train_cnf_matrix.sum(axis=1)[:, np.newaxis]
validation_cnf_matrix_norm = validation_cnf_matrix.astype('float')/ validation_cnf_matrix.sum(axis=1)[:, np.newaxis]
train_df_cm = pd.DataFrame(train_cnf_matrix_norm, index=labels, columns=labels)
 validation_df_cm = pd.DataFrame(validation_cnf_matrix_norm, index=labels, columns=labels)
 sns.heatmap(train_df_cm, annot=True, fmt='.2f', cmap="Blues", ax=ax1 ).set_title('Train')
 sns.heatmap(validation_df_cm, annot=True, fmt='.2f', cmap=sns.cubehelix_palette(8), ax=ax2 ).set_title('Validation')
 plt.show()
embedding_file_name = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
emb = Embeddings()
emb.load(embedding_file_name)
 print('Embedding loaded' )<split>
print("Train Cohen Kappa score: %.3f" % cohen_kappa_score(train_preds,train_labels, weights='quadratic'))
print("Validation Cohen Kappa score: %.3f" % cohen_kappa_score(validation_preds, validation_labels, weights='quadratic'))
print("Complete set Cohen Kappa score: %.3f" % cohen_kappa_score(train_preds+validation_preds, train_labels+validation_labels, weights='quadratic'))
val_size = 0.03
train_df, val_df, _, _ = train_test_split(dl.train_df, dl.train_df['target'],
test_size=val_size,
stratify=dl.train_df['target'],
shuffle=True, random_state=seed)
 val_X = emb.sequences_to_vec(val_df["question_text"], max_text_len)
 val_y = val_df['target'].values<import_modules>
test_generator.reset()
STEP_SIZE_TEST = test_generator.n//test_generator.batch_size
preds = model.predict_generator(test_generator, steps=STEP_SIZE_TEST)
 predictions = [np.argmax(pred)for pred in preds]
filenames = test_generator.filenames
results = pd.DataFrame({'id_code':filenames, 'diagnosis':predictions})
 results['id_code'] = results['id_code'].map(lambda x: str(x)[:-4] )
from keras.models import Sequential
from keras.layers import CuDNNLSTM, Dense, Bidirectional<choose_model_class>
results.to_csv('submission.csv', index=False)
 results.head(10 )