Kaggle competition: RSNA Pneumonia Detection Challenge
<SOS> metric: RSNAObjectDetectionAP Kaggle data source: rsna-pneumonia-detection-challenge<compute_train_metric>
import os
import csv
import random
import pydicom
import numpy as np
import pandas as pd
from skimage import io
from skimage import measure
from skimage.transform import resize
import tensorflow as tf
from tensorflow import keras
from matplotlib import pyplot as plt
import matplotlib.patches as patches
ts = time.time()
random_forest = RandomForestClassifier()
scoring = ['accuracy', 'precision_macro', 'recall_macro' , 'f1_weighted', 'roc_auc']
scores = cross_validate(random_forest, X_train, y_train, scoring=scoring, cv=5)
 sorted(scores.keys())
 forest_fit_time = scores['fit_time'].mean()
forest_score_time = scores['score_time'].mean()
forest_accuracy = scores['test_accuracy'].mean()
forest_precision = scores['test_precision_macro'].mean()
forest_recall = scores['test_recall_macro'].mean()
forest_f1 = scores['test_f1_weighted'].mean()
forest_roc = scores['test_roc_auc'].mean()
print("Time spent: ", time.time() -ts )<compute_train_metric>
pneumonia_locations = {}
with open(os.path.join('.. /input/stage_1_train_labels.csv'), mode='r')as infile:
reader = csv.reader(infile)
 next(reader, None)
 for rows in reader:
filename = rows[0]
location = rows[1:5]
pneumonia = rows[5]
if pneumonia == '1':
location = [int(float(i)) for i in location]
if filename in pneumonia_locations:
pneumonia_locations[filename].append(location)
 else:
pneumonia_locations[filename] = [location]
ts = time.time()
KNN = KNeighborsClassifier()
scoring = ['accuracy', 'precision_macro', 'recall_macro' , 'f1_weighted', 'roc_auc']
scores = cross_validate(KNN, X_train, y_train, scoring=scoring, cv=5)
 sorted(scores.keys())
 KNN_fit_time = scores['fit_time'].mean()
KNN_score_time = scores['score_time'].mean()
KNN_accuracy = scores['test_accuracy'].mean()
KNN_precision = scores['test_precision_macro'].mean()
KNN_recall = scores['test_recall_macro'].mean()
KNN_f1 = scores['test_f1_weighted'].mean()
KNN_roc = scores['test_roc_auc'].mean()
print("Time spent: ", time.time() -ts )<compute_train_metric>
folder = '.. /input/stage_1_train_images'
filenames = os.listdir(folder)
 random.shuffle(filenames)
 n_valid_samples = 2560
train_filenames = filenames[n_valid_samples:]
valid_filenames = filenames[:n_valid_samples]
print('n train samples', len(train_filenames))
print('n valid samples', len(valid_filenames))
n_train_samples = len(filenames)- n_valid_samples
ts = time.time()
bayes = GaussianNB()
scoring = ['accuracy', 'precision_macro', 'recall_macro' , 'f1_weighted', 'roc_auc']
scores = cross_validate(bayes, X_train, y_train, scoring=scoring, cv=5)
 sorted(scores.keys())
 bayes_fit_time = scores['fit_time'].mean()
bayes_score_time = scores['score_time'].mean()
bayes_accuracy = scores['test_accuracy'].mean()
bayes_precision = scores['test_precision_macro'].mean()
bayes_recall = scores['test_recall_macro'].mean()
bayes_f1 = scores['test_f1_weighted'].mean()
bayes_roc = scores['test_roc_auc'].mean()
print("Time spent: ", time.time() -ts )<train_model>
class generator(keras.utils.Sequence):
def __init__(self, folder, filenames, pneumonia_locations=None, batch_size=32, image_size=256, shuffle=True, augment=False, predict=False):
self.folder = folder
self.filenames = filenames
self.pneumonia_locations = pneumonia_locations
self.batch_size = batch_size
self.image_size = image_size
self.shuffle = shuffle
self.augment = augment
self.predict = predict
self.on_epoch_end()
def __load__(self, filename):
img = pydicom.dcmread(os.path.join(self.folder, filename)).pixel_array
msk = np.zeros(img.shape)
 filename = filename.split('.')[0]
if filename in self.pneumonia_locations:
for location in self.pneumonia_locations[filename]:
x, y, w, h = location
msk[y:y+h, x:x+w] = 1
img = resize(img,(self.image_size, self.image_size), mode='reflect')
 msk = resize(msk,(self.image_size, self.image_size), mode='reflect')> 0.5
if self.augment and random.random() > 0.5:
img = np.fliplr(img)
 msk = np.fliplr(msk)
 img = np.expand_dims(img, -1)
 msk = np.expand_dims(msk, -1)
 return img, msk
def __loadpredict__(self, filename):
img = pydicom.dcmread(os.path.join(self.folder, filename)).pixel_array
img = resize(img,(self.image_size, self.image_size), mode='reflect')
 img = np.expand_dims(img, -1)
 return img
def __getitem__(self, index):
filenames = self.filenames[index*self.batch_size:(index+1)*self.batch_size]
if self.predict:
imgs = [self.__loadpredict__(filename)for filename in filenames]
imgs = np.array(imgs)
 return imgs, filenames
else:
items = [self.__load__(filename)for filename in filenames]
imgs, msks = zip(*items)
 imgs = np.array(imgs)
 msks = np.array(msks)
 return imgs, msks
def on_epoch_end(self):
if self.shuffle:
random.shuffle(self.filenames)
 def __len__(self):
if self.predict:
return int(np.ceil(len(self.filenames)/ self.batch_size))
else:
return int(len(self.filenames)/ self.batch_size )
models_initial = pd.DataFrame({
'Model' : ['Logistic Regression', 'Support Vector Machine', 'Linear Discriminant Analysis', 'Quadratic Discriminant Analysis', 'Random Forest', 'K-Nearest Neighbors', 'Bayes'],
'Fitting time': [LR_fit_time, SVM_fit_time, LDA_fit_time, QDA_fit_time, forest_fit_time, KNN_fit_time, bayes_fit_time],
'Scoring time': [LR_score_time, SVM_score_time, LDA_score_time, QDA_score_time, forest_score_time, KNN_score_time, bayes_score_time],
'Accuracy' : [LR_accuracy, SVM_accuracy, LDA_accuracy, QDA_accuracy, forest_accuracy, KNN_accuracy, bayes_accuracy],
'Precision' : [LR_precision, SVM_precision, LDA_precision, QDA_precision, forest_precision, KNN_precision, bayes_precision],
'Recall' : [LR_recall, SVM_recall, LDA_recall, QDA_recall, forest_recall, KNN_recall, bayes_recall],
'F1_score' : [LR_f1,SVM_f1, LDA_f1, QDA_f1, forest_f1, KNN_f1, bayes_f1],
'AUC_ROC' : [LR_roc, SVM_roc, LDA_roc, QDA_roc, forest_roc, KNN_roc, bayes_roc],
}, columns = ['Model', 'Fitting time', 'Scoring time', 'Accuracy', 'Precision', 'Recall', 'F1_score', 'AUC_ROC'])
 models_initial.sort_values(by='AUC_ROC', ascending=False )<train_model>
def create_downsample(channels, inputs):
x = keras.layers.BatchNormalization(momentum=0.9 )(inputs)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 1, padding='same', use_bias=False )(x)
 x = keras.layers.MaxPool2D(2 )(x)
 return x
def create_resblock(channels, inputs):
x = keras.layers.BatchNormalization(momentum=0.9 )(inputs)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(x)
 x = keras.layers.BatchNormalization(momentum=0.9 )(x)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(x)
 return keras.layers.add([x, inputs])
 def create_network(input_size, channels, n_blocks=2, depth=4):
inputs = keras.Input(shape=(input_size, input_size, 1))
x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(inputs)
 for d in range(depth):
channels = channels * 2
x = create_downsample(channels, x)
 for b in range(n_blocks):
x = create_resblock(channels, x)
 x = keras.layers.BatchNormalization(momentum=0.9 )(x)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(1, 1, activation='sigmoid' )(x)
 outputs = keras.layers.UpSampling2D(2**depth )(x)
 model = keras.Model(inputs=inputs, outputs=outputs)
 return model
def optimize_lgb(X_train, y_train, X_valid, y_valid):
n_HP_points_to_test = 100
fit_params={"early_stopping_rounds":30,
"eval_metric" : 'auc',
"eval_set" : [(X_valid, y_valid)],
'eval_names': ['valid'],
'verbose': 0,
'categorical_feature': 'auto'}
param_test ={'max_depth': [4,5,6,7],
'num_leaves': sp_randint(6, 50),
'min_child_samples': sp_randint(100, 500),
'min_child_weight': [1e-5, 1e-3, 1e-2, 1e-1, 1, 1e1, 1e2, 1e3, 1e4],
'subsample': sp_uniform(loc=0.2, scale=0.8),
'colsample_bytree': sp_uniform(loc=0.4, scale=0.6),
'reg_alpha': [0, 1e-1, 1, 2, 5, 7, 10, 50, 100],
'reg_lambda': [0, 1e-1, 1, 5, 10, 20, 50, 100]}
clf = lgb.LGBMClassifier(max_depth=-1, random_state=21, silent=True, metric='auc', n_jobs=4, n_estimators=1000)
 gs = RandomizedSearchCV(
 estimator=clf, param_distributions=param_test,
n_iter=n_HP_points_to_test,
scoring='roc_auc',
cv=5,
refit=True,
random_state=21,
verbose=0)
 gs.fit(X_train, y_train, **fit_params)
 print('Best score reached: {} with params: {} '.format(gs.best_score_, gs.best_params_))
return gs, gs.best_score_, gs.best_params_, fit_params
X_train, X_valid, y_train, y_valid = split_dataset(train, 0.2)
 clf_1, best_score_1, optimal_params_1, fit_params = optimize_lgb(X_train, y_train, X_valid, y_valid )<remove_duplicates>
def iou_loss(y_true, y_pred):
y_true = tf.reshape(y_true, [-1])
 y_pred = tf.reshape(y_pred, [-1])
 intersection = tf.reduce_sum(y_true * y_pred)
 score =(intersection + 1.) /(tf.reduce_sum(y_true)+ tf.reduce_sum(y_pred)- intersection + 1.)
return 1 - score
def iou_bce_loss(y_true, y_pred):
return 0.5 * keras.losses.binary_crossentropy(y_true, y_pred)+ 0.5 * iou_loss(y_true, y_pred)
 def mean_iou(y_true, y_pred):
y_pred = tf.round(y_pred)
 intersect = tf.reduce_sum(y_true * y_pred, axis=[1, 2, 3])
 union = tf.reduce_sum(y_true, axis=[1, 2, 3])+ tf.reduce_sum(y_pred, axis=[1, 2, 3])
 smooth = tf.ones(tf.shape(intersect))
return tf.reduce_mean(( intersect + smooth)/(union - intersect + smooth))
model = create_network(input_size=256, channels=32, n_blocks=2, depth=4)
 model.compile(optimizer='adam',
loss=iou_bce_loss,
metrics=['accuracy', mean_iou])
 def cosine_annealing(x):
lr = 0.001
epochs = 25
return lr*(np.cos(np.pi*x/epochs)+1.) /2
learning_rate = tf.keras.callbacks.LearningRateScheduler(cosine_annealing)
 folder = '.. /input/stage_1_train_images'
train_gen = generator(folder, train_filenames, pneumonia_locations, batch_size=32, image_size=256, shuffle=True, augment=True, predict=False)
 valid_gen = generator(folder, valid_filenames, pneumonia_locations, batch_size=32, image_size=256, shuffle=False, predict=False)
 history = model.fit_generator(train_gen, validation_data=valid_gen, callbacks=[learning_rate], epochs=25, workers=4, use_multiprocessing=True )
def clean_duplicates(df1, df2):
remove = []
cols = df1.columns
for i in range(len(cols)-1):
v = df1[cols[i]].values
for j in range(i+1,len(cols)) :
if np.array_equal(v,df1[cols[j]].values):
remove.append(cols[j])
 df1.drop(remove, axis=1, inplace=True)
 df2.drop(remove, axis=1, inplace=True)
 return df1, df2
def clean_constant_columns(df1, df2, threshold):
constant_cols = []
for i in df1.columns:
counts = df1[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(df1)* 100 > threshold:
constant_cols.append(i)
 df1 = df1.drop(constant_cols, axis=1)
 df2 = df2.drop(constant_cols, axis=1)
 return df1, df2
train_df = train.copy()
test_df = test.copy()
train_df.var3.replace(-999999, -0.5, inplace=True)
 test_df.var3.replace(-999999, -0.5, inplace=True)
 train_id = train_df['ID']
test_id = test['ID']
train_df.drop('ID', axis=1, inplace=True)
 test_df.drop('ID', axis=1, inplace=True)
 train_df, test_df = clean_duplicates(train_df, test_df)
 train_df, test_df = clean_constant_columns(train_df, test_df, 99.9)
 X_train, X_valid, y_train, y_valid = split_dataset(train_df, 0.20)
 clf_2, best_score_2, optimal_params_2, fit_params = optimize_lgb(X_train, y_train, X_valid, y_valid )<count_unique_values>
for imgs, msks in valid_gen:
preds = model.predict(imgs)
 f, axarr = plt.subplots(4, 8, figsize=(20,15))
axarr = axarr.ravel()
axidx = 0
for img, msk, pred in zip(imgs, msks, preds):
axarr[axidx].imshow(img[:, :, 0])
 comp = msk[:, :, 0] > 0.5
comp = measure.label(comp)
 predictionString = ''
for region in measure.regionprops(comp):
y, x, y2, x2 = region.bbox
height = y2 - y
width = x2 - x
axarr[axidx].add_patch(patches.Rectangle(( x,y),width,height,linewidth=2,edgecolor='b',facecolor='none'))
comp = pred[:, :, 0] > 0.5
comp = measure.label(comp)
 predictionString = ''
for region in measure.regionprops(comp):
y, x, y2, x2 = region.bbox
height = y2 - y
width = x2 - x
axarr[axidx].add_patch(patches.Rectangle(( x,y),width,height,linewidth=2,edgecolor='r',facecolor='none'))
axidx += 1
plt.show()
break
<choose_model_class><EOS> 
folder = '.. /input/stage_1_test_images'
test_filenames = os.listdir(folder)
 print('n test samples:', len(test_filenames))
test_gen = generator(folder, test_filenames, None, batch_size=25, image_size=256, shuffle=False, predict=True)
 submission_dict = {}
for imgs, filenames in test_gen:
preds = model.predict(imgs)
 for pred, filename in zip(preds, filenames):
pred = resize(pred,(1024, 1024), mode='reflect')
 comp = pred[:, :, 0] > 0.5
comp = measure.label(comp)
 predictionString = ''
for region in measure.regionprops(comp):
y, x, y2, x2 = region.bbox
height = y2 - y
width = x2 - x
conf = np.mean(pred[y:y+height, x:x+width])
 predictionString += str(conf)+ ' ' + str(x)+ ' ' + str(y)+ ' ' + str(width)+ ' ' + str(height)+ ' '
filename = filename.split('.')[0]
submission_dict[filename] = predictionString
if len(submission_dict)>= len(test_filenames):
break
sub = pd.DataFrame.from_dict(submission_dict,orient='index')
 sub.index.names = ['patientId']
sub.columns = ['PredictionString']
sub.to_csv('submission.csv' )