Kaggle competition: RSNA Pneumonia Detection Challenge
<SOS> metric: RSNAObjectDetectionAP Kaggle data source: rsna-pneumonia-detection-challenge<categorify>
import os
import csv
import random
import pydicom
import numpy as np
import pandas as pd
from skimage import measure
from skimage.transform import resize
import tensorflow as tf
from tensorflow import keras
from matplotlib import pyplot as plt
train_input = bert_encode(X_train.text.values, tokenizer, max_len=maxlen)
 val_input = bert_encode(X_val.text.values, tokenizer, max_len=maxlen)
 train_labels = y_train.target.values
val_labels = y_val.target.values<choose_model_class>
pneumonia_locations = {}
with open(os.path.join('.. /input/stage_1_train_labels.csv'), mode='r')as infile:
reader = csv.reader(infile)
 next(reader, None)
 for rows in reader:
filename = rows[0]
location = rows[1:5]
pneumonia = rows[5]
if pneumonia == '1':
location = [int(float(i)) for i in location]
if filename in pneumonia_locations:
pneumonia_locations[filename].append(location)
 else:
pneumonia_locations[filename] = [location]
bert_model = build_model(bert_layer, max_len=maxlen, lr=lr)
 bert_model.summary()<train_model>
folder = '.. /input/stage_1_train_images'
filenames = os.listdir(folder)
 random.shuffle(filenames)
 n_valid_samples = 1284
train_filenames = filenames[n_valid_samples:]
valid_filenames = filenames[:n_valid_samples]
print('n train samples', len(train_filenames))
print('n valid samples', len(valid_filenames))
n_train_samples = len(filenames)- n_valid_samples
checkpoint = ModelCheckpoint('bertmodel.h5', monitor='val_accuracy', save_best_only=True)
 bert_history = bert_model.fit(
 train_input, train_labels,
validation_data=(val_input, val_labels),
epochs=num_epochs,
callbacks=[checkpoint],
batch_size=batch_size
)<train_model>
class generator(keras.utils.Sequence):
def __init__(self, folder, filenames, pneumonia_locations=None, batch_size=32, image_size=320, shuffle=True, augment=False, predict=False):
self.folder = folder
self.filenames = filenames
self.pneumonia_locations = pneumonia_locations
self.batch_size = batch_size
self.image_size = image_size
self.shuffle = shuffle
self.augment = augment
self.predict = predict
self.on_epoch_end()
def __load__(self, filename):
img = pydicom.dcmread(os.path.join(self.folder, filename)).pixel_array
msk = np.zeros(img.shape)
 filename = filename.split('.')[0]
if filename in pneumonia_locations:
for location in pneumonia_locations[filename]:
x, y, w, h = location
msk[y:y+h, x:x+w] = 1
if self.augment and random.random() > 0.5:
img = np.fliplr(img)
 msk = np.fliplr(msk)
 img = resize(img,(self.image_size, self.image_size), mode='reflect')
 msk = resize(msk,(self.image_size, self.image_size), mode='reflect')> 0.5
img = np.expand_dims(img, -1)
 msk = np.expand_dims(msk, -1)
 return img, msk
def __loadpredict__(self, filename):
img = pydicom.dcmread(os.path.join(self.folder, filename)).pixel_array
img = resize(img,(self.image_size, self.image_size), mode='reflect')
 img = np.expand_dims(img, -1)
 return img
def __getitem__(self, index):
filenames = self.filenames[index*self.batch_size:(index+1)*self.batch_size]
if self.predict:
imgs = [self.__loadpredict__(filename)for filename in filenames]
imgs = np.array(imgs)
 return imgs, filenames
else:
items = [self.__load__(filename)for filename in filenames]
imgs, msks = zip(*items)
 imgs = np.array(imgs)
 msks = np.array(msks)
 return imgs, msks
def on_epoch_end(self):
if self.shuffle:
random.shuffle(self.filenames)
 def __len__(self):
if self.predict:
return int(np.ceil(len(self.filenames)/ self.batch_size))
else:
return int(len(self.filenames)/ self.batch_size )
clf = RandomForestClassifier(n_estimators=500, max_depth=15, min_samples_split=20, min_samples_leaf=2, n_jobs=-1, random_state=0)
 clf.fit(X_train.drop('text',axis=1), y_train.target.values)
 clf_pred = clf.predict_proba(X_val.drop('text',axis=1))
print('Validation Accuracy:', accuracy_score(y_val.target.values, clf_pred.argmax(axis=-1)) )<predict_on_test>
BATCH_SIZE = 16
IMAGE_SIZE = 320
bert_model.load_weights('bertmodel.h5')
 val = X_val.copy()
val = val[['text']]
val['target'] = y_val
val['pred'] = bert_model.predict(val_input)
 val['pred'] = val['pred'].apply(lambda x: 1 if x >=0.5 else 0)
 error = val[val['target'] != val['pred']]
error.head()<feature_engineering>
def create_downsample(channels, inputs):
x = keras.layers.BatchNormalization(momentum=0.9 )(inputs)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 1, padding='same', use_bias=False )(x)
 x = keras.layers.MaxPool2D(2 )(x)
 return x
def create_resblock(channels, inputs):
x = keras.layers.BatchNormalization(momentum=0.9 )(inputs)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(x)
 x = keras.layers.BatchNormalization(momentum=0.9 )(x)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(x)
 return keras.layers.add([x, inputs])
 def create_network(input_size, channels, n_blocks=2, depth=4):
inputs = keras.Input(shape=(input_size, input_size, 1))
x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(inputs)
 for d in range(depth):
channels = channels * 2
x = create_downsample(channels, x)
 for b in range(n_blocks):
x = create_resblock(channels, x)
 x = keras.layers.BatchNormalization(momentum=0.9 )(x)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(256, 1, activation=None )(x)
 x = keras.layers.BatchNormalization(momentum=0.9 )(x)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2DTranspose(128,(8,8),(4,4), padding="same", activation=None )(x)
 x = keras.layers.BatchNormalization(momentum=0.9 )(x)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(1, 1, activation='sigmoid' )(x)
 outputs = keras.layers.UpSampling2D(2**(depth-2))(x)
 model = keras.Model(inputs=inputs, outputs=outputs)
 return model
test['char_len'] = test.text.str.len()
word_tokens = [len(word_tokenize(tweet)) for tweet in test.text]
test['word_len'] = word_tokens
sent_tokens = [len(sent_tokenize(tweet)) for tweet in test.text]
test['sent_len'] = sent_tokens<feature_engineering>
def iou_loss(y_true, y_pred):
y_true = tf.reshape(y_true, [-1])
 y_pred = tf.reshape(y_pred, [-1])
 intersection = tf.reduce_sum(y_true * y_pred)
 score =(intersection + 1.) /(tf.reduce_sum(y_true)+ tf.reduce_sum(y_pred)- intersection + 1.)
return 1 - score
def iou_bce_loss(y_true, y_pred):
return 0.5 * keras.losses.binary_crossentropy(y_true, y_pred)+ 0.5 * iou_loss(y_true, y_pred)
 def mean_iou(y_true, y_pred):
y_pred = tf.round(y_pred)
 intersect = tf.reduce_sum(y_true * y_pred, axis=[1, 2, 3])
 union = tf.reduce_sum(y_true, axis=[1, 2, 3])+ tf.reduce_sum(y_pred, axis=[1, 2, 3])
 smooth = tf.ones(tf.shape(intersect))
return tf.reduce_mean(( intersect + smooth)/(union - intersect + smooth))
model = create_network(input_size=IMAGE_SIZE, channels=32, n_blocks=2, depth=4)
 model.compile(optimizer='adam',
loss=iou_bce_loss,
metrics=['accuracy', mean_iou])
 def cosine_annealing(x):
lr = 0.001
epochs = 20
return lr*(np.cos(np.pi*x/epochs)+1.) /2
learning_rate = tf.keras.callbacks.LearningRateScheduler(cosine_annealing)
 folder = '.. /input/stage_1_train_images'
train_gen = generator(folder, train_filenames, pneumonia_locations, batch_size=BATCH_SIZE, image_size=IMAGE_SIZE, shuffle=True, augment=True, predict=False)
 valid_gen = generator(folder, valid_filenames, pneumonia_locations, batch_size=BATCH_SIZE, image_size=IMAGE_SIZE, shuffle=False, predict=False)
 print(model.summary() )
test['polarity'] = [TextBlob(tweet ).sentiment.polarity for tweet in test.text]
test['subjectivity'] = [TextBlob(tweet ).sentiment.subjectivity for tweet in test.text]
test['exclaimation_num'] = [tweet.count('!')for tweet in test.text]
test['questionmark_num'] = [tweet.count('?')for tweet in test.text]
def count_url_hashtag_mention(text):
urls_num = len(re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F])) +', text))
word_tokens = text.split()
hash_num = len([word for word in word_tokens if word[0] == '
mention_num = len([word for word in word_tokens if word[0] == '@' and word.count('@')== 1])
 return urls_num, hash_num, mention_num
url_num, hash_num, mention_num = zip(*[count_url_hashtag_mention(tweet)for tweet in test.text])
 test = test.assign(url_num = url_num, hash_num = hash_num, mention_num = mention_num)
 contractions = ["'t", "'re", "'s", "'d", "'ll", "'ve", "'m"]
test['contraction_num'] = [sum([tweet.count(cont)for cont in contractions])for tweet in test.text]<categorify>
history = model.fit_generator(train_gen, validation_data=valid_gen, callbacks=[learning_rate], epochs=15, shuffle=True )
<categorify><EOS> 
folder = '.. /input/stage_1_test_images'
test_filenames = os.listdir(folder)
 print('n test samples:', len(test_filenames))
test_gen = generator(folder, test_filenames, None, batch_size=16, image_size=IMAGE_SIZE, shuffle=False, predict=True)
 submission_dict = {}
for imgs, filenames in test_gen:
preds = model.predict(imgs)
 for pred, filename in zip(preds, filenames):
pred = resize(pred,(1024, 1024), mode='reflect')
 comp = pred[:, :, 0] > 0.5
comp = measure.label(comp)
 predictionString = ''
for region in measure.regionprops(comp):
y, x, y2, x2 = region.bbox
height = y2 - y
width = x2 - x
conf = np.mean(pred[y:y+height, x:x+width])
 predictionString += str(conf)+ ' ' + str(x)+ ' ' + str(y)+ ' ' + str(width)+ ' ' + str(height)+ ' '
filename = filename.split('.')[0]
submission_dict[filename] = predictionString
if len(submission_dict)>= len(test_filenames):
break
print("Done predicting...")
 sub = pd.DataFrame.from_dict(submission_dict,orient='index')
 sub.index.names = ['patientId']
sub.columns = ['PredictionString']
sub.to_csv('submission.csv' )