Kaggle competition: SIIM-ACR Pneumothorax Segmentation
<SOS> metric: SIIMDice Kaggle data source: siim-acr-pneumothorax-segmentation<predict_on_test>
!pip install albumentations > /dev/null
!pip install -U efficientnet==0.0.4
plt.style.use('seaborn-white')
 sns.set_style("white")
 seed = 10
np.random.seed(seed)
 random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.set_random_seed(seed)
 %matplotlib inline
test_preds = model.predict(testdat)
 print(test_preds.shape)
 test_preds = np.array([np.argmax(y)for y in test_preds])
 print(test_preds.shape)
<define_variables>
!mkdir masks
!unzip -q.. /input/data-repack-and-image-statistics/masks.zip -d masks
!mkdir train
!unzip -q.. /input/data-repack-and-image-statistics/train.zip -d train
!mkdir test
!unzip -q.. /input/data-repack-and-image-statistics/test.zip -d test
testId = [*range(1, len(test_preds)+1)]<save_to_csv>
all_mask_fn = glob.glob('./masks/*')
 mask_df = pd.DataFrame()
mask_df['file_names'] = all_mask_fn
mask_df['mask_percentage'] = 0
mask_df.set_index('file_names',inplace=True)
 for fn in all_mask_fn:
mask_df.loc[fn,'mask_percentage'] = np.array(Image.open(fn)).sum() /(256*256*255)
 mask_df.reset_index(inplace=True)
 sns.distplot(mask_df.mask_percentage)
 mask_df['labels'] = 0
mask_df.loc[mask_df.mask_percentage>0,'labels'] = 1
output = pd.DataFrame({'ImageId': testId,
'Label': test_preds})
 output.to_csv('submission.csv', index=False)
<load_from_csv>
all_train_fn = glob.glob('./train/*')
 total_samples = len(all_train_fn)
 idx = np.arange(total_samples)
 train_fn,val_fn = train_test_split(all_train_fn,stratify=mask_df.labels,test_size=0.1,random_state=10)
 print('No.of train files:', len(train_fn))
print('No.of val files:', len(val_fn))
masks_train_fn = [fn.replace('./train','./masks')for fn in train_fn]
masks_val_fn = [fn.replace('./train','./masks')for fn in val_fn]
train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 print(train.shape)
 train.head()<load_from_csv>
train_im_path,train_mask_path = './keras_im_train','./keras_mask_train'
h,w,batch_size = 256,256,16
val_im_path,val_mask_path = './keras_im_val','./keras_mask_val'
class DataGenerator(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, train_im_path=train_im_path,train_mask_path=train_mask_path,
augmentations=None, batch_size=batch_size,img_size=256, n_channels=3, shuffle=True):
'Initialization'
self.batch_size = batch_size
self.train_im_paths = glob.glob(train_im_path+'/*')
 self.train_im_path = train_im_path
self.train_mask_path = train_mask_path
self.img_size = img_size
self.n_channels = n_channels
self.shuffle = shuffle
self.augment = augmentations
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.ceil(len(self.train_im_paths)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[index*self.batch_size:min(( index+1)*self.batch_size,len(self.train_im_paths)) ]
list_IDs_im = [self.train_im_paths[k] for k in indexes]
X, y = self.data_generation(list_IDs_im)
 if self.augment is None:
return X,np.array(y)/255
else:
im,mask = [],[]
for x,y in zip(X,y):
augmented = self.augment(image=x, mask=y)
 im.append(augmented['image'])
 mask.append(augmented['mask'])
 return np.array(im),np.array(mask)/255
def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.train_im_paths))
if self.shuffle == True:
np.random.shuffle(self.indexes)
 def data_generation(self, list_IDs_im):
'Generates data containing batch_size samples'
X = np.empty(( len(list_IDs_im),self.img_size,self.img_size, self.n_channels))
y = np.empty(( len(list_IDs_im),self.img_size,self.img_size, 1))
for i, im_path in enumerate(list_IDs_im):
im = np.array(Image.open(im_path))
mask_path = im_path.replace(self.train_im_path,self.train_mask_path)
 mask = np.array(Image.open(mask_path))
if len(im.shape)==2:
im = np.repeat(im[...,None],3,2)
 X[i,] = cv2.resize(im,(self.img_size,self.img_size))
y[i,] = cv2.resize(mask,(self.img_size,self.img_size)) [..., np.newaxis]
y[y>0] = 255
return np.uint8(X),np.uint8(y )
test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
 print(test.shape)
 test.head()<data_type_conversions>
Compose, HorizontalFlip, CLAHE, HueSaturationValue,
RandomBrightness, RandomContrast, RandomGamma,OneOf,
ToFloat, ShiftScaleRotate,GridDistortion, ElasticTransform, JpegCompression, HueSaturationValue,
RGBShift, RandomBrightness, RandomContrast, Blur, MotionBlur, MedianBlur, GaussNoise,CenterCrop,
IAAAdditiveGaussianNoise,GaussNoise,OpticalDistortion,RandomSizedCrop 
)
 AUGMENTATIONS_TRAIN = Compose([
HorizontalFlip(p=0.5),
OneOf([
RandomContrast() ,
RandomGamma() ,
RandomBrightness() ,
], p=0.3),
OneOf([
ElasticTransform(alpha=120, sigma=120 * 0.05, alpha_affine=120 * 0.03),
GridDistortion() ,
OpticalDistortion(distort_limit=2, shift_limit=0.5),
], p=0.3),
RandomSizedCrop(min_max_height=(176, 256), height=h, width=w,p=0.25),
ToFloat(max_value=1)
 ],p=1)
 AUGMENTATIONS_TEST = Compose([
ToFloat(max_value=1)
 ],p=1 )
X_train = train.iloc[:, 1:].values.astype('float32')
 y_train = train.iloc[:, 0].values.astype('int32')
 X_test = test.values.astype('float32' )<set_options>
def get_iou_vector(A, B):
batch_size = A.shape[0]
metric = 0.0
for batch in range(batch_size):
t, p = A[batch], B[batch]
true = np.sum(t)
 pred = np.sum(p)
 if true == 0:
metric +=(pred == 0)
 continue
intersection = np.sum(t * p)
 union = true + pred - intersection
iou = intersection / union
iou = np.floor(max(0,(iou - 0.45)*20)) / 10
metric += iou
metric /= batch_size
return metric
def my_iou_metric(label, pred):
return tf.py_func(get_iou_vector, [label, pred > 0.5], tf.float64 )
%matplotlib inline<data_type_conversions>
def dice_coef(y_true, y_pred):
y_true_f = K.flatten(y_true)
 y_pred = K.cast(y_pred, 'float32')
 y_pred_f = K.cast(K.greater(K.flatten(y_pred), 0.5), 'float32')
 intersection = y_true_f * y_pred_f
score = 2.* K.sum(intersection)/(K.sum(y_true_f)+ K.sum(y_pred_f))
return score
def dice_loss(y_true, y_pred):
smooth = 1.
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = y_true_f * y_pred_f
score =(2.* K.sum(intersection)+ smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 return 1.- score
def bce_dice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)+ dice_loss(y_true, y_pred)
 def bce_logdice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)- K.log(1.- dice_loss(y_true, y_pred))
mean_px = X_train.mean().astype(np.float32)
 std_px = X_train.std().astype(np.float32)
 print(mean_px, std_px)
 def standardize(x):
return(x - mean_px)/ std_px<categorify>
class SnapshotCallbackBuilder:
def __init__(self, nb_epochs, nb_snapshots, init_lr=0.1):
self.T = nb_epochs
self.M = nb_snapshots
self.alpha_zero = init_lr
def get_callbacks(self, model_prefix='Model'):
callback_list = [
callbacks.ModelCheckpoint("./keras.model",monitor='val_my_iou_metric',
mode = 'max', save_best_only=True, verbose=1),
swa,
callbacks.LearningRateScheduler(schedule=self._cosine_anneal_schedule)
 ]
return callback_list
def _cosine_anneal_schedule(self, t):
cos_inner = np.pi *(t %(self.T // self.M))
cos_inner /= self.T // self.M
cos_out = np.cos(cos_inner)+ 1
return float(self.alpha_zero / 2 * cos_out )
y_train = to_categorical(y_train)
 y_train.shape<define_variables>
def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True):
x = Conv2D(filters, size, strides=strides, padding=padding )(x)
 x = BatchNormalization()(x)
 if activation == True:
x = LeakyReLU(alpha=0.1 )(x)
 return x
def residual_block(blockInput, num_filters=16):
x = LeakyReLU(alpha=0.1 )(blockInput)
 x = BatchNormalization()(x)
 blockInput = BatchNormalization()(blockInput)
 x = convolution_block(x, num_filters,(3,3))
x = convolution_block(x, num_filters,(3,3), activation=False)
 x = Add()([x, blockInput])
 return x
num_classes = y_train.shape[1]
num_classes<define_variables>
def UEfficientNet(input_shape=(None, None, 3),dropout_rate=0.1):
backbone = EfficientNetB4(weights='imagenet',
include_top=False,
input_shape=input_shape)
 input = backbone.input
start_neurons = 8
conv4 = backbone.layers[342].output
conv4 = LeakyReLU(alpha=0.1 )(conv4)
 pool4 = MaxPooling2D(( 2, 2))(conv4)
 pool4 = Dropout(dropout_rate )(pool4)
 convm = Conv2D(start_neurons * 32,(3, 3), activation=None, padding="same",name='conv_middle' )(pool4)
 convm = residual_block(convm,start_neurons * 32)
 convm = residual_block(convm,start_neurons * 32)
 convm = LeakyReLU(alpha=0.1 )(convm)
 deconv4 = Conv2DTranspose(start_neurons * 16,(3, 3), strides=(2, 2), padding="same" )(convm)
 deconv4_up1 = Conv2DTranspose(start_neurons * 16,(3, 3), strides=(2, 2), padding="same" )(deconv4)
 deconv4_up2 = Conv2DTranspose(start_neurons * 16,(3, 3), strides=(2, 2), padding="same" )(deconv4_up1)
 deconv4_up3 = Conv2DTranspose(start_neurons * 16,(3, 3), strides=(2, 2), padding="same" )(deconv4_up2)
 uconv4 = concatenate([deconv4, conv4])
 uconv4 = Dropout(dropout_rate )(uconv4)
 uconv4 = Conv2D(start_neurons * 16,(3, 3), activation=None, padding="same" )(uconv4)
 uconv4 = residual_block(uconv4,start_neurons * 16)
 uconv4 = LeakyReLU(alpha=0.1 )(uconv4)
 deconv3 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(uconv4)
 deconv3_up1 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(deconv3)
 deconv3_up2 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(deconv3_up1)
 conv3 = backbone.layers[154].output
uconv3 = concatenate([deconv3,deconv4_up1, conv3])
 uconv3 = Dropout(dropout_rate )(uconv3)
 uconv3 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(uconv3)
 uconv3 = residual_block(uconv3,start_neurons * 8)
 uconv3 = LeakyReLU(alpha=0.1 )(uconv3)
 deconv2 = Conv2DTranspose(start_neurons * 4,(3, 3), strides=(2, 2), padding="same" )(uconv3)
 deconv2_up1 = Conv2DTranspose(start_neurons * 4,(3, 3), strides=(2, 2), padding="same" )(deconv2)
 conv2 = backbone.layers[92].output
uconv2 = concatenate([deconv2,deconv3_up1,deconv4_up2, conv2])
 uconv2 = Dropout(0.1 )(uconv2)
 uconv2 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(uconv2)
 uconv2 = residual_block(uconv2,start_neurons * 4)
 uconv2 = LeakyReLU(alpha=0.1 )(uconv2)
 deconv1 = Conv2DTranspose(start_neurons * 2,(3, 3), strides=(2, 2), padding="same" )(uconv2)
 conv1 = backbone.layers[30].output
uconv1 = concatenate([deconv1,deconv2_up1,deconv3_up2,deconv4_up3, conv1])
 uconv1 = Dropout(0.1 )(uconv1)
 uconv1 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(uconv1)
 uconv1 = residual_block(uconv1,start_neurons * 2)
 uconv1 = LeakyReLU(alpha=0.1 )(uconv1)
 uconv0 = Conv2DTranspose(start_neurons * 1,(3, 3), strides=(2, 2), padding="same" )(uconv1)
 uconv0 = Dropout(0.1 )(uconv0)
 uconv0 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(uconv0)
 uconv0 = residual_block(uconv0,start_neurons * 1)
 uconv0 = LeakyReLU(alpha=0.1 )(uconv0)
 uconv0 = Dropout(dropout_rate/2 )(uconv0)
 output_layer = Conv2D(1,(1,1), padding="same", activation="sigmoid" )(uconv0)
 model = Model(input, output_layer)
 model.name = 'u-xception'
return model
seed = 43
np.random.seed(seed )<import_modules>
K.clear_session()
img_size = 256
model = UEfficientNet(input_shape=(img_size,img_size,3),dropout_rate=0.5 )
from keras.models import Sequential
from keras.layers.core import Lambda, Dense, Flatten, Dropout
from keras.callbacks import EarlyStopping
from keras.layers import BatchNormalization, Convolution2D, MaxPooling2D<choose_model_class>
class SWA(keras.callbacks.Callback):
def __init__(self, filepath, swa_epoch):
super(SWA, self ).__init__()
self.filepath = filepath
self.swa_epoch = swa_epoch
def on_train_begin(self, logs=None):
self.nb_epoch = self.params['epochs']
print('Stochastic weight averaging selected for last {} epochs.' 
.format(self.nb_epoch - self.swa_epoch))
def on_epoch_end(self, epoch, logs=None):
if epoch == self.swa_epoch:
self.swa_weights = self.model.get_weights()
elif epoch > self.swa_epoch:
for i in range(len(self.swa_weights)) :
self.swa_weights[i] =(self.swa_weights[i] * 
(epoch - self.swa_epoch)+ self.model.get_weights() [i])/(( epoch - self.swa_epoch)+ 1)
 else:
pass
def on_train_end(self, logs=None):
self.model.set_weights(self.swa_weights)
 print('Final model parameters set to stochastic weight average.')
 self.model.save_weights(self.filepath)
 print('Final stochastic averaged weights saved to file.' )
model = Sequential()
model.add(Lambda(standardize, input_shape=(28, 28, 1)))
 model.add(Flatten())
 model.add(Dense(10, activation='softmax'))<choose_model_class>
model.compile(loss=bce_dice_loss, optimizer='adam', metrics=[my_iou_metric] )
model.compile(optimizer=RMSprop(lr=0.001),
loss='categorical_crossentropy',
metrics=['accuracy'] )<split>
epochs = 70
snapshot = SnapshotCallbackBuilder(nb_epochs=epochs,nb_snapshots=1,init_lr=1e-3)
 batch_size = 16
swa = SWA('./keras_swa.model',67)
 valid_im_path,valid_mask_path = './keras_im_val','./keras_mask_val'
training_generator = DataGenerator(augmentations=AUGMENTATIONS_TRAIN,img_size=img_size)
 validation_generator = DataGenerator(train_im_path = valid_im_path ,
train_mask_path=valid_mask_path,augmentations=AUGMENTATIONS_TEST,
img_size=img_size)
 history = model.fit_generator(generator=training_generator,
validation_data=validation_generator,
use_multiprocessing=False,
epochs=epochs,verbose=2,
callbacks=snapshot.get_callbacks() )
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1, random_state=42)
 batches = gen.flow(X_train, y_train, batch_size=64)
 val_batches = gen.flow(X_val, y_val, batch_size=64 )<define_variables>
try:
print('using swa weight model')
 model.load_weights('./keras_swa.model')
 except Exception as e:
print(e)
 model.load_weights('./keras.model' )
print(len(batches))
print(batches.n // 64 )<train_model>
def predict_result(model,validation_generator,img_size):
preds_test1 = model.predict_generator(validation_generator ).reshape(-1, img_size, img_size)
 return preds_test1
history = model.fit_generator(generator=batches,
steps_per_epoch=len(batches),
epochs=30,
validation_data=val_batches,
validation_steps=len(val_batches))<find_best_params>
validation_generator = DataGenerator(train_im_path = valid_im_path ,
train_mask_path=valid_mask_path,augmentations=AUGMENTATIONS_TEST,
img_size=img_size,shuffle=False)
 AUGMENTATIONS_TEST_FLIPPED = Compose([
HorizontalFlip(p=1),
ToFloat(max_value=1)
 ],p=1)
 validation_generator_flipped = DataGenerator(train_im_path = valid_im_path ,
train_mask_path=valid_mask_path,augmentations=AUGMENTATIONS_TEST_FLIPPED,
img_size=img_size,shuffle=False)
 preds_valid_orig = predict_result(model,validation_generator,img_size)
 preds_valid_flipped = predict_result(model,validation_generator_flipped,img_size)
 preds_valid_flipped = np.array([np.fliplr(x)for x in preds_valid_flipped])
 preds_valid = 0.5*preds_valid_orig + 0.5*preds_valid_flipped
loss = history_dict['loss']
val_loss = history_dict['val_loss']
epochs = range(1, len(loss)+ 1 )<choose_model_class>
valid_fn = glob.glob('./keras_mask_val/*')
 y_valid_ori = np.array([cv2.resize(np.array(Image.open(fn)) ,(img_size,img_size)) for fn in valid_fn])
 assert y_valid_ori.shape == preds_valid.shape
def get_fc_model() :
model = Sequential([
Lambda(standardize, input_shape=(28, 28, 1)) ,
Flatten() ,
Dense(512, activation='relu'),
Dense(10, activation='softmax')
 ])
 model.compile(optimizer='Adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
 return model<find_best_params>
threshold_best = 0.5
max_images = 64
grid_width = 16
grid_height = int(max_images / grid_width)
 fig, axs = plt.subplots(grid_height, grid_width, figsize=(grid_width, grid_height))
validation_generator = DataGenerator(train_im_path = valid_im_path ,
train_mask_path=valid_mask_path,augmentations=AUGMENTATIONS_TEST,
img_size=img_size,batch_size=64,shuffle=False)
 images,masks = validation_generator.__getitem__(0)
 for i,(im, mask)in enumerate(zip(images,masks)) :
pred = preds_valid[i]
ax = axs[int(i / grid_width), i % grid_width]
ax.imshow(im[...,0], cmap="bone")
 ax.imshow(mask.squeeze() , alpha=0.5, cmap="Reds")
 ax.imshow(np.array(np.round(pred > threshold_best), dtype=np.float32), alpha=0.5, cmap="Greens")
 ax.axis('off')
 plt.suptitle("Green:Prediction , Red: Pneumothorax." )
fc = get_fc_model()
fc.optimizer.lr<train_model>
thresholds = np.linspace(0.2, 0.9, 31)
 ious = np.array([iou_metric_batch(y_valid_ori, np.int32(preds_valid > threshold)) for threshold in tqdm_notebook(thresholds)] )
history = fc.fit_generator(generator=batches,
steps_per_epoch=len(batches),
epochs=30,
validation_data=val_batches,
validation_steps=len(val_batches))<choose_model_class>
max_images = 64
grid_width = 16
grid_height = int(max_images / grid_width)
 fig, axs = plt.subplots(grid_height, grid_width, figsize=(grid_width, grid_height))
validation_generator = DataGenerator(train_im_path = valid_im_path ,
train_mask_path=valid_mask_path,augmentations=AUGMENTATIONS_TEST,
img_size=img_size,batch_size=64,shuffle=False)
 images,masks = validation_generator.__getitem__(0)
 for i,(im, mask)in enumerate(zip(images,masks)) :
pred = preds_valid[i]
ax = axs[int(i / grid_width), i % grid_width]
ax.imshow(im[...,0], cmap="bone")
 ax.imshow(mask.squeeze() , alpha=0.5, cmap="Reds")
 ax.imshow(np.array(np.round(pred > threshold_best), dtype=np.float32), alpha=0.5, cmap="Greens")
 ax.axis('off')
 plt.suptitle("Green:Prediction , Red: Pneumothorax." )
def get_cnn_model() :
model = Sequential([
Lambda(standardize, input_shape=(28, 28, 1)) ,
Convolution2D(32,(3, 3), activation='relu'),
Convolution2D(32,(3, 3), activation='relu'),
MaxPooling2D() ,
Convolution2D(64,(3, 3), activation='relu'),
Convolution2D(64,(3, 3), activation='relu'),
MaxPooling2D() ,
Flatten() ,
Dense(512, activation='relu'),
Dense(10, activation='softmax')
 ])
 model.compile(Adam() ,
loss='categorical_crossentropy',
metrics=['accuracy'])
 return model<find_best_params>
test_fn = glob.glob('./test/*')
 x_test = [cv2.resize(np.array(Image.open(fn)) ,(img_size,img_size)) for fn in test_fn]
x_test = np.array(x_test)
 x_test = np.array([np.repeat(im[...,None],3,2)for im in x_test])
 print(x_test.shape)
 preds_test_orig = model.predict(x_test,batch_size=batch_size)
 x_test = np.array([np.fliplr(x)for x in x_test])
 preds_test_flipped = model.predict(x_test,batch_size=batch_size)
 preds_test_flipped = np.array([np.fliplr(x)for x in preds_test_flipped])
 preds_test = 0.5*preds_test_orig + 0.5*preds_test_flipped
model = get_cnn_model()<train_model>
sys.path.insert(0, '.. /input/siim-acr-pneumothorax-segmentation')
 rles = []
i,max_img = 1,10
plt.figure(figsize=(16,4))
for p in tqdm_notebook(preds_test):
p = p.squeeze()
im = cv2.resize(p,(1024,1024))
im = im > threshold_best
if im.sum() <1024*2:
im[:] = 0
im =(im.T*255 ).astype(np.uint8)
 rles.append(mask2rle(im, 1024, 1024))
i += 1
if i<max_img:
plt.subplot(1,max_img,i)
 plt.imshow(im)
 plt.axis('off' )
history = model.fit_generator(generator=batches,
steps_per_epoch=len(batches),
epochs=30,
validation_data=val_batches,
validation_steps=len(val_batches))<define_variables>
ids = [o.split('/')[-1][:-4] for o in test_fn]
sub_df = pd.DataFrame({'ImageId': ids, 'EncodedPixels': rles})
 sub_df.loc[sub_df.EncodedPixels=='', 'EncodedPixels'] = '-1'
sub_df.head()
<train_model><EOS> 
sub_df.to_csv('submission.csv', index=False )