Kaggle competition: SIIM-ACR Pneumothorax Segmentation
<SOS> metric: SIIMDice Kaggle data source: siim-acr-pneumothorax-segmentation<choose_model_class>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
sys.path.insert(0, '.. /input/siim-acr-pneumothorax-segmentation')
 fastai.__version__
datagen = ImageDataGenerator(zoom_range = 0.1, width_shift_range = 0.1, height_shift_range = 0.1, rotation_range = 10 )<choose_model_class>
sz = 256
bs = 16
n_acc = 64//bs
nfolds = 4
SEED = 2019
noise_th = 75.0*(sz/128.0)**2
best_thr0 = 0.2
if sz == 256:
stats =([0.540,0.540,0.540],[0.264,0.264,0.264])
 TRAIN = '.. /input/siimacr-pneumothorax-segmentation-data-256/train'
TEST = '.. /input/siimacr-pneumothorax-segmentation-data-256/test'
MASKS = '.. /input/siimacr-pneumothorax-segmentation-data-256/masks'
elif sz == 128:
stats =([0.615,0.615,0.615],[0.291,0.291,0.291])
 TRAIN = '.. /input/siimacr-pneumothorax-segmentation-data-128/train'
TEST = '.. /input/siimacr-pneumothorax-segmentation-data-128/test'
MASKS = '.. /input/siimacr-pneumothorax-segmentation-data-128/masks'
Path('/tmp/.cache/torch/checkpoints/' ).mkdir(exist_ok=True, parents=True)
 !cp '.. /input/resnet34/resnet34.pth' '/tmp/.cache/torch/checkpoints/resnet34-333f7ec4.pth'
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(SEED )
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(3, 3), activation = 'relu', input_shape =(28, 28, 1)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters = 32, kernel_size =(3, 3), activation = 'relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 32, kernel_size =(5, 5), activation = 'relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(strides =(2,2)))
 model.add(Dropout(0.4))
model.add(Conv2D(filters = 64, kernel_size =(3, 3), activation = 'relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 64, kernel_size =(3, 3), activation = 'relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 64, kernel_size =(5, 5), activation = 'relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(strides =(2,2)))
 model.add(Dropout(0.4))
model.add(Flatten())
 model.add(Dense(512, activation = 'relu'))
model.add(Dropout(0.4))
model.add(Dense(1024, activation = 'relu'))
model.add(Dropout(0.4))
model.add(Dense(10, activation = 'softmax'))<choose_model_class>
class Hcolumns(nn.Module):
def __init__(self, hooks:Collection[Hook], nc:Collection[int]=None):
super(Hcolumns,self ).__init__()
self.hooks = hooks
self.n = len(self.hooks)
 self.factorization = None
if nc is not None:
self.factorization = nn.ModuleList()
for i in range(self.n):
self.factorization.append(nn.Sequential(
 conv2d(nc[i],nc[-1],3,padding=1,bias=True),
conv2d(nc[-1],nc[-1],3,padding=1,bias=True)))
 def forward(self, x:Tensor):
n = len(self.hooks)
 out = [F.interpolate(self.hooks[i].stored if self.factorization is None
else self.factorization[i](self.hooks[i].stored), scale_factor=2**(self.n-i),
mode='bilinear',align_corners=False)for i in range(self.n)] + [x]
return torch.cat(out, dim=1)
 class DynamicUnet_Hcolumns(SequentialEx):
"Create a U-Net from a given architecture."
def __init__(self, encoder:nn.Module, n_classes:int, blur:bool=False, blur_final=True,
self_attention:bool=False,
y_range:Optional[Tuple[float,float]]=None,
last_cross:bool=True, bottle:bool=False, **kwargs):
imsize =(256,256)
 sfs_szs = model_sizes(encoder, size=imsize)
 sfs_idxs = list(reversed(_get_sfs_idxs(sfs_szs)))
 self.sfs = hook_outputs([encoder[i] for i in sfs_idxs])
 x = dummy_eval(encoder, imsize ).detach()
ni = sfs_szs[-1][1]
middle_conv = nn.Sequential(conv_layer(ni, ni*2, **kwargs),
conv_layer(ni*2, ni, **kwargs)).eval()
x = middle_conv(x)
 layers = [encoder, batchnorm_2d(ni), nn.ReLU() , middle_conv]
self.hc_hooks = [Hook(layers[-1], _hook_inner, detach=False)]
hc_c = [x.shape[1]]
for i,idx in enumerate(sfs_idxs):
not_final = i!=len(sfs_idxs)-1
up_in_c, x_in_c = int(x.shape[1]), int(sfs_szs[idx][1])
 do_blur = blur and(not_final or blur_final)
 sa = self_attention and(i==len(sfs_idxs)-3)
 unet_block = UnetBlock(up_in_c, x_in_c, self.sfs[i], final_div=not_final,
blur=blur, self_attention=sa, **kwargs ).eval()
layers.append(unet_block)
 x = unet_block(x)
 self.hc_hooks.append(Hook(layers[-1], _hook_inner, detach=False))
hc_c.append(x.shape[1])
 ni = x.shape[1]
if imsize != sfs_szs[0][-2:]: layers.append(PixelShuffle_ICNR(ni, **kwargs))
if last_cross:
layers.append(MergeLayer(dense=True))
ni += in_channels(encoder)
 layers.append(res_block(ni, bottle=bottle, **kwargs))
hc_c.append(ni)
 layers.append(Hcolumns(self.hc_hooks, hc_c))
layers += [conv_layer(ni*len(hc_c), n_classes, ks=1, use_activ=False, **kwargs)]
if y_range is not None: layers.append(SigmoidRange(*y_range))
super().__init__(*layers)
 def __del__(self):
if hasattr(self, "sfs"): self.sfs.remove()
def unet_learner(data:DataBunch, arch:Callable, pretrained:bool=True, blur_final:bool=True,
norm_type:Optional[NormType]=NormType, split_on:Optional[SplitFuncOrIdxList]=None,
blur:bool=False, self_attention:bool=False, y_range:Optional[Tuple[float,float]]=None,
last_cross:bool=True, bottle:bool=False, cut:Union[int,Callable]=None,
hypercolumns=True, **learn_kwargs:Any)->Learner:
"Build Unet learner from `data` and `arch`."
meta = cnn_config(arch)
 body = create_body(arch, pretrained, cut)
 M = DynamicUnet_Hcolumns if hypercolumns else DynamicUnet
model = to_device(M(body, n_classes=data.c, blur=blur, blur_final=blur_final,
self_attention=self_attention, y_range=y_range, norm_type=norm_type,
last_cross=last_cross, bottle=bottle), data.device)
 learn = Learner(data, model, **learn_kwargs)
 learn.split(ifnone(split_on, meta['split']))
if pretrained: learn.freeze()
apply_init(model[2], nn.init.kaiming_normal_)
 return learn
model.compile(optimizer = 'adam', metrics = ['accuracy'], loss = 'categorical_crossentropy' )<choose_model_class>
class AccumulateOptimWrapper(OptimWrapper):
def step(self): pass
def zero_grad(self): pass
def real_step(self): super().step()
def real_zero_grad(self): super().zero_grad()
def acc_create_opt(self, lr:Floats, wd:Floats=0.) :
"Create optimizer with `lr` learning rate and `wd` weight decay."
self.opt = AccumulateOptimWrapper.create(self.opt_func, lr, self.layer_groups,
wd=wd, true_wd=self.true_wd, bn_wd=self.bn_wd)
 Learner.create_opt = acc_create_opt
@dataclass
class AccumulateStep(LearnerCallback):

 def __init__(self, learn:Learner, n_step:int = 1):
super().__init__(learn)
 self.n_step = n_step
def on_epoch_begin(self, **kwargs):
"init samples and batches, change optimizer"
self.acc_batches = 0
def on_batch_begin(self, last_input, last_target, **kwargs):
"accumulate samples and batches"
self.acc_batches += 1
def on_backward_end(self, **kwargs):
"step if number of desired batches accumulated, reset samples"
if(self.acc_batches % self.n_step)== self.n_step - 1:
for p in(self.learn.model.parameters()):
if p.requires_grad: p.grad.div_(self.acc_batches)
 self.learn.opt.real_step()
self.learn.opt.real_zero_grad()
self.acc_batches = 0
def on_epoch_end(self, **kwargs):
"step the rest of the accumulated grads"
if self.acc_batches > 0:
for p in(self.learn.model.parameters()):
if p.requires_grad: p.grad.div_(self.acc_batches)
 self.learn.opt.real_step()
self.learn.opt.real_zero_grad()
self.acc_batches = 0
learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy', patience=3, verbose=1, factor=0.5, min_lr=0.00001 )<train_model>
def set_BN_momentum(model,momentum=0.1*bs/64):
for i,(name, layer)in enumerate(model.named_modules()):
if isinstance(layer, nn.BatchNorm2d)or isinstance(layer, nn.BatchNorm1d):
layer.momentum = momentum
hist = model.fit_generator(datagen.flow(X_train, y_train, batch_size = 32), epochs = 35, validation_data =(X_test, y_test), callbacks = [learning_rate_reduction] )<compute_test_metric>
def dice(input:Tensor, targs:Tensor, iou:bool=False, eps:float=1e-8)->Rank0Tensor:
n = targs.shape[0]
input = torch.softmax(input, dim=1)[:,1,...].view(n,-1)
 input =(input > best_thr0 ).long()
input[input.sum(-1)< noise_th,...] = 0.0
targs = targs.view(n,-1)
 intersect =(input * targs ).sum(-1 ).float()
union =(input+targs ).sum(-1 ).float()
if not iou: return(( 2.0*intersect + eps)/(union+eps)).mean()
else: return(( intersect + eps)/(union - intersect + eps)).mean()
final_loss, final_acc = model.evaluate(X_test, y_test, verbose=0)
 print("Final loss: {0:.4f}, final accuracy: {1:.4f}".format(final_loss, final_acc))<predict_on_test>
def dice_overall(preds, targs):
n = preds.shape[0]
preds = preds.view(n, -1)
 targs = targs.view(n, -1)
 intersect =(preds * targs ).sum(-1 ).float()
union =(preds+targs ).sum(-1 ).float()
u0 = union==0
intersect[u0] = 1
union[u0] = 2
return(2.* intersect / union )
test = test.values.reshape(-1, 28, 28, 1)/ 255
y_pred = model.predict(test, batch_size = 64)
 y_pred = np.argmax(y_pred,axis = 1)
 y_pred = pd.Series(y_pred,name="Label")
 y_pred<save_to_csv>
def pred_with_flip(learn:fastai.basic_train.Learner,
ds_type:fastai.basic_data.DatasetType=DatasetType.Valid):
preds, ys = learn.get_preds(ds_type)
 preds = preds[:,1,...]
learn.data.dl(ds_type ).dl.dataset.tfms.append(flip_lr())
 preds_lr, ys = learn.get_preds(ds_type)
 del learn.data.dl(ds_type ).dl.dataset.tfms[-1]
preds_lr = preds_lr[:,1,...]
ys = ys.squeeze()
preds = 0.5*(preds + torch.flip(preds_lr,[-1]))
del preds_lr
gc.collect()
torch.cuda.empty_cache()
return preds, ys
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),y_pred],axis = 1)
 submission.to_csv("submission.csv",index=False )<load_from_csv>
class SegmentationLabelList(SegmentationLabelList):
def open(self, fn): return open_mask(fn, div=True)
 class SegmentationItemList(SegmentationItemList):
_label_cls = SegmentationLabelList
def transform(self, tfms:Optional[Tuple[TfmList,TfmList]]=(None,None), **kwargs):
if not tfms: tfms=(None,None)
 assert is_listy(tfms)and len(tfms)== 2
self.train.transform(tfms[0], **kwargs)
 self.valid.transform(tfms[1], **kwargs)
 kwargs['tfm_y'] = False
if self.test: self.test.transform(tfms[1], **kwargs)
 return self
fastai.data_block.ItemLists.transform = transform
submission = pd.read_csv("/kaggle/input/digit-recognizer/sample_submission.csv")
 submission.head()<load_from_csv>
def get_data(fold):
kf = KFold(n_splits=nfolds, shuffle=True, random_state=SEED)
 valid_idx = list(kf.split(list(range(len(Path(TRAIN ).ls())))))[fold][1]
data =(SegmentationItemList.from_folder(TRAIN)
.split_by_idx(valid_idx)
.label_from_func(lambda x : str(x ).replace('train', 'masks'), classes=[0,1])
.add_test(Path(TEST ).ls() , label=None)
.transform(get_transforms() , size=sz, tfm_y=True)
.databunch(path=Path('.'), bs=bs)
.normalize(stats))
return data
get_data(0 ).show_batch()
traindat = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 traindat.head()<load_from_csv>
unet_learner(get_data(0), models.resnet34, metrics=[dice] ).model
testdat = pd.read_csv("/kaggle/input/digit-recognizer/test.csv")
 testdat.head()<prepare_x_and_y>
scores, best_thrs = [],[]
for fold in range(nfolds):
print('fold: ', fold)
 data = get_data(fold)
 learn = unet_learner(data, models.resnet34, metrics=[dice])
 learn.clip_grad(1.0);
set_BN_momentum(learn.model)
 lr = 1e-3
learn.fit_one_cycle(6, lr, callbacks = [AccumulateStep(learn,n_acc)])
 learn.unfreeze()
learn.fit_one_cycle(12, slice(lr/80, lr/2), callbacks=[AccumulateStep(learn,n_acc)])
 learn.save('fold'+str(fold)) ;
preds, ys = pred_with_flip(learn)
 pt, _ = pred_with_flip(learn,DatasetType.Test)
 if fold == 0: preds_test = pt
else: preds_test += pt
preds_save =(preds*255.0 ).byte()
torch.save(preds_save, 'preds_fold'+str(fold)+'.pt')
 np.save('items_fold'+str(fold), data.valid_ds.items)
 preds[preds.view(preds.shape[0],-1 ).sum(-1)< noise_th,...] = 0.0
dices = []
thrs = np.arange(0.01, 1, 0.01)
 for th in progress_bar(thrs):
preds_m =(preds>th ).long()
dices.append(dice_overall(preds_m, ys ).mean())
 dices = np.array(dices)
 scores.append(dices.max())
 best_thrs.append(thrs[dices.argmax() ])
 if fold != nfolds-1: del preds, ys, preds_save
gc.collect()
torch.cuda.empty_cache()
preds_test /= nfolds
Y = traindat['label']
X = traindat.drop(['label'], axis = 1)
 X.head()<train_model>
print('scores: ', scores)
 print('mean score: ', np.array(scores ).mean())
 print('thresholds: ', best_thrs)
 best_thr = np.array(best_thrs ).mean()
print('best threshold: ', best_thr )
Y = np.array(Y)
 X = np.array(X)/255.
 X = np.array([x.reshape(( 28, 28, 1)) for x in X])
 X = np.append(X, X, axis = 0)
 Y = np.append(Y, Y, axis = 0)
 X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size = 0.2, random_state = 40)
 print(X_train.shape)
 print(X_test.shape)
 print(y_train.shape)
 print(y_test.shape )<choose_model_class>
preds_save =(preds_test*255.0 ).byte()
torch.save(preds_save, 'preds_test.pt')
 preds_test[preds_test.view(preds_test.shape[0],-1 ).sum(-1)< noise_th,...] = 0.0
model = Sequential([
Conv2D(input_shape =(28,28, 1), filters = 128, kernel_size =(3, 3), activation = 'relu'),
BatchNormalization() ,
MaxPooling2D(pool_size =(2,2)) ,
Conv2D(filters = 128, kernel_size =(3, 3), activation = 'relu'),
BatchNormalization() ,
Conv2D(filters = 128, kernel_size =(3, 3), activation = 'relu'),
BatchNormalization() ,
MaxPooling2D(pool_size =(2,2)) ,
Conv2D(filters = 64, kernel_size =(3, 3), activation = 'relu'),
BatchNormalization() ,
Flatten() ,
Dense(128, activation = 'relu'),
BatchNormalization() ,
Dropout(0.5),
Dense(64, activation = 'relu'),
Dense(10, activation = 'softmax')
 ])
 model.compile(optimizer = 'adam', metrics = ['accuracy'], loss = 'sparse_categorical_crossentropy')
 model.summary()<train_model>
preds_test =(preds_test>best_thr ).long().numpy()
rles = []
for p in progress_bar(preds_test):
im = PIL.Image.fromarray(( p.T*255 ).astype(np.uint8)).resize(( 1024,1024))
im = np.asarray(im)
 rles.append(mask2rle(im, 1024, 1024))
<train_model><EOS> 
ids = [o.stem for o in data.test_ds.items]
sub_df = pd.DataFrame({'ImageId': ids, 'EncodedPixels': rles})
 sub_df.loc[sub_df.EncodedPixels=='', 'EncodedPixels'] = '-1'
sub_df.to_csv('submission.csv', index=False)
 sub_df.head()