Kaggle competition: HuBMAP - Hacking the Kidney
<SOS> metric: Dice Kaggle data source: hubmap-hacking-the-kidney<choose_model_class>
!pip install --no-deps.. /input/hubmap-packages/pretrained-models.pytorch-master/pretrained-models.pytorch-master > /dev/null
def make_conv_layer(filter_size, suffix, inputs):
x = Conv2D(
 filter_size, kernel_size=(3, 3), padding='same', activation='relu',
kernel_initializer=he_normal(seed=rand_seed), name='conv_' + suffix )(inputs)
 outputs = BatchNormalization(name='bn_' + suffix )(x)
 return outputs<choose_model_class>
!pip install --no-deps.. /input/hubmap-packages/efficientnet_pytorch-0.6.3/efficientnet_pytorch-0.6.3 > /dev/null
def make_model() :
inputs = Input(shape=(28, 28, 1), name="input")
 x = Lambda(lambda v: v / 255.0, name='scaling' )(inputs)
 x = make_conv_layer(64, '1_1', x)
 x = make_conv_layer(64, '1_2', x)
 x = make_conv_layer(64, '1_3', x)
 x = MaxPooling2D(pool_size=(2, 2), name="maxpool_1" )(x)
 x = SpatialDropout2D(0.25, name='sp_dpout_1' )(x)
 x = make_conv_layer(128, '2_1', x)
 x = make_conv_layer(128, '2_2', x)
 x = make_conv_layer(128, '2_3', x)
 x = MaxPooling2D(pool_size=(2, 2), name='maxpool_2' )(x)
 x = SpatialDropout2D(0.25, name='sp_dpout_2' )(x)
 x = make_conv_layer(256, '3_1', x)
 x = make_conv_layer(256, '3_2', x)
 x = make_conv_layer(256, '3_3', x)
 x = GlobalAveragePooling2D(name='gblavgpool' )(x)
 x = Dense(256, activation='relu', name='dense' )(x)
 x = Dropout(0.25, name='dropout' )(x)
 outputs = Dense(label_count, activation='softmax', name='outputs' )(x)
 model = Model(inputs=inputs, outputs=outputs)
 model.compile(
 optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'])
 return model<choose_model_class>
!pip install --no-deps.. /input/hubmap-packages/timm-0.4.5-py3-none-any.whl > /dev/null
train_image_generator = ImageDataGenerator(
 rotation_range=20,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=10,
zoom_range=0.2,
fill_mode='constant',
cval=0,
data_format='channels_last')
 test_image_generator = ImageDataGenerator()<define_variables>
!pip install --no-deps.. /input/hubmap-packages/segmentation_models.pytorch-master/segmentation_models.pytorch-master > /dev/null
batch_size = 256
steps_per_epoch =(X_train.shape[0] + batch_size - 1)// batch_size
train_image_flow = train_image_generator.flow(
 X_train, y_train, batch_size=batch_size, shuffle=True, seed=rand_seed)
 test_image_flow = test_image_generator.flow(
 X_test, y_test, batch_size=batch_size, shuffle=True, seed=rand_seed)
 print("batch_size: {0}".format(batch_size))
print("steps_per_epoch: {0}".format(steps_per_epoch))<set_options>
warnings.filterwarnings("ignore" )
def get_lr(model):
return K.get_value(model.optimizer.lr)
 def set_lr(model, lr):
K.set_value(model.optimizer.lr, lr )<find_best_params>
sz = 512
step = 176
reduce = 2
TH = 0.4
S_TH = 0.55
DATA = '.. /input/hubmap-kidney-segmentation/test/'
df_sample = pd.read_csv('.. /input/hubmap-kidney-segmentation/sample_submission.csv')
 bs = 8
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )
def find_lr_on_batch_end(model, logs, lr_list, loss_list, lr_mult):
lr = get_lr(model)
 lr_list.append(lr)
 loss = logs['loss']
loss_list.append(loss)
 set_lr(model, lr * lr_mult )<find_best_params>
def enc2mask(encs, shape):
img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 for m,enc in enumerate(encs):
if isinstance(enc,np.float)and np.isnan(enc): continue
s = enc.split()
for i in range(len(s)//2):
start = int(s[2*i])- 1
length = int(s[2*i+1])
 img[start:start+length] = 1 + m
return img.reshape(shape ).T
def mask2enc(mask, n=1):
pixels = mask.T.flatten()
encs = []
for i in range(1,n+1):
p =(pixels == i ).astype(np.int8)
 if p.sum() == 0: encs.append(np.nan)
 else:
p = np.concatenate([[0], p, [0]])
 runs = np.where(p[1:] != p[:-1])[0] + 1
runs[1::2] -= runs[::2]
encs.append(' '.join(str(x)for x in runs))
return encs
def rle_encode_less_memory(img):
pixels = img.T.flatten()
pixels[0] = 0
pixels[-1] = 0
runs = np.where(pixels[1:] != pixels[:-1])[0] + 2
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
find_lr_start_lr = 1e-10
find_lr_end_lr = 1.0
find_lr_epochs = 3
find_lr_total_batch_count = find_lr_epochs * steps_per_epoch
find_lr_lr_mult =(find_lr_end_lr / find_lr_start_lr)**(1.0 / find_lr_total_batch_count)
 print("find_lr_epochs: {0}".format(find_lr_epochs))
print("find_lr_total_batch_count: {0}".format(find_lr_total_batch_count))
print("find_lr_lr_mult: {0}".format(find_lr_lr_mult))<find_best_params>
mean = np.array([0.63701425, 0.47097038, 0.68173952])
 std = np.array([0.15979014, 0.22442915, 0.14194921])
 s_th = 40
p_th = 200*sz//256
identity = rasterio.Affine(1, 0, 0, 0, 1, 0)
 def img2tensor(img,dtype:np.dtype=np.float32):
if img.ndim==2 : img = np.expand_dims(img,2)
 img = np.transpose(img,(2,0,1))
return torch.from_numpy(img.astype(dtype, copy=False))
class HuBMAPDataset(Dataset):
def __init__(self, idx, sz=sz, step=step, reduce=reduce):
self.data = rasterio.open(os.path.join(DATA,idx+'.tiff'), transform = identity,
num_threads='all_cpus')
 if self.data.count != 3:
print('Image file with subdatasets as channels')
 self.layers = [rasterio.open(subd)for subd in self.data.subdatasets]
self.shape = self.data.shape
self.reduce = reduce
self.sz = reduce*sz
self.step = reduce*step
self.pad0 =(self.sz - self.shape[0]%self.step)%self.step
self.pad1 =(self.sz - self.shape[1]%self.step)%self.step
self.n0max =(self.shape[0] + self.pad0 - self.sz)//self.step + 1
self.n1max =(self.shape[1] + self.pad1 - self.sz)//self.step + 1
def __len__(self):
return self.n0max*self.n1max
def __getitem__(self, idx):
n0,n1 = idx//self.n1max, idx%self.n1max
x0,y0 = -self.pad0//2 + n0*self.step, -self.pad1//2 + n1*self.step
p00,p01 = max(0,x0), min(x0+self.sz,self.shape[0])
 p10,p11 = max(0,y0), min(y0+self.sz,self.shape[1])
 img = np.zeros(( self.sz,self.sz,3),np.uint8)
 if self.data.count == 3:
img[(p00-x0):(p01-x0),(p10-y0):(p11-y0)] = np.moveaxis(self.data.read([1,2,3],
window=Window.from_slices(( p00,p01),(p10,p11))), 0, -1)
 else:
tile = np.zeros(( p01-p00, p11-p10, 3), dtype=np.uint8)
 for fl in range(3):
tile[:,:,fl] = self.layers[fl].read(window=Window.from_slices(( p00,p01),(p10,p11)))
 img[(p00-x0):(p01-x0),(p10-y0):(p11-y0)] = tile
if self.reduce != 1:
img = cv2.resize(img,(self.sz//self.reduce,self.sz//self.reduce),
interpolation = cv2.INTER_AREA)
 hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
 h,s,v = cv2.split(hsv)
 if(s>s_th ).sum() <= p_th or img.sum() <= p_th:
return img2tensor(( img/255.0 - mean)/std), -1, x0, y0, p00, p01, p10, p11
else: return img2tensor(( img/255.0 - mean)/std), idx, x0, y0, p00, p01, p10, p11
find_lr_model = make_model()
find_lr_lr_list = []
find_lr_loss_list = []
find_lr_callback = LambdaCallback(
 on_batch_end=lambda batch, logs: find_lr_on_batch_end(
 find_lr_model, logs, find_lr_lr_list, find_lr_loss_list, find_lr_lr_mult))<train_model>
class Model_pred:
def __init__(self, models, dl, reduce, tta:bool=True):
self.models = models
self.dl = dl
self.reduce = reduce
self.tta = tta
def __iter__(self):
count=0
with torch.no_grad() :
for x,y,x0,y0,p00,p01,p10,p11 in iter(self.dl):
if(( y>=0 ).sum() > 0):
x = x[y>=0].to(device)
 x0 = x0[y>=0]
y0 = y0[y>=0]
p00 = p00[y>=0]
p01 = p01[y>=0]
p10 = p10[y>=0]
p11 = p11[y>=0]
y = y[y>=0]
with autocast(enabled=True):
py = None
for model in self.models:
p = model(x)
 p = torch.sigmoid(p ).detach()
if py is None: py = p
else: py += p
if self.tta:
flips = [[-1],[-2],[-2,-1]]
for f in flips:
xf = torch.flip(x,f)
 for model in self.models:
p = model(xf)
 p = torch.flip(p,f)
 py += torch.sigmoid(p ).detach()
py /=(1+len(flips))
py /= len(self.models)
 py = py.permute(0,2,3,1 ).half().cpu()
batch_size = len(py)
 for i in range(batch_size):
yield py[i],y[i],x0[i],y0[i],p00[i],p01[i],p10[i],p11[i]
count += 1
def __len__(self):
return len(self.dl.dataset )
set_lr(find_lr_model, find_lr_start_lr)
 find_lr_history = find_lr_model.fit_generator(
 train_image_flow,
steps_per_epoch=steps_per_epoch,
epochs=find_lr_epochs,
validation_data=test_image_flow,
callbacks=[find_lr_callback],
verbose=2 )<choose_model_class>
class FReLU(nn.Module):
def __init__(self, c1, k=3):
super().__init__()
self.conv = nn.Conv2d(c1, c1, k, 1, 1, groups=c1, bias=False)
 self.bn = nn.BatchNorm2d(c1)
 def forward(self, x):
return torch.max(x, self.bn(self.conv(x)))
 class Conv2dAct(nn.Sequential):
def __init__(
 self,
in_channels,
out_channels,
kernel_size,
padding=0,
stride=1,
use_batchnorm=True,
act_layer='relu' 
):
conv = nn.Conv2d(
 in_channels,
out_channels,
kernel_size,
stride=stride,
padding=padding,
bias=not(use_batchnorm), 
)
 if act_layer == 'frelu':
act = FReLU(out_channels)
 else:
act = create_act_layer(act_layer)
 if use_batchnorm:
bn = nn.BatchNorm2d(out_channels)
 else:
bn = nn.Identity()
super(Conv2dAct, self ).__init__(conv, bn, act)
 class DecoderBlockV2(nn.Module):
def __init__(
 self,
in_channels,
skip_channels,
out_channels,
use_batchnorm=True,
act_layer='relu',
attention_type=None, 
):
super().__init__()
self.conv1 = Conv2dAct(
 in_channels + skip_channels,
out_channels,
kernel_size=3,
padding=1,
use_batchnorm=use_batchnorm,
act_layer=act_layer 
)
 self.attention1 = md.Attention(attention_type, in_channels=in_channels + skip_channels)
 self.conv2 = Conv2dAct(
 out_channels,
out_channels,
kernel_size=3,
padding=1,
use_batchnorm=use_batchnorm,
act_layer=act_layer 
)
 self.attention2 = md.Attention(attention_type, in_channels=out_channels)
 def forward(self, x, skip=None):
x = F.interpolate(x, scale_factor=2, mode="nearest")
 if skip is not None:
x = torch.cat([x, skip], dim=1)
 x = self.attention1(x)
 x = self.conv1(x)
 x = self.conv2(x)
 x = self.attention2(x)
 return x
class UnetDecoderV2(nn.Module):
def __init__(
 self,
encoder_channels,
decoder_channels,
n_blocks=5,
use_batchnorm=True,
act_layer='relu',
attention_type=None, 
):
super().__init__()
if n_blocks != len(decoder_channels):
raise ValueError(
 "Model depth is {}, but you provide `decoder_channels` for {} blocks.".format(
 n_blocks, len(decoder_channels)
)
)
 encoder_channels = encoder_channels[1:]
encoder_channels = encoder_channels[::-1]
head_channels = encoder_channels[0]
in_channels = [head_channels] + list(decoder_channels[:-1])
 skip_channels = list(encoder_channels[1:])+ [0]
out_channels = decoder_channels
self.center = nn.Identity()
kwargs = dict(use_batchnorm=use_batchnorm, act_layer=act_layer, attention_type=attention_type)
 blocks = [
DecoderBlockV2(in_ch, skip_ch, out_ch, **kwargs)
 for in_ch, skip_ch, out_ch in zip(in_channels, skip_channels, out_channels)
 ]
self.blocks = nn.ModuleList(blocks)
 def forward(self, *features):
features = features[1:]
features = features[::-1]
head = features[0]
skips = features[1:]
x = self.center(head)
 for i, decoder_block in enumerate(self.blocks):
skip = skips[i] if i < len(skips)else None
x = decoder_block(x, skip)
 return x
train_epochs = 150
def lr_schedule(epoch_index, current_lr):
if epoch_index == 0:
new_lr = 1e-3
elif epoch_index == 49:
new_lr = 3e-4
elif epoch_index == 99:
new_lr = 1e-4
else:
new_lr = current_lr
if new_lr != current_lr:
print(
 "Epoch {0}: Learning late changed from {1:.5f} to {2:.5f}".format(
 epoch_index + 1, current_lr, new_lr))
return new_lr
lr_scheduler = LearningRateScheduler(lr_schedule, verbose=0 )<choose_model_class>
class CSPDarkNet53Encoder(nn.Module):
def __init__(self, pretrain=True, act_layer='leaky_relu'):
super(CSPDarkNet53Encoder, self ).__init__()
base = timm.create_model('cspdarknet53', pretrained=pretrain, act_layer=get_act_layer(act_layer))
self.stem = base.stem
self.layer0 = base.stages[0]
self.layer1 = base.stages[1]
self.layer2 = base.stages[2]
self.layer3 = base.stages[3]
self.layer4 = base.stages[4]
del base
self.depth = 5
self.out_channels =(32, 64, 128, 256, 512, 1024)
 self.in_channels = 3
def get_stages(self):
return [
self.stem,
self.layer0,
self.layer1,
self.layer2,
self.layer3,
self.layer4,
]
def forward(self, x):
stages = self.get_stages()
features = []
for i in range(self.depth + 1):
x = stages[i](x)
 features.append(x)
 return features
class CSPDarkNet53FPN(smp.base.SegmentationModel):
def __init__(
 self,
pretrain=True,
decoder_pyramid_channels: int = 256,
decoder_segmentation_channels: int = 128,
decoder_merge_policy: str = "add",
decoder_dropout: float = 0.2,
in_channels: int = 3,
classes: int = 1,
activation: Optional[str] = None,
upsampling: int = 4,
aux_params: Optional[dict] = None, 
):
super().__init__()
self.encoder = CSPDarkNet53Encoder(pretrain)
 self.decoder = smp.fpn.decoder.FPNDecoder(
 encoder_channels=self.encoder.out_channels,
encoder_depth=self.encoder.depth,
pyramid_channels=decoder_pyramid_channels,
segmentation_channels=decoder_segmentation_channels,
dropout=decoder_dropout,
merge_policy=decoder_merge_policy, 
)
 self.segmentation_head = smp.base.SegmentationHead(
 in_channels=self.decoder.out_channels,
out_channels=classes,
activation=activation,
kernel_size=1,
upsampling=upsampling, 
)
 if aux_params is not None:
self.classification_head = smp.base.ClassificationHead(
 in_channels=self.encoder.out_channels[-1], **aux_params 
)
 else:
self.classification_head = None
self.name = "fpn-cspdarknet53"
self.initialize()
class CSPDarkNet53Unet(smp.base.SegmentationModel):
def __init__(
 self,
pretrain=True,
act_layer='leaky_relu',
decoder_use_batchnorm: bool = True,
decoder_channels: List[int] =(256, 128, 64, 32, 16),
decoder_attention_type: Optional[str] = None,
in_channels: int = 3,
classes: int = 1,
activation: Optional[str] = None,
aux_params: Optional[dict] = None, 
):
super().__init__()
self.encoder = CSPDarkNet53Encoder(pretrain, act_layer)
 self.decoder = smp.unet.decoder.UnetDecoder(
 encoder_channels=self.encoder.out_channels,
decoder_channels=decoder_channels,
n_blocks=self.encoder.depth,
use_batchnorm=decoder_use_batchnorm,
center=False,
attention_type=decoder_attention_type 
)
 self.segmentation_head = smp.base.SegmentationHead(
 in_channels=decoder_channels[-1],
out_channels=classes,
activation=activation,
kernel_size=3, 
)
 if aux_params is not None:
self.classification_head = smp.base.ClassificationHead(
 in_channels=self.encoder.out_channels[-1], **aux_params 
)
 else:
self.classification_head = None
self.name = "unet-cspdarknet53"
self.initialize()
class CSPDarkNet53UnetV2(smp.base.SegmentationModel):
def __init__(
 self,
pretrain=True,
act_layer='leaky_relu',
decoder_use_batchnorm: bool = True,
decoder_channels: List[int] =(256, 128, 64, 32, 16),
decoder_act_layer: str = 'relu',
decoder_attention_type: Optional[str] = None,
in_channels: int = 3,
classes: int = 1,
activation: Optional[str] = None,
aux_params: Optional[dict] = None, 
):
super().__init__()
self.encoder = CSPDarkNet53Encoder(pretrain, act_layer)
 self.decoder = UnetDecoderV2(
 encoder_channels=self.encoder.out_channels,
decoder_channels=decoder_channels,
n_blocks=self.encoder.depth,
use_batchnorm=decoder_use_batchnorm,
act_layer=decoder_act_layer,
attention_type=decoder_attention_type 
)
 self.segmentation_head = smp.base.SegmentationHead(
 in_channels=decoder_channels[-1],
out_channels=classes,
activation=activation,
kernel_size=3, 
)
 if aux_params is not None:
self.classification_head = smp.base.ClassificationHead(
 in_channels=self.encoder.out_channels[-1], **aux_params 
)
 else:
self.classification_head = None
self.name = "unetv2-cspdarknet53"
self.initialize()
best_model_file_name = "best_model.hdf5"
model_check_point = ModelCheckpoint(
 best_model_file_name, monitor='val_sparse_categorical_accuracy', mode='max',
verbose=0, save_best_only=True, save_weights_only=True, period=1 )<train_model>
class PlModel(pl.LightningModule):
def __init__(self, learning_rate = 1e-3):
super(PlModel, self ).__init__()
self.learning_rate = learning_rate
self.model = smp.Unet('timm-efficientnet-b3', decoder_attention_type = None, encoder_weights = None)
 self.criterion = None
def forward(self, x):
return self.model(x )
train_history = train_model.fit_generator(
 train_image_flow,
steps_per_epoch=steps_per_epoch,
epochs=train_epochs,
validation_data=test_image_flow,
callbacks=[lr_scheduler, model_check_point],
verbose=2 )<load_pretrained>
models = []
B3_MODELS = [
'.. /input/nfl-submissions/Mar291351_0_epoch20_val_metric0.940.ckpt',
'.. /input/nfl-submissions/Mar291351_1_epoch20_val_metric0.940.ckpt',
'.. /input/nfl-submissions/Mar291351_2_epoch18_val_metric0.947.ckpt',
'.. /input/nfl-submissions/Mar291351_3_epoch20_val_metric0.939.ckpt',
]
for path in B3_MODELS:
model = PlModel()
model = model.load_from_checkpoint(path)
 model.float()
model.eval()
model.to(device)
 models.append(model )
best_train_model = make_model()
best_train_model.load_weights(best_model_file_name )<compute_test_metric>
D53_MODELS = [ 
('.. /input/hubmap-trained-models/unetv2-d53_sf-672-pseudo5-external_s2.pth', 'frelu'), 
('.. /input/hubmap-trained-models/unetv2-d53_sf_valid_8242609fa_best.pth', 'frelu'), 
('.. /input/hubmap-trained-models/unetv2-d53_sf_valid_b2dc8411c_best.pth', 'frelu'), 
('.. /input/hubmap-trained-models/unetv2-d53_sr_valid_4ef6695ce_best.pth', 'relu'), 
('.. /input/hubmap-trained-models/unetv2-d53_sf_valid_2f6ecfcdf_best.pth', 'frelu')
 ]
for path, act in D53_MODELS:
state_dict = torch.load(path,map_location=torch.device('cpu'))
model = CSPDarkNet53UnetV2(
 pretrain=False,
act_layer='swish',
decoder_act_layer=act,
in_channels=3,
classes=1 ).cuda()
model.load_state_dict(state_dict['model'])
 model.float()
model.eval()
model.to(device)
 models.append(model)
 del state_dict
train_result = best_train_model.evaluate(X_test, y_test)
 print(train_result )<load_from_csv>
print(f'Total number of trained models {len(models)}' )
test_df = pd.read_csv('.. /input/Kannada-MNIST/test.csv')
 test_df.head()<data_type_conversions>
def get_gauss_filter(half):
y,x = np.mgrid[-half:half,-half:half]
y = half-abs(y)
 x = half-abs(x)
 w = np.minimum(x,y)
 w = w/w.max()
w = np.minimum(w,1)
 return w
test_pixels_df = test_df.drop(columns='id')
 test_pixels_array = test_pixels_df.to_numpy(dtype=np.uint8)
 test_images = test_pixels_array.reshape(-1, 28, 28, 1)
 print(test_images.shape )<predict_on_test>
def reduce_fp(predict, shape, s, t=0.4):
mask =(predict > t ).astype(np.uint8)
 mask = ms.label(mask)
 l_probs = []
for i in range(1, mask.max() +1):
l_probs.append(predict[mask == i].max())
 l_probs = np.array(l_probs)
 mask[np.isin(mask,(np.where(l_probs < s)[0] + 1)) ] = 0
return cv2.resize(( mask > 0 ).astype(np.uint8), shape, interpolation=cv2.INTER_NEAREST )
model_preds = best_train_model.predict(test_images)
 print(model_preds.shape )<load_from_csv>
half = sz//2
w = get_gauss_filter(half ).astype(np.half)
 names,preds = [],[]
for idx,row in tqdm(df_sample.iterrows() ,total=len(df_sample)) :
idx = row['id']
ds = HuBMAPDataset(idx, sz=sz, step=step, reduce=reduce)
 dl = DataLoader(ds,bs,num_workers=0,shuffle=False,pin_memory=True)
 mp = Model_pred(models,dl,reduce)
 predict = np.zeros(( ds.shape[0]//reduce, ds.shape[1]//reduce), dtype=np.half)
 count = np.zeros_like(predict)
 for p,i,x0,y0,p00,p01,p10,p11 in iter(mp):
tile = p.squeeze(-1 ).numpy()
c = np.array([x0,y0,p00,p01,p10,p11])
 c = c//reduce
x0, y0, p00, p01, p10, p11 = c
predict[p00:p01,p10:p11] += tile[(p00-x0):(p01-x0),(p10-y0):(p11-y0)] * w[(p00-x0):(p01-x0),(p10-y0):(p11-y0)]
count[p00:p01,p10:p11] += w[(p00-x0):(p01-x0),(p10-y0):(p11-y0)]
m =(count != 0)
 predict[m] /= count[m]
del count
gc.collect()
predict = reduce_fp(predict,(ds.shape[1],ds.shape[0]), S_TH, t=TH)
 rle = rle_encode_less_memory(predict)
 names.append(idx)
 preds.append(rle)
 del tile, c, predict, ds, dl
gc.collect()
<prepare_output><EOS> 
df = pd.DataFrame({'id':names,'predicted':preds})
 df.to_csv('submission.csv',index=False )