Kaggle competition: HuBMAP - Hacking the Kidney
<SOS> metric: Dice Kaggle data source: hubmap-hacking-the-kidney<install_modules>
!mkdir -p /tmp/pip/cache/
!cp.. /input/segmentationmodelspytorch/segmentation_models/efficientnet_pytorch-0.6.3.xyz /tmp/pip/cache/efficientnet_pytorch-0.6.3.tar.gz
!cp.. /input/segmentationmodelspytorch/segmentation_models/pretrainedmodels-0.7.4.xyz /tmp/pip/cache/pretrainedmodels-0.7.4.tar.gz
!cp.. /input/segmentationmodelspytorch/segmentation_models/segmentation-models-pytorch-0.1.2.xyz /tmp/pip/cache/segmentation_models_pytorch-0.1.2.tar.gz
!cp.. /input/segmentationmodelspytorch/segmentation_models/timm-0.1.20-py3-none-any.whl /tmp/pip/cache/
!cp.. /input/segmentationmodelspytorch/segmentation_models/timm-0.2.1-py3-none-any.whl /tmp/pip/cache/
!pip install --no-index --find-links /tmp/pip/cache/ efficientnet-pytorch
!pip install --no-index --find-links /tmp/pip/cache/ segmentation-models-pytorch
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null<define_variables>
warnings.filterwarnings("ignore" )
trained_weight_path = '.. /input/efficidentdet7fullsize/best-checkpoint-054epoch.bin'<categorify>
VERBOSE = True
DATA_DIR = '.. /input/hubmap-kidney-segmentation/test'
REDUCTION = 3
TILE_SZ = 512
MEAN = np.array([0.63482309,0.47376275,0.67814029])
 STD = np.array([0.17405236,0.23305763,0.1585981])
 MODELS_FRESH_FROZEN = [f'.. /input/ret-r101-multi3468-lf/model_{i}.pth' for i in [0,2]] + \
[f'.. /input/ens-red345/model_effb7_{i}.pth' for i in [1,2,3]] + \
[f'.. /input/ens-red345/model_effb5_{i}.pth' for i in [0,1]]
MODELS_PATHS = [MODELS_FRESH_FROZEN, MODELS_FRESH_FROZEN]
S_TH = 40
P_TH = 200*TILE_SZ//256
PUBLIC_ONLY = False
X_OVERLAP = [0., 0.5]
Y_OVERLAP = [0., 0.5]
CUSTOM_REDS = [1, 1]
THS = [0.3, 0.3]
N_BINS = 255
BATCH_SIZE = 8
NUM_WORKERS = 4
HALF_PRECISION = False
TTA_FLIPS = [[-1], [-2], [-2, -1]]
ROT_TTA_FLIPS = [0]
MASK_SZ = 4096
class BaseWheatTTA:

 image_size = 1024
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )<concatenate>
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 if os.path.exists('tmp'):
if VERBOSE:
print("Removing 'tmp' directory")
 shutil.rmtree('tmp' )
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))<predict_on_test>
class FPN(nn.Module):
def __init__(self, input_channels:list, output_channels:list):
super().__init__()
self.convs = nn.ModuleList(
 [nn.Sequential(nn.Conv2d(in_ch, out_ch*2, kernel_size=3, padding=1),
nn.ReLU(inplace=True), nn.BatchNorm2d(out_ch*2),
nn.Conv2d(out_ch*2, out_ch, kernel_size=3, padding=1))
for in_ch, out_ch in zip(input_channels, output_channels)])
 def forward(self, xs:list, last_layer):
hcs = [F.interpolate(c(x),scale_factor=2**(len(self.convs)-i),mode='bilinear')
 for i,(c,x)in enumerate(zip(self.convs, xs)) ]
hcs.append(last_layer)
 return torch.cat(hcs, dim=1)
 class UnetBlock(nn.Module):
def __init__(self, up_in_c:int, x_in_c:int, nf:int=None, blur:bool=False,
self_attention:bool=False, **kwargs):
super().__init__()
self.shuf = PixelShuffle_ICNR(up_in_c, up_in_c//2, blur=blur, **kwargs)
 self.bn = nn.BatchNorm2d(x_in_c)
 ni = up_in_c//2 + x_in_c
nf = nf if nf is not None else max(up_in_c//2,32)
 self.conv1 = ConvLayer(ni, nf, norm_type=None, **kwargs)
 self.conv2 = ConvLayer(nf, nf, norm_type=None,
xtra=SelfAttention(nf)if self_attention else None, **kwargs)
 self.relu = nn.ReLU(inplace=True)
 def forward(self, up_in:torch.Tensor, left_in:torch.Tensor)-> torch.Tensor:
s = left_in
up_out = self.shuf(up_in)
 cat_x = self.relu(torch.cat([up_out, self.bn(s)], dim=1))
return self.conv2(self.conv1(cat_x))
class _ASPPModule(nn.Module):
def __init__(self, inplanes, planes, kernel_size, padding, dilation, groups=1):
super().__init__()
self.atrous_conv = nn.Conv2d(inplanes, planes, kernel_size=kernel_size,
stride=1, padding=padding, dilation=dilation, bias=False, groups=groups)
 self.bn = nn.BatchNorm2d(planes)
 self.relu = nn.ReLU()
self._init_weight()
def forward(self, x):
x = self.atrous_conv(x)
 x = self.bn(x)
 return self.relu(x)
 def _init_weight(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d):
torch.nn.init.kaiming_normal_(m.weight)
 elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
 m.bias.data.zero_()
class ASPP(nn.Module):
def __init__(self, inplanes=512, mid_c=256, dilations=[6, 12, 18, 24], out_c=None):
super().__init__()
self.aspps = [_ASPPModule(inplanes, mid_c, 1, padding=0, dilation=1)] + \
[_ASPPModule(inplanes, mid_c, 3, padding=d, dilation=d,groups=4)for d in dilations]
self.aspps = nn.ModuleList(self.aspps)
 self.global_pool = nn.Sequential(nn.AdaptiveMaxPool2d(( 1, 1)) ,
nn.Conv2d(inplanes, mid_c, 1, stride=1, bias=False),
nn.BatchNorm2d(mid_c), nn.ReLU())
 out_c = out_c if out_c is not None else mid_c
self.out_conv = nn.Sequential(nn.Conv2d(mid_c*(2+len(dilations)) , out_c, 1, bias=False),
nn.BatchNorm2d(out_c), nn.ReLU(inplace=True))
self.conv1 = nn.Conv2d(mid_c*(2+len(dilations)) , out_c, 1, bias=False)
 self._init_weight()
def forward(self, x):
x0 = self.global_pool(x)
 xs = [aspp(x)for aspp in self.aspps]
x0 = F.interpolate(x0, size=xs[0].size() [2:], mode='bilinear', align_corners=True)
 x = torch.cat([x0] + xs, dim=1)
 return self.out_conv(x)
 def _init_weight(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d):
torch.nn.init.kaiming_normal_(m.weight)
 elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
 m.bias.data.zero_()
def make_tta_predictions(images, nets, score_threshold=0.25):
all_predictions = []
images = torch.stack(images ).float().cuda()
for fold_num, net in enumerate(nets):
with torch.no_grad() :
predictions = []
for tta_transform in tta_transforms:
result = []
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 all_predictions.append(predictions)
 return all_predictions
def run_wbf(all_predictions, image_index, image_size=1024, iou_thr=0.44, skip_box_thr=0.43, weights=None):
for i in range(len(nets)) :
predictions = all_predictions[i]
if i == 0:
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).tolist() for prediction in predictions]
else:
boxes += [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores += [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels += [np.ones(prediction[image_index]['scores'].shape[0] ).tolist() for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<choose_model_class>
class UneXt50(nn.Module):
def __init__(self, stride=1, **kwargs):
super().__init__()
m = ResNet(Bottleneck, [3, 4, 6, 3], groups=32, width_per_group=4)
 self.enc0 = nn.Sequential(m.conv1, m.bn1, nn.ReLU(inplace=True))
self.enc1 = nn.Sequential(nn.MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1),
m.layer1)
 self.enc2 = m.layer2
self.enc3 = m.layer3
self.enc4 = m.layer4
self.aspp = ASPP(2048,256,out_c=512,dilations=[stride*1,stride*2,stride*3,stride*4])
 self.drop_aspp = nn.Dropout2d(0.5)
 self.dec4 = UnetBlock(512,1024,256)
 self.dec3 = UnetBlock(256,512,128)
 self.dec2 = UnetBlock(128,256,64)
 self.dec1 = UnetBlock(64,64,32)
 self.fpn = FPN([512,256,128,64],[16]*4)
 self.drop = nn.Dropout2d(0.1)
 self.final_conv = ConvLayer(32+16*4, 1, ks=1, norm_type=None, act_cls=None)
 def forward(self, x):
enc0 = self.enc0(x)
 enc1 = self.enc1(enc0)
 enc2 = self.enc2(enc1)
 enc3 = self.enc3(enc2)
 enc4 = self.enc4(enc3)
 enc5 = self.aspp(enc4)
 dec3 = self.dec4(self.drop_aspp(enc5),enc3)
 dec2 = self.dec3(dec3,enc2)
 dec1 = self.dec2(dec2,enc1)
 dec0 = self.dec1(dec1,enc0)
 x = self.fpn([enc5, dec3, dec2, dec1], dec0)
 x = self.final_conv(self.drop(x))
x = F.interpolate(x,scale_factor=2,mode='bilinear')
 return x
class UneXt101(nn.Module):
def __init__(self, stride=1, **kwargs):
super().__init__()
m = ResNet(Bottleneck, [3, 4, 23, 3], groups=32, width_per_group=16)
 self.enc0 = nn.Sequential(m.conv1, m.bn1, nn.ReLU(inplace=True))
self.enc1 = nn.Sequential(nn.MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1),
m.layer1)
 self.enc2 = m.layer2
self.enc3 = m.layer3
self.enc4 = m.layer4
self.aspp = ASPP(2048,256,out_c=512,dilations=[stride*1,stride*2,stride*3,stride*4])
 self.drop_aspp = nn.Dropout2d(0.5)
 self.dec4 = UnetBlock(512,1024,256)
 self.dec3 = UnetBlock(256,512,128)
 self.dec2 = UnetBlock(128,256,64)
 self.dec1 = UnetBlock(64,64,32)
 self.fpn = FPN([512,256,128,64],[16]*4)
 self.drop = nn.Dropout2d(0.1)
 self.final_conv = ConvLayer(32+16*4, 1, ks=1, norm_type=None, act_cls=None)
 def forward(self, x):
enc0 = self.enc0(x)
 enc1 = self.enc1(enc0)
 enc2 = self.enc2(enc1)
 enc3 = self.enc3(enc2)
 enc4 = self.enc4(enc3)
 enc5 = self.aspp(enc4)
 dec3 = self.dec4(self.drop_aspp(enc5),enc3)
 dec2 = self.dec3(dec3,enc2)
 dec1 = self.dec2(dec2,enc1)
 dec0 = self.dec1(dec1,enc0)
 x = self.fpn([enc5, dec3, dec2, dec1], dec0)
 x = self.final_conv(self.drop(x))
x = F.interpolate(x,scale_factor=2,mode='bilinear')
 return x
class Unet50(nn.Module):
def __init__(self, stride=1, **kwargs):
super().__init__()
m = ResNet(Bottleneck, [3, 4, 6, 3])
 self.enc0 = nn.Sequential(m.conv1, m.bn1, nn.ReLU(inplace=True))
self.enc1 = nn.Sequential(nn.MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1),
m.layer1)
 self.enc2 = m.layer2
self.enc3 = m.layer3
self.enc4 = m.layer4
self.aspp = ASPP(2048,256,out_c=512,dilations=[stride*1,stride*2,stride*3,stride*4])
 self.drop_aspp = nn.Dropout2d(0.5)
 self.dec4 = UnetBlock(512,1024,256)
 self.dec3 = UnetBlock(256,512,128)
 self.dec2 = UnetBlock(128,256,64)
 self.dec1 = UnetBlock(64,64,32)
 self.fpn = FPN([512,256,128,64],[16]*4)
 self.drop = nn.Dropout2d(0.1)
 self.final_conv = ConvLayer(32+16*4, 1, ks=1, norm_type=None, act_cls=None)
 def forward(self, x):
enc0 = self.enc0(x)
 enc1 = self.enc1(enc0)
 enc2 = self.enc2(enc1)
 enc3 = self.enc3(enc2)
 enc4 = self.enc4(enc3)
 enc5 = self.aspp(enc4)
 dec3 = self.dec4(self.drop_aspp(enc5),enc3)
 dec2 = self.dec3(dec3,enc2)
 dec1 = self.dec2(dec2,enc1)
 dec0 = self.dec1(dec1,enc0)
 x = self.fpn([enc5, dec3, dec2, dec1], dec0)
 x = self.final_conv(self.drop(x))
x = F.interpolate(x,scale_factor=2,mode='bilinear')
 return x
def create_model_efficientdet_test(compound_coef:int, weight_path=None,
num_classes=1, image_size=1024, pretrained=False, model_ema=False):
config = get_efficientdet_config(f'tf_efficientdet_d{compound_coef}')
 model = EfficientDet(config, pretrained_backbone=pretrained)
 config.num_classes = num_classes
config.image_size = image_size
model.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(weight_path)
 if model_ema:
model.load_state_dict(checkpoint['state_dict_ema'])
 else:
model.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
model = DetBenchEval(model, config)
 anchors.MAX_DETECTIONS_PER_IMAGE=250
model.eval() ;
return model.cuda()
nets = [create_model_efficientdet_test(compound_coef=4,
weight_path='.. /input/efficidentdet4-fullsize/best-checkpoint-058epoch.bin',
pretrained=False,
model_ema=True),
create_model_efficientdet_test(compound_coef=5,
weight_path='.. /input/efficientdet5-fullsize/best-checkpoint-052epoch.bin',
pretrained=False),
create_model_efficientdet_test(compound_coef=7,
weight_path=trained_weight_path,
pretrained=False,
model_ema=True)] 
<categorify>
MODELS = []
for models_list in MODELS_PATHS:
models_i = []
for ij,path in enumerate(models_list):
state_dict = torch.load(path,map_location=torch.device('cpu'))
if ij < 2:
model = UneXt101()
elif ij < 5:
model = smp.Unet(encoder_name='efficientnet-b7', classes=1, activation=None, encoder_weights=None)
 else:
model = smp.Unet(encoder_name='efficientnet-b5', classes=1, activation=None, encoder_weights=None)
 model.load_state_dict(state_dict)
 model.float()
model.eval()
model.to(device)
 models_i.append(model)
 del state_dict
MODELS.append(models_i )
def get_test_transforms() :
return A.Compose([
A.Resize(height=1024, width=1024, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<data_type_conversions>
def _tile_resize_save(img, img_id, tile_sz, reduce=1):

 x = 0
while x < img.shape[0]:
y = 0
while y < img.shape[1]:
img_tile = img[x:x+tile_sz,y:y+tile_sz]
if reduce > 1:
new_dim =(img_tile.shape[1]//reduce,img_tile.shape[0]//reduce)
 img_tile = cv2.resize(img_tile, new_dim, interpolation = cv2.INTER_AREA)
 save_path = "%s_%d_%d.png" %(img_id, x//reduce, y//reduce)
 Image.fromarray(img_tile ).save(save_path)
 y += tile_sz
x += tile_sz
final_x =(( x-tile_sz)//tile_sz)*(tile_sz//reduce)+ img_tile.shape[0]
final_y =(( y-tile_sz)//tile_sz)*(tile_sz//reduce)+ img_tile.shape[1]
return(final_x, final_y, 3 )
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class DatasetRetrieverTest(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]<categorify>
def load_resize(idx, reduce):

 img = load_image(os.path.join(DATA_DIR,idx+'.tiff'))
init_shape = img.shape
shape = _tile_resize_save(img, idx,(MASK_SZ*REDUCTION), reduce=REDUCTION)
 img = _reconstruct_img(idx,(MASK_SZ*REDUCTION)//REDUCTION, shape)
 return img, init_shape
test_dataset = DatasetRetrieverTest(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_test_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
test_data_loader = DataLoader(
 test_dataset,
batch_size=2,
shuffle=False,
num_workers=4,
drop_last=False,
collate_fn=collate_fn
)<predict_on_test>
def _get_nored_pads(initW, initH, upW, upH, xa, xb, ya, yb):

 px = xa/(xa+xb)
 py = ya/(ya+yb)
 padx = upW - initW
pady = upH - initH
assert padx > 0
assert pady > 0
xa = int(px*padx)
 xb = padx - xa
ya = int(py*pady)
 yb = pady - ya
return xa, xb, ya, yb
def _add_padding(img, init_sz, img_shape, p0, p1):

 start = time.time()
if VERBOSE:
print(" > Adding padding to make the image dividable into tiles...")
 if VERBOSE:
print(" > Before reduction:", img_shape)
 print(" > After reduction:", img.shape)
 pad0_ = TILE_SZ - img.shape[0]%TILE_SZ
x_pad = int(TILE_SZ*p0)
 xa =(pad0_//2 + x_pad)
 xb = pad0_+TILE_SZ-(pad0_//2 + x_pad)
 pad0_lr = [xa, xb]
pad1_ = TILE_SZ - img.shape[1]%TILE_SZ
y_pad = int(TILE_SZ*p1)
 ya =(pad1_//2 + y_pad)
 yb = pad1_+TILE_SZ-(pad1_//2 + y_pad)
 pad1_lr = [ya, yb]
img = np.pad(img,[pad0_lr, pad1_lr,[0,0]],constant_values=0)
 if VERBOSE:
print(" > After padding:", img.shape, "Time =", time.time() - start, "s")
 xa, xb, ya, yb = _get_nored_pads(img_shape[0], img_shape[1],
REDUCTION*img.shape[0], REDUCTION*img.shape[1],
xa, xb, ya, yb)
 return img, xa, xb, ya, yb, img.shape
def predict_test(data_loader,nets):
results = []
testdf_psuedo = []
for images, image_ids in data_loader:
predictions = make_tta_predictions(images=images, nets=nets)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
for box in boxes:
result = {
'image_id': image_id,
'source': 'BULLBULL',
'width':1024,
'height':1024,
'x': box[0],
'y': box[1],
'w': box[2],
'h': box[3]
}
testdf_psuedo.append(result)
 test_df_pseudo = pd.DataFrame(testdf_psuedo, columns=['image_id', 'width', 'height', 'source', 'x', 'y', 'w', 'h'])
 return test_df_pseudo<predict_on_test>
def _split_image(img):

 start = time.time()
if VERBOSE:
print(" > Splitting image into tiles...")
 assert not img.shape[0]%TILE_SZ
assert not img.shape[1]%TILE_SZ
img = img.reshape(img.shape[0]//TILE_SZ,
TILE_SZ,
img.shape[1]//TILE_SZ,
TILE_SZ,
3)
 img = img.transpose(0,2,1,3,4 ).reshape(-1,TILE_SZ,TILE_SZ,3)
 if VERBOSE:
print(" > Splitting done! Time =", time.time() - start)
 return img
test_df_pseudo = predict_test(test_data_loader, nets )<load_from_csv>
def img2tensor(img, dtype:np.dtype=np.float32):
if img.ndim==2: img = np.expand_dims(img,2)
 img = np.transpose(img,(2,0,1))
return torch.from_numpy(img.astype(dtype, copy=False))
class HuBMAPTestDataset(Dataset):
def __init__(self, idxs):
self.fnames = idxs
def __len__(self):
return len(self.fnames)
 def __getitem__(self, idx):
im = cv2.imread("tmp/%d.png" %(self.fnames[idx],))
return img2tensor(( im/255.0 - MEAN)/STD )
def generate_train(test_df_pseudo, fold_=1):
print("Loading training set...")
 marking = pd.read_csv('.. /input/wheat-cleaned-dataset/clean_train_v2.csv')
 marking = marking[marking['area']> 300]
df_with_pseudo = pd.concat([marking, test_df_pseudo])
 index = list(set(marking.image_id))
fold = 0
val_index = index[len(index)*fold//5:len(index)*(fold+1)//5]
train_index = index[:len(index)*fold//5] + index[len(index)*(fold+1)//5:]
train_df_pseudo_fold = test_df_pseudo['image_id'].unique()
train_fold = np.concatenate(( train_index, train_df_pseudo_fold))
print('train dataset size:', len(train_fold))
val_fold = np.array(val_index)
 print('val dataset size:', len(val_fold))
return df_with_pseudo, train_fold, val_fold<split>
def _make_tiles_dataloader(idxs):

 start = time.time()
ds = HuBMAPTestDataset(idxs)
 dl = DataLoader(ds, BATCH_SIZE,
num_workers=NUM_WORKERS,
shuffle=False,
pin_memory=True)
 if VERBOSE:
print(" > Tiles dataset created! Time =", time.time() - start)
 return dl
df_with_pesudo, train_fold, val_fold = generate_train(test_df_pseudo )<define_search_model>
def _generate_masks(dl, idxs, n_tiles, init_sz, group):

 start = time.time()
if VERBOSE:
print(" > Generating masks...")
 red = CUSTOM_REDS[group]
mp = Model_pred(MODELS[group], dl, red)
 mask = torch.zeros(n_tiles,
init_sz,
init_sz,
dtype=torch.uint8)
 for i, p in zip(idxs,iter(mp)) : mask[i] = p.squeeze(-1)
 if VERBOSE:
print(" > Masks generated! Time =", time.time() - start)
 return mask
def get_train_transforms() :
return A.Compose(
 [
A.OneOf([
A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit= 0.2,
val_shift_limit=0.2, p=0.9),
A.RandomBrightnessContrast(brightness_limit=0.2,
contrast_limit=0.2, p=0.9),
],p=0.9),
A.ToGray(p=0.01),
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.RandomRotate90(p=0.5),
A.Resize(height=1024, width=1024, p=1),
ToTensorV2(p=1.0),
],
p=1.0,
bbox_params=A.BboxParams(
 format='pascal_voc',
min_area=0,
min_visibility=0,
label_fields=['labels'] 
)
)
 def get_valid_transforms() :
return A.Compose(
 [
A.Resize(height=1024, width=1024, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0,
bbox_params=A.BboxParams(
 format='pascal_voc',
min_area=0,
min_visibility=0,
label_fields=['labels'] 
)
 )<define_variables>
class Model_pred:
def __init__(self, models, dl, red, half:bool=False):
self.models = models
self.dl = dl
self.half = half
self.red = red
def __iter__(self):
with torch.no_grad() :
for x in iter(self.dl):
x = x.to(device)
 x = F.interpolate(x, scale_factor=1/self.red, mode='bilinear')
 if self.half: x = x.half()
py = 0.
 for rot_flip in ROT_TTA_FLIPS:
for model in self.models:
xr = torch.rot90(x, rot_flip, [-2, -1])
 p = model(xr)
 p = torch.rot90(p, -rot_flip, [-2, -1])
 p = torch.sigmoid(p ).detach()
py += p
for f in TTA_FLIPS:
xf = torch.rot90(x, rot_flip, [-2, -1])
 xf = torch.flip(xf,f)
 for model in self.models:
p = model(xf)
 p = torch.flip(p,f)
 p = torch.rot90(p, -rot_flip, [-2, -1])
 py += torch.sigmoid(p ).detach()
py /=(1+len(TTA_FLIPS)) *len(ROT_TTA_FLIPS)
 py /= len(self.models)
 py = F.upsample(py, scale_factor=REDUCTION*self.red, mode="bilinear")
 py = py.permute(0,2,3,1 ).float().cpu()
py =(N_BINS*py ).int()
batch_size = len(py)
 for i in range(batch_size):
yield py[i]
def __len__(self):
return len(self.dl.dataset )
TRAIN_ROOT_PATH = '.. /input/global-wheat-detection/train'
class DatasetRetriever(Dataset):
def __init__(self, marking, image_ids, transforms=None, test=False, mixup=True):
super().__init__()
self.image_ids = image_ids
self.marking = marking
self.transforms = transforms
self.test = test
self.mixup = mixup
self.mosaic_border = [-1024//2, -1024//2]
def __getitem__(self, index: int):
image_id = self.image_ids[index]
origin_image = False
if self.test or random.random() > 0.5:
image, boxes = self.load_image_and_boxes(index)
 origin_image = True
else:
image, boxes = self.load_cutmix_image_and_boxes(index)
 if(random.random() < 0.9)and(self.test == False):
image, boxes = cutout(image, boxes)
 labels = torch.ones(( boxes.shape[0],), dtype=torch.int64)
 target = {}
target['boxes'] = boxes
target['labels'] = labels
target['image_id'] = torch.tensor([index])
 if self.transforms:
for i in range(10):
sample = self.transforms(**{
'image': image,
'bboxes': target['boxes'],
'labels': labels
})
 if len(sample['bboxes'])> 0:
image = sample['image']
target['labels'] = torch.ones(( len(sample['bboxes']),), dtype=torch.int64)
 target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)
 target['boxes'][:,[0,1,2,3]] = target['boxes'][:,[1,0,3,2]]
break
return image, target, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def load_image_and_boxes(self, index):
image_id = self.image_ids[index]
if os.path.exists(f'{TRAIN_ROOT_PATH}/{image_id}.jpg'):
image = cv2.imread(f'{TRAIN_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 else:
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image = cv2.resize(image,(1024, 1024))
image /= 255.0
records = self.marking[self.marking['image_id'] == image_id]
boxes = records[['x', 'y', 'w', 'h']].values
boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
return image, boxes
def load_cutmix_image_and_boxes(self, index, imsize=1024):

 w, h = imsize, imsize
s = imsize
xc, yc = [int(random.uniform(-x, 2 * s + x)) for x in self.mosaic_border]
indexes = [index] + [random.randint(0, self.image_ids.shape[0] - 1)for _ in range(3)]
result_image = np.full(( imsize*2, imsize*2, 3), 114/255, dtype=np.float32)
 result_boxes = []
for i, index in enumerate(indexes):
image, boxes = self.load_image_and_boxes(index)
 if i == 0:
x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc
x1b, y1b, x2b, y2b = w -(x2a - x1a), h -(y2a - y1a), w, h
elif i == 1:
x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc
x1b, y1b, x2b, y2b = 0, h -(y2a - y1a), min(w, x2a - x1a), h
elif i == 2:
x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = w -(x2a - x1a), 0, max(xc, w), min(y2a - y1a, h)
 elif i == 3:
x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)
 result_image[y1a:y2a, x1a:x2a] = image[y1b:y2b, x1b:x2b]
padw = x1a - x1b
padh = y1a - y1b
boxes[:, 0] += padw
boxes[:, 1] += padh
boxes[:, 2] += padw
boxes[:, 3] += padh
result_boxes.append(boxes)
 result_boxes = np.concatenate(result_boxes, 0)
 np.clip(result_boxes[:, 0:], 0, 2 * s, out=result_boxes[:, 0:])
 result_boxes = result_boxes.astype(np.int32)
 result_boxes = result_boxes[np.where(( result_boxes[:, 2] - result_boxes[:, 0])*(result_boxes[:, 3] - result_boxes[:, 1])> 300.0)]
result_boxes = result_boxes[np.where(( result_boxes[:, 2] - result_boxes[:, 0])>= 10.0)]
result_boxes = result_boxes[np.where(( result_boxes[:, 3] - result_boxes[:, 1])>= 10.0)]
result_image, result_boxes = random_affine(result_image, targets=result_boxes)
 return result_image, result_boxes<create_dataframe>
def _reshape_depad_mask(mask, init_shape, init_sz, p0, p1, xa, xb, ya, yb):

 start = time.time()
if VERBOSE:
print(" > Merge tiled masks into one mask and crop padding...")
 mask = mask.view(init_shape[0]//TILE_SZ,
init_shape[1]//TILE_SZ,
init_sz,
init_sz ).\
permute(0,2,1,3 ).reshape(init_shape[0]*REDUCTION,
init_shape[1]*REDUCTION)
 mask = mask[xa:-xb,ya:-yb]
if VERBOSE:
print(" > Mask created! Shape =", mask.shape,"Time =", time.time() - start)
 return mask
train_dataset = DatasetRetriever(
 image_ids=train_fold,
marking=df_with_pesudo,
transforms=get_train_transforms() ,
test=False, 
)
 validation_dataset = DatasetRetriever(
 image_ids=val_fold,
marking=df_with_pesudo,
transforms=get_valid_transforms() ,
test=True,
)<normalization>
def _save_mask_tiles(mask, idx, p0, p1):
start = time.time()
if VERBOSE:
print(" > Saving tiles in HDD memory...")
 x = 0
while x < mask.shape[0]:
y = 0
while y < mask.shape[1]:
mask_tile = mask[x:x+MASK_SZ,y:y+MASK_SZ].numpy()
save_path = "%s_%d_%d_%s_%s.png" %(idx, x, y, str(p0), str(p1))
Image.fromarray(mask_tile ).save(save_path)
 y += MASK_SZ
x += MASK_SZ
if VERBOSE:
print("Tiles saved! Time =", time.time() - start )
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count<define_variables>
def make_one_prediction(img, group, idx, img_shape, p0, p1):

 init_sz = TILE_SZ*REDUCTION
img, xa, xb, ya, yb, img_shape_p = _add_padding(img, init_sz, img_shape,
p0, p1)
 img = _split_image(img)
 n_tiles = img.shape[0]
idxs = _select_tiles(img)
 dl = _make_tiles_dataloader(idxs)
 mask = _generate_masks(dl, idxs, n_tiles, init_sz, group)
 mask = _reshape_depad_mask(mask, img_shape_p, init_sz,
p0, p1, xa, xb, ya, yb)
 gc.collect()
shutil.rmtree('tmp')
 _save_mask_tiles(mask, idx, p0, p1 )
class TrainGlobalConfig:
num_workers = 2
batch_size = 2
use_amp = True
use_ema = True
n_epochs = 8
total_batch_size = 8
clip_grad = True
accumulate_steps = int(total_batch_size / batch_size)
 lr = 0.0085
folder = 'effdet7'
verbose = True
verbose_step = 1
step_scheduler = False
validation_scheduler = True
SchedulerClass = CosineLRScheduler
scheduler_params = dict(
 t_initial=n_epochs,
t_mul=1.0,
lr_min = 1e-8, 
)
<choose_model_class>
def get_mask_tiles(idx, p0_list, p1_list):

 group = _get_group(os.path.join(DATA_DIR,idx+'.tiff'))
TH = THS[group]
img, init_shape = load_resize(idx, REDUCTION)
 for p0 in p0_list:
for p1 in p1_list:
make_one_prediction(img, group, idx, init_shape, p0, p1)
 return init_shape, TH
def is_parallel(model):
return type(model)in(nn.parallel.DataParallel, nn.parallel.DistributedDataParallel)
 class ModelEMA:

 def __init__(self, model, decay=0.9999, updates=0):
self.ema = deepcopy(model.module if is_parallel(model)else model ).eval()
self.updates = updates
self.decay = lambda x: decay *(1 - math.exp(-x / 2000))
for p in self.ema.parameters() :
p.requires_grad_(False)
 def update(self, model):
with torch.no_grad() :
self.updates += 1
d = self.decay(self.updates)
 msd = model.module.state_dict() if is_parallel(model)else model.state_dict()
for k, v in self.ema.state_dict().items() :
if v.dtype.is_floating_point:
v *= d
v +=(1.- d)* msd[k].detach()
def update_attr(self, model, include=() , exclude=('process_group', 'reducer')) :
copy_attr(self.ema, model, include, exclude )<set_options>
def make_predictions(idx):

 init_shape, TH = get_mask_tiles(idx, X_OVERLAP, Y_OVERLAP)
 mask = torch.zeros(*init_shape[:2], dtype=torch.uint8)
 x = 0
while x < init_shape[0]:
y = 0
while y < init_shape[1]:
mask_tile = 0.
 for p0 in X_OVERLAP:
for p1 in Y_OVERLAP:
tile_path = "%s_%d_%d_%s_%s.png" %(idx, x, y, str(p0), str(p1))
mask_tile += torch.tensor(np.asarray(Image.open(tile_path), dtype=int))
os.remove(tile_path)
 NEW_TH = int(N_BINS*len(X_OVERLAP)*len(Y_OVERLAP)*TH)
 mask[x:x+MASK_SZ,y:y+MASK_SZ] = mask_tile>NEW_TH
y += MASK_SZ
x += MASK_SZ
if VERBOSE:
print(" > Converting to RLE...")
 rle = rle_encode_less_memory(mask.numpy())
 del mask
return rle
warnings.filterwarnings("ignore")
 class Fitter:
def __init__(self, model, device, config):
self.config = config
self.epoch = 0
self.base_dir = f'./{config.folder}'
if not os.path.exists(self.base_dir):
os.makedirs(self.base_dir)
 self.log_path = f'{self.base_dir}/log.txt'
self.best_summary_loss = 10**5
self.model = model
self.device = device
self.use_amp = config.use_amp
self.accumulate_steps = config.accumulate_steps
self.clip_grad = config.clip_grad
self.max_norm = 10.0
self.use_ema = config.use_ema
self.model_ema = None
self.model_ema_decay = 0.999
pg0, pg1, pg2 = [], [], []
for k, v in self.model.named_parameters() :
if v.requires_grad:
if '.bias' in k:
pg2.append(v)
 elif '.weight' in k and '.bn' not in k:
pg1.append(v)
 else:
pg0.append(v)
 self.optimizer = torch.optim.SGD(pg0, lr=config.lr, momentum=0.9, nesterov=True)
 self.optimizer.add_param_group({'params': pg1, 'weight_decay': 5e-4})
 self.optimizer.add_param_group({'params': pg2})
 self.model.to(self.device)
 if self.use_amp:
self.model, self.optimizer = amp.initialize(self.model, self.optimizer, opt_level="O1",verbosity=0)
 if self.use_ema:
self.model_ema = ModelEMA(self.model, decay=self.model_ema_decay)
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 self.log(f'Fitter prepared.Device is {self.device}')
 def fit(self, train_loader, validation_loader):
best_epoch = 0
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
self.log(f'
{timestamp}
LR: {lr}')
 t = time.time()
summary_loss = self.train_one_epoch(train_loader)
 self.log(f'[RESULT]: Train.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 self.save(f'{self.base_dir}/last-checkpoint.bin')
 t = time.time()
if self.use_ema:
summary_loss = self.validation(self.model_ema.ema.module if hasattr(self.model_ema.ema, 'module')else self.model_ema.ema,
validation_loader)
 else:
summary_loss = self.validation(self.model, validation_loader)
 self.log(f'[RESULT]: Val.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 if summary_loss.avg < self.best_summary_loss:
self.best_summary_loss = summary_loss.avg
self.model.eval()
self.save(f'{self.base_dir}/best-checkpoint-{str(self.epoch ).zfill(3)}epoch.bin')
 for path in sorted(glob(f'{self.base_dir}/best-checkpoint-*epoch.bin')) [:-3]:
os.remove(path)
 best_epoch = e
if self.config.validation_scheduler:
self.scheduler.step(e+1)
 self.epoch += 1
return best_epoch
def validation(self, model, val_loader):
model.eval()
summary_loss = AverageMeter()
t = time.time()
for step,(images, targets, image_ids)in enumerate(val_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Val Step {step}/{len(val_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 with torch.no_grad() :
images = torch.stack(images ).to(self.device ).float()
batch_size = images.shape[0]
target_input = {'bbox':[target['boxes'].to(self.device ).float() for target in targets],
'cls': [target['labels'].to(self.device ).float() for target in targets]}
loss = model(images, target_input['bbox'], target_input['cls'])
 loss, class_loss, box_loss = loss
summary_loss.update(loss.detach().item() , batch_size)
 return summary_loss
def train_one_epoch(self, train_loader):
self.model.train()
summary_loss = AverageMeter()
t = time.time()
self.optimizer.zero_grad()
for step,(images, targets, image_ids)in enumerate(train_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Train Step {step}/{len(train_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 images = torch.stack(images ).to(self.device ).float()
batch_size = images.shape[0]
target_input = {'bbox':[target['boxes'].to(self.device ).float() for target in targets],
'cls': [target['labels'].to(self.device ).float() for target in targets]}
loss = self.model(images, target_input['bbox'], target_input['cls'])
 loss, class_loss, box_loss = loss
if self.use_amp:
with amp.scale_loss(loss, self.optimizer)as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
if(step+1)% self.accumulate_steps == 0:
if self.clip_grad:
if self.use_amp:
torch.nn.utils.clip_grad_norm_(amp.master_params(self.optimizer), self.max_norm)
 else:
torch.nn.utils.clip_grad_norm_(self.model.parameters() , self.max_norm)
 self.optimizer.step()
self.optimizer.zero_grad()
if self.model_ema is not None:
self.model_ema.update(self.model)
 summary_loss.update(loss.detach().item() , batch_size)
 if self.config.step_scheduler:
self.scheduler.step()
return summary_loss
def save(self, path):
self.model.eval()
if self.use_ema:
torch.save({
'model_state_dict': self.model.model.state_dict() ,
'state_dict_ema': self.model_ema.ema.module.state_dict() if hasattr(self.model_ema.ema, 'module')else self.model_ema.ema.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_summary_loss': self.best_summary_loss,
'epoch': self.epoch,
}, path)
 else:
torch.save({
'model_state_dict': self.model.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_summary_loss': self.best_summary_loss,
'epoch': self.epoch,
}, path)
 def load(self, path):
checkpoint = torch.load(path)
 self.model.model.load_state_dict(checkpoint['model_state_dict'])
 self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
 self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
 self.best_summary_loss = checkpoint['best_summary_loss']
self.epoch = checkpoint['epoch'] + 1
def log(self, message):
if self.config.verbose:
print(message)
 with open(self.log_path, 'a+')as logger:
logger.write(f'{message}
' )<load_pretrained>
def rle_encode_less_memory(img):
pixels = img.T.flatten()
pixels[0] = 0
pixels[-1] = 0
runs = np.where(pixels[1:] != pixels[:-1])[0] + 2
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
def collate_fn(batch):
return tuple(zip(*batch))
def run_training() :
device = torch.device('cuda:0')
 net.to(device)
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=RandomSampler(train_dataset),
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig.num_workers,
collate_fn=collate_fn, 
)
 val_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
num_workers=TrainGlobalConfig.num_workers,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False,
collate_fn=collate_fn, 
)
 fitter = Fitter(model=net, device=device, config=TrainGlobalConfig)
 best_epoch = fitter.fit(train_loader, val_loader)
 return best_epoch<choose_model_class>
df_sample = pd.read_csv('.. /input/hubmap-kidney-segmentation/sample_submission.csv')
 names,preds = [],[]
if PUBLIC_ONLY:
samples = ['d488c759a', 'aa05346ff','57512b7f1','3589adb90','2ec3f1bb9']
samples_n = [id for id in df_sample.id if id not in samples]
for x in samples_n:
names += [x]
preds += [np.NaN]*len(samples_n)
 df_sample = df_sample.loc[df_sample.id.isin(samples)]
def get_net() :
config = get_efficientdet_config('tf_efficientdet_d7')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size = 1024
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(trained_weight_path)
 net.load_state_dict(checkpoint['state_dict_ema'])
 return DetBenchTrain(net, config)
 net = get_net()<train_model>
for idx,row in tqdm(df_sample.iterrows() ,total=len(df_sample)) :
idx = row['id']
print("Computing predictions for image", idx)
 rle = make_predictions(idx)
 names.append(idx)
 preds.append(rle )
<load_pretrained><EOS> 
df = pd.DataFrame({'id': names, 'predicted': preds})
 df.to_csv('submission.csv',index=False )