Kaggle competition: HuBMAP - Hacking the Kidney
<SOS> metric: Dice Kaggle data source: hubmap-hacking-the-kidney<categorify>
!pip install --no-index --find-links=.. /input/preinstall efficientnet
class BaseWheatTTA:

 image_size = 1024
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes)
 tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))
def make_tta_predictions(images, net, score_threshold=0.25):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for tta_transform in tta_transforms:
result = []
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def run_wbf(predictions, image_index, image_size=1024, iou_thr=0.44, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
boxes, scores, labels = ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<categorify>
import numpy as np
import pandas as pd
import os
import glob
import gc
from functools import partial
import json
import rasterio
from rasterio.windows import Window
import yaml
import pprint
import pathlib
from tqdm.notebook import tqdm
import cv2
import tensorflow as tf
import efficientnet as efn
import efficientnet.tfkeras
import time
def get_test_transforms() :
return A.Compose([
A.Resize(height=1024, width=1024, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<categorify>
mod_paths = ['.. /input/hubmap-ensamble-model1/','.. /input/hubmap-ensamble-model2/']
THRESHOLD = 0.5
BATCH_SIZE = 256
CHECKSUM = False
test_dataset = DatasetRetrieverTest(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_test_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
test_data_loader = DataLoader(
 test_dataset,
batch_size=2,
shuffle=False,
num_workers=4,
drop_last=False,
collate_fn=collate_fn
)<categorify>
identity = rasterio.Affine(1, 0, 0, 0, 1, 0)
 fold_models = []
for mod_path in mod_paths:
with open(mod_path+'params.yaml')as file:
P = yaml.load(file, Loader=yaml.FullLoader)
 pprint.pprint(P)
 with open(mod_path + 'metrics.json')as json_file:
M = json.load(json_file)
 print('Model run datetime: '+M['datetime'])
 print('OOF val_dice_coef: ' + str(M['oof_dice_coef']))
for fold_model_path in glob.glob(mod_path+'*.h5'):
fold_models.append(tf.keras.models.load_model(fold_model_path,compile = False))
print(len(fold_models))
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 results = []
for images, image_ids in test_data_loader:
predictions = make_tta_predictions(images, net=net)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )<save_to_csv>
WINDOW = P['TILE'] if 'TILE' in P.keys() else P['DIM_FROM']
CROP_SIZE = WINDOW//2
INPUT_SIZE = P['INPUT_SIZE']
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<import_modules>
MIN_OVERLAP = WINDOW - CROP_SIZE
BOARD_CUT =(WINDOW - CROP_SIZE)//2
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import cv2
import glob
import os
import sys
from tqdm import tqdm
from itertools import product
import torch
import albumentations as A
from albumentations.pytorch.transforms import ToTensorV2
from IPython.display import clear_output<install_modules>
def rle_encode_less_memory(img):
pixels = np.concatenate([[False], img.T.flatten() , [False]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def make_grid(shape, window, min_overlap=0, board_cut = 0):

 step = window - min_overlap
x, y = shape
start_x = np.arange(0, max(x-window+board_cut,1), step, dtype=np.int64)
 start_y = np.arange(0, max(y-window+board_cut,1), step, dtype=np.int64)
 if start_x[-1] < x-window+board_cut and x-window+board_cut > 0:
start_x = np.concatenate([start_x,[x-window+board_cut]])
 if start_y[-1] < y-window+board_cut and y-window+board_cut > 0:
start_y = np.concatenate([start_y,[y-window+board_cut]])
 slices = np.zeros(( len(start_x),len(start_y), 4), dtype=np.int64)
 for i in range(len(start_x)) :
for j in range(len(start_y)) :
slices[i,j] = start_x[i], min(start_x[i]+window,x), start_y[j], min(start_y[j]+window,y)
 return slices.reshape(len(start_x)*len(start_y),4 )
!pip install /wbf
!cd /cocoapi/PythonAPI/ && pip install -e.
 sys.path.append('/cocoapi/PythonAPI/')
 !pip install /addict
!pip install /mmcv
!pip install /terminaltables
!pip install -v -e /mmdetection
sys.path.append('/mmdetection' )<load_pretrained>
AUTO = tf.data.experimental.AUTOTUNE
image_feature = {
'image': tf.io.FixedLenFeature([], tf.string),
'x1': tf.io.FixedLenFeature([], tf.int64),
'y1': tf.io.FixedLenFeature([], tf.int64)
 }
def _parse_image(example_proto):
example = tf.io.parse_single_example(example_proto, image_feature)
 image = tf.reshape(tf.io.decode_raw(example['image'],out_type=np.dtype('uint8')) ,(P['DIM'],P['DIM'], 3))
return image, example['x1'], example['y1']
def load_dataset(filenames, ordered=True):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(_parse_image)
 return dataset
def get_dataset(FILENAME):
dataset = load_dataset(FILENAME)
 dataset = dataset.batch(64)
 dataset = dataset.prefetch(AUTO)
 return dataset
<predict_on_test><EOS> 
submission = pd.DataFrame.from_dict(subm, orient='index')
 submission.to_csv('submission.csv', index=False)
 submission.head()