Kaggle competition: HuBMAP - Hacking the Kidney
<SOS> metric: Dice Kaggle data source: hubmap-hacking-the-kidney<define_variables>
sys.path.append(".. /input/zarrkaggleinstall")
 sys.path.append(".. /input/segmentation-models-pytorch-install")
 !pip install -q --no-deps.. /input/deepflash2-lfs
MMDETECTION_PATH = '/mmdetection/'
TEST_IMAGES_DIR = '/kaggle/input/global-wheat-detection/test/'
test_files=glob.glob(f'{TEST_IMAGES_DIR}*.jpg' )<define_variables>
def rle_encode_less_memory(img):
pixels = img.T.flatten()
pixels[0] = 0
pixels[-1] = 0
runs = np.where(pixels[1:] != pixels[:-1])[0] + 2
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def load_model_weights(model, file, strict=True):
state = torch.load(file, map_location='cpu')
 stats = state['stats']
model_state = state['model']
model.load_state_dict(model_state, strict=strict)
 return model, stats
CONFIGS_PATHS = ['/kaggle/input/glf-configs/',
'/kaggle/input/glf-configs/',
'/kaggle/input/glf-configs/']
WEIGHTS_PATHS = ['/kaggle/input/glf-weights/',
'/kaggle/input/glf-weights/',
'/kaggle/input/glf-weights/']
WEIGHTS_NAMES = ['m1_s2_epoch_24.pth',
'm2_s7_epoch_30_multiscale.pth',
'm2_s3_epoch_24.pth']
MODEL_NAMES = ['gfl_x101',
'gfl_x101',
'gfl_x101']
models = []
for CONFIGS_PATH, WEIGHTS_PATH, WEIGHTS_NAME, MODEL_NAME in zip(CONFIGS_PATHS, WEIGHTS_PATHS, WEIGHTS_NAMES, MODEL_NAMES):
config_file = os.path.join(CONFIGS_PATH, MODEL_NAME + '.py')
 checkpoint_file = os.path.join(WEIGHTS_PATH,WEIGHTS_NAME)
 models.append(init_detector(config_file, checkpoint_file, device='cuda:0'))
if len(models)==1:
wbf_weights = None
else:
wbf_weights = [1] *(len(models)+1 )<init_hyperparams>
@patch
def read_img(self:BaseDataset, *args, **kwargs):
image = tifffile.imread(args[0])
 if len(image.shape)== 5:
image = image.squeeze().transpose(1, 2, 0)
 elif image.shape[0] == 3:
image = image.transpose(1, 2, 0)
 return image
@patch
def apply(self:DeformationField, data, offset=(0, 0), pad=(0, 0), order=1):
"Apply deformation field to image using interpolation"
outshape = tuple(int(s - p)for(s, p)in zip(self.shape, pad))
coords = [np.squeeze(d ).astype('float32' ).reshape(*outshape)for d in self.get(offset, pad)]
sl = []
for i in range(len(coords)) :
cmin, cmax = int(coords[i].min()), int(coords[i].max())
 dmax = data.shape[i]
if cmin<0:
cmax = max(-cmin, cmax)
 cmin = 0
elif cmax>dmax:
cmin = min(cmin, 2*dmax-cmax)
 cmax = dmax
coords[i] -= cmin
else: coords[i] -= cmin
sl.append(slice(cmin, cmax))
if len(data.shape)== len(self.shape)+ 1:
tile = np.empty(( *outshape, data.shape[-1]))
for c in range(data.shape[-1]):
tile[..., c] = cv2.remap(data[sl[0],sl[1], c]/255, coords[1],coords[0], interpolation=order, borderMode=cv2.BORDER_REFLECT)
 else:
tile = cv2.remap(data[sl[0], sl[1]], coords[1], coords[0], interpolation=order, borderMode=cv2.BORDER_REFLECT)
 return tile
confidence_thresh = 0.4
nms_thresh = 0.5
pseudolabelling_thresh = 0.4
with_tta = True
with_yolo = True<find_best_model_class>
class CONFIG() :
data_path = Path('.. /input/hubmap-kidney-segmentation')
 models_path = Path('.. /input/hubmap-efficient-sampling-deepflash2-train')
 models_file = np.array([x for x in models_path.iterdir() if x.name.startswith('u')])
 scale = 3
tile_shape =(512, 512)
 padding =(100,100)
 encoder_name = "efficientnet-b4"
encoder_weights = None
in_channels = 3
classes = 2
batch_size = 16
threshold = 0.5
cfg = CONFIG()
file_name = test_files[9]
image = cv2.imread(file_name)
 models_results = []
if with_yolo:
image_yolo = convert_image_to_yolo(image)
 yolo_predictions = make_tta_predictions(image_yolo, score_threshold=yolo_score_threshold, iou_th=yolo_iou_threshold)
 models_results.append(yolo_predictions)
 for model_i, model in enumerate(models):
if with_tta:
wheat_result_i = detect_tta(model, image)
 else:
res_values = inference_detector(model, image)
 wheat_result_i = res_values[0]
models_results.append(wheat_result_i)
 bboxes_wbf = [model_result[:,:4]/1024 for model_result in models_results]
scores_wbf = [model_result[:,4] for model_result in models_results]
labels_wbf = [np.zeros_like(score_wbf)for score_wbf in scores_wbf]
wheat_bboxes, wheat_scores, _ = weighted_boxes_fusion(bboxes_wbf, scores_wbf, labels_wbf, weights=wbf_weights, iou_thr=nms_thresh, skip_box_thr=confidence_thresh)
 print('Before WBF')
 print(f'scores: {scores_wbf}')
 print('After WBF')
 print(f'scores: {wheat_scores}')
 wheat_bboxes *= 1024
print(f'Before wbf {sum([len(model_result)for model_result in models_results])}')
 print(f'After wbf {len(wheat_bboxes)}')
 num_bboxes = 0
print(f'Before confidence thresholding {len(wheat_bboxes)}')
 for bbox, score in zip(wheat_bboxes, wheat_scores):
if score>=confidence_thresh:
bbx_int = [int(bbx)for bbx in bbox]
cv2.rectangle(image,(bbx_int[0],bbx_int[1]),(bbx_int[2],bbx_int[3]),(255,0,0), 2)
 num_bboxes+=1
print(f'After confidence thresholding {num_bboxes}')
 fig=plt.figure(figsize=(18, 16))
plt.imshow(image)
 plt.show() 
<predict_on_test>
print(cfg.models_file)
 print(len(cfg.models_file))
df_sample = pd.read_csv(cfg.data_path/'sample_submission.csv', index_col='id')
 names,preds = [],[]
sub = None
results, pseudo_labels = predict(models,
test_files,
confidence_thresh=confidence_thresh,
nms_thresh = nms_thresh,
pseudolabelling_thresh = pseudolabelling_thresh,
with_tta=with_tta,
with_wbf=True,
wbf_weights=wbf_weights,
with_yolo=with_yolo )<define_variables>
names,preds = [],[]
for idx, _ in df_sample.iterrows() :
print(f'
f = cfg.data_path/'test'/f'{idx}.tiff'
ds = TileDataset([f], scale=cfg.scale, tile_shape=cfg.tile_shape, padding=cfg.padding)
 shape = ds.data[f.name].shape
print('Shape:', shape)
 names.append(idx)
 msk = None
print('Prediction')
 for model_path in cfg.models_file:
model = smp.Unet(encoder_name=cfg.encoder_name,
encoder_weights=cfg.encoder_weights,
in_channels=cfg.in_channels,
classes=cfg.classes)
 model, stats = load_model_weights(model, model_path)
 batch_tfms = [Normalize.from_stats(*stats)]
dls = DataLoaders.from_dsets(ds, batch_size=cfg.batch_size, after_batch=batch_tfms, shuffle=False, drop_last=False)
 if torch.cuda.is_available() : dls.cuda() , model.cuda()
learn = Learner(dls, model, loss_func='')
 res = learn.predict_tiles(dl=dls.train, path='/kaggle/temp/', use_tta=False, uncertainty_estimates=False)
 if msk is None:
msk = res[0][f.name][..., 1]
else:
msk += res[0][f.name][..., 1]
del model, stats, learn
msk = msk/len(cfg.models_file)
 msk =(msk > cfg.threshold ).astype(np.uint8)
 print('Rezising')
 msk = cv2.resize(msk,(shape[1], shape[0]))
rle = rle_encode_less_memory(msk)
 preds.append(rle)
 print('Plotting')
 fig, ax = plt.subplots(figsize=(15,15))
ax.imshow(cv2.resize(msk,(1024, 1024)))
 plt.show()
_ = [shutil.rmtree(p, ignore_errors=True)for p in Path('/kaggle/temp/' ).iterdir() ]
_ = [shutil.rmtree(p, ignore_errors=True)for p in Path('/tmp/' ).iterdir() if p.name.startswith('zarr')]
<save_to_csv><EOS> 
df = pd.DataFrame({'id':names,'predicted':preds} ).set_index('id')
 df_sample.loc[df.index.values] = df.values
df_sample.to_csv('submission.csv' )