Kaggle competition: HuBMAP - Hacking the Kidney
<SOS> metric: Dice Kaggle data source: hubmap-hacking-the-kidney<prepare_output>
warnings.filterwarnings("ignore" )
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.head()<save_to_csv>
Threshold = 35
test_df.to_csv('/kaggle/working/submission.csv', index=False )<install_modules>
def rle_encode_less_memory(img):
pixels = img.T.flatten()
pixels[0] = 0
pixels[-1] = 0
runs = np.where(pixels[1:] != pixels[:-1])[0] + 2
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
!pip install --no-deps '.. /input/wheatdep/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/wheatdep/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null<define_variables>
df_sample = pd.read_csv('.. /input/hubmap-kidney-segmentation/sample_submission.csv' )
NMS_THRESHOLD = 0.94<define_variables>
names,preds = [],[]
MIN_CLASS_SCORE = -5.0
_DUMMY_DETECTION_SCORE = -1e5
MAX_DETECTION_POINTS = 5000
MAX_DETECTIONS_PER_IMAGE = 100
def decode_box_outputs(rel_codes, anchors, output_xyxy=False):

 ycenter_a =(anchors[0] + anchors[2])/ 2
xcenter_a =(anchors[1] + anchors[3])/ 2
ha = anchors[2] - anchors[0]
wa = anchors[3] - anchors[1]
ty, tx, th, tw = rel_codes
w = torch.exp(tw)* wa
h = torch.exp(th)* ha
ycenter = ty * ha + ycenter_a
xcenter = tx * wa + xcenter_a
ymin = ycenter - h / 2.
 xmin = xcenter - w / 2.
 ymax = ycenter + h / 2.
 xmax = xcenter + w / 2.
 if output_xyxy:
out = torch.stack([xmin, ymin, xmax, ymax], dim=1)
 else:
out = torch.stack([ymin, xmin, ymax, xmax], dim=1)
 return out
def _generate_anchor_configs(min_level, max_level, num_scales, aspect_ratios):

 anchor_configs = {}
for level in range(min_level, max_level + 1):
anchor_configs[level] = []
for scale_octave in range(num_scales):
for aspect in aspect_ratios:
anchor_configs[level].append(( 2 ** level, scale_octave / float(num_scales), aspect))
return anchor_configs
def _generate_anchor_boxes(image_size, anchor_scale, anchor_configs):

 boxes_all = []
for _, configs in anchor_configs.items() :
boxes_level = []
for config in configs:
stride, octave_scale, aspect = config
if image_size % stride != 0:
raise ValueError("input size must be divided by the stride.")
 base_anchor_size = anchor_scale * stride * 2 ** octave_scale
anchor_size_x_2 = base_anchor_size * aspect[0] / 2.0
anchor_size_y_2 = base_anchor_size * aspect[1] / 2.0
x = np.arange(stride / 2, image_size, stride)
 y = np.arange(stride / 2, image_size, stride)
 xv, yv = np.meshgrid(x, y)
 xv = xv.reshape(-1)
 yv = yv.reshape(-1)
 boxes = np.vstack(( yv - anchor_size_y_2, xv - anchor_size_x_2,
yv + anchor_size_y_2, xv + anchor_size_x_2))
boxes = np.swapaxes(boxes, 0, 1)
 boxes_level.append(np.expand_dims(boxes, axis=1))
boxes_level = np.concatenate(boxes_level, axis=1)
 boxes_all.append(boxes_level.reshape([-1, 4]))
anchor_boxes = np.vstack(boxes_all)
 return anchor_boxes
def generate_detections(cls_outputs, box_outputs, anchor_boxes, indices, classes, image_scale):

 anchor_boxes = anchor_boxes[indices, :]
boxes = decode_box_outputs(box_outputs.T.float() , anchor_boxes.T, output_xyxy=True)
 scores = cls_outputs.sigmoid().squeeze(1 ).float()
top_detection_idx = batched_nms(boxes, scores, classes, iou_threshold=NMS_THRESHOLD)
 top_detection_idx = top_detection_idx[:MAX_DETECTIONS_PER_IMAGE]
boxes = boxes[top_detection_idx]
scores = scores[top_detection_idx, None]
classes = classes[top_detection_idx, None]
boxes[:, 2] -= boxes[:, 0]
boxes[:, 3] -= boxes[:, 1]
boxes *= image_scale
classes += 1
detections = torch.cat([boxes, scores, classes.float() ], dim=1)
 if len(top_detection_idx)< MAX_DETECTIONS_PER_IMAGE:
detections = torch.cat([
detections,
torch.zeros(
(MAX_DETECTIONS_PER_IMAGE - len(top_detection_idx), 6), device=detections.device, dtype=detections.dtype)
 ], dim=0)
 return detections
class Anchors(nn.Module):

 def __init__(self, min_level, max_level, num_scales, aspect_ratios, anchor_scale, image_size):

 super(Anchors, self ).__init__()
self.min_level = min_level
self.max_level = max_level
self.num_scales = num_scales
self.aspect_ratios = aspect_ratios
self.anchor_scale = anchor_scale
self.image_size = image_size
self.config = self._generate_configs()
self.register_buffer('boxes', self._generate_boxes())
 def _generate_configs(self):

 return _generate_anchor_configs(self.min_level, self.max_level, self.num_scales, self.aspect_ratios)
 def _generate_boxes(self):

 boxes = _generate_anchor_boxes(self.image_size, self.anchor_scale, self.config)
 boxes = torch.from_numpy(boxes ).float()
return boxes
def get_anchors_per_location(self):
return self.num_scales * len(self.aspect_ratios)
 class AnchorLabeler(nn.Module):

 def __init__(self, anchors, num_classes, match_threshold=0.5):

 super(AnchorLabeler, self ).__init__()
similarity_calc = region_similarity_calculator.IouSimilarity()
matcher = argmax_matcher.ArgMaxMatcher(
 match_threshold,
unmatched_threshold=match_threshold,
negatives_lower_than_unmatched=True,
force_match_for_each_row=True)
 box_coder = faster_rcnn_box_coder.FasterRcnnBoxCoder()
self.target_assigner = target_assigner.TargetAssigner(similarity_calc, matcher, box_coder)
 self.anchors = anchors
self.match_threshold = match_threshold
self.num_classes = num_classes
def _unpack_labels(self, labels):

 labels_unpacked = []
anchors = self.anchors
count = 0
for level in range(anchors.min_level, anchors.max_level + 1):
feat_size = int(anchors.image_size / 2 ** level)
 steps = feat_size ** 2 * anchors.get_anchors_per_location()
indices = torch.arange(count, count + steps, device=labels.device)
 count += steps
labels_unpacked.append(
 torch.index_select(labels, 0, indices ).view([feat_size, feat_size, -1]))
return labels_unpacked
def label_anchors(self, gt_boxes, gt_labels):

 gt_box_list = box_list.BoxList(gt_boxes)
 anchor_box_list = box_list.BoxList(self.anchors.boxes)
 cls_targets, _, box_targets, _, matches = self.target_assigner.assign(anchor_box_list, gt_box_list, gt_labels)
 cls_targets -= 1
cls_targets = cls_targets.long()
cls_targets_dict = self._unpack_labels(cls_targets)
 box_targets_dict = self._unpack_labels(box_targets)
 num_positives =(matches.match_results != -1 ).float().sum()
return cls_targets_dict, box_targets_dict, num_positives

 def _post_process(config, cls_outputs, box_outputs):

 batch_size = cls_outputs[0].shape[0]
cls_outputs_all = torch.cat([
cls_outputs[level].permute(0, 2, 3, 1 ).reshape([batch_size, -1, config.num_classes])
 for level in range(config.num_levels)], 1)
 box_outputs_all = torch.cat([
box_outputs[level].permute(0, 2, 3, 1 ).reshape([batch_size, -1, 4])
 for level in range(config.num_levels)], 1)
 _, cls_topk_indices_all = torch.topk(cls_outputs_all.reshape(batch_size, -1), dim=1, k=MAX_DETECTION_POINTS)
 indices_all = cls_topk_indices_all / config.num_classes
classes_all = cls_topk_indices_all % config.num_classes
box_outputs_all_after_topk = torch.gather(
 box_outputs_all, 1, indices_all.unsqueeze(2 ).expand(-1, -1, 4))
cls_outputs_all_after_topk = torch.gather(
 cls_outputs_all, 1, indices_all.unsqueeze(2 ).expand(-1, -1, config.num_classes))
cls_outputs_all_after_topk = torch.gather(
 cls_outputs_all_after_topk, 2, classes_all.unsqueeze(2))
return cls_outputs_all_after_topk, box_outputs_all_after_topk, indices_all, classes_all
class DetBenchEval(nn.Module):
def __init__(self, model, config):
super(DetBenchEval, self ).__init__()
self.config = config
self.model = model
self.anchors = Anchors(
 config.min_level, config.max_level,
config.num_scales, config.aspect_ratios,
config.anchor_scale, config.image_size)
 def forward(self, x, image_scales):
class_out, box_out = self.model(x)
 class_out, box_out, indices, classes = _post_process(self.config, class_out, box_out)
 batch_detections = []
for i in range(x.shape[0]):
detections = generate_detections(
 class_out[i], box_out[i], self.anchors.boxes, indices[i], classes[i], image_scales[i])
 batch_detections.append(detections)
 return torch.stack(batch_detections, dim=0)
 class DetBenchTrain(nn.Module):
def __init__(self, model, config):
super(DetBenchTrain, self ).__init__()
self.config = config
self.model = model
anchors = Anchors(
 config.min_level, config.max_level,
config.num_scales, config.aspect_ratios,
config.anchor_scale, config.image_size)
 self.anchor_labeler = AnchorLabeler(anchors, config.num_classes, match_threshold=0.5)
 self.loss_fn = DetectionLoss(self.config)
 def forward(self, x, gt_boxes, gt_labels):
class_out, box_out = self.model(x)
 cls_targets = []
box_targets = []
num_positives = []
for i in range(x.shape[0]):
gt_class_out, gt_box_out, num_positive = self.anchor_labeler.label_anchors(gt_boxes[i], gt_labels[i])
 cls_targets.append(gt_class_out)
 box_targets.append(gt_box_out)
 num_positives.append(num_positive)
 return self.loss_fn(class_out, box_out, cls_targets, box_targets, num_positives)
<define_variables>
for idx,row in tqdm(df_sample.iterrows() ,total=len(df_sample)) :
idx = row['id']
pred1 = np.load(f"./pred_{idx}_reduce2.npz")['arr_0'].astype(np.uint8)
 pred2 = np.load(f"./pred_{idx}_reduce4.npz")['arr_0'].astype(np.uint8)
 mask =(pred1 + pred2)> 2 * Threshold
rle = rle_encode_less_memory(mask)
 names.append(idx)
 preds.append(rle)
 del mask, pred1, pred2, rle
gc.collect()
<split><EOS> 
df = pd.DataFrame({'id':names,'predicted':preds})
 df.to_csv('submission.csv',index=False )