Repository: open-mmlab/mmdetection

Issue:
Soft Segmentation Masks
Currently in the segmentation heads it appears that masks are always given as binary targets.

For some problems, it may be more natural to instead express the ground truth as either a weighted mask, or more simply a mask of fractional "soft" values.

From what I can tell, this might be as simple as conditionally executing `resized_masks = (targets >= 0.5).cpu().numpy()` in the `crop_and_resize` function of `BitmapMasks` (in mmdet/core/mask/structures.py) based on the `cfg` passed to `mask_target` (in mmdet/core/mask/mask_target.py).



Discussion:

Resolving patch:
diff --git a/mmdet/core/mask/mask_target.py b/mmdet/core/mask/mask_target.py
--- a/mmdet/core/mask/mask_target.py
+++ b/mmdet/core/mask/mask_target.py
@@ -43,6 +43,7 @@ def mask_target_single(pos_proposals, pos_assigned_gt_inds, gt_masks, cfg):
     """
     device = pos_proposals.device
     mask_size = _pair(cfg.mask_size)
+    binarize = not cfg.get('soft_mask_target', False)
     num_pos = pos_proposals.size(0)
     if num_pos > 0:
         proposals_np = pos_proposals.cpu().numpy()
@@ -52,8 +53,11 @@ def mask_target_single(pos_proposals, pos_assigned_gt_inds, gt_masks, cfg):
         pos_assigned_gt_inds = pos_assigned_gt_inds.cpu().numpy()
 
         mask_targets = gt_masks.crop_and_resize(
-            proposals_np, mask_size, device=device,
-            inds=pos_assigned_gt_inds).to_ndarray()
+            proposals_np,
+            mask_size,
+            device=device,
+            inds=pos_assigned_gt_inds,
+            binarize=binarize).to_ndarray()
 
         mask_targets = torch.from_numpy(mask_targets).float().to(device)
     else:
diff --git a/mmdet/core/mask/structures.py b/mmdet/core/mask/structures.py
--- a/mmdet/core/mask/structures.py
+++ b/mmdet/core/mask/structures.py
@@ -81,7 +81,8 @@ def crop_and_resize(self,
                         out_shape,
                         inds,
                         device,
-                        interpolation='bilinear'):
+                        interpolation='bilinear',
+                        binarize=True):
         """Crop and resize masks by the given bboxes.
 
         This function is mainly used in mask targets computation.
@@ -94,6 +95,9 @@ def crop_and_resize(self,
             inds (ndarray): Indexes to assign masks to each bbox
             device (str): Device of bboxes
             interpolation (str): See `mmcv.imresize`
+            binarize (bool): if True fractional values are rounded to 0 or 1
+                after the resize operation. if False and unsupported an error
+                will be raised. Defaults to True.
 
         Return:
             BaseInstanceMasks: the cropped and resized masks.
@@ -323,7 +327,8 @@ def crop_and_resize(self,
                         out_shape,
                         inds,
                         device='cpu',
-                        interpolation='bilinear'):
+                        interpolation='bilinear',
+                        binarize=True):
         """See :func:`BaseInstanceMasks.crop_and_resize`."""
         if len(self.masks) == 0:
             empty_masks = np.empty((0, *out_shape), dtype=np.uint8)
@@ -345,7 +350,10 @@ def crop_and_resize(self,
                 0, inds).to(dtype=rois.dtype)
             targets = roi_align(gt_masks_th[:, None, :, :], rois, out_shape,
                                 1.0, 0, 'avg', True).squeeze(1)
-            resized_masks = (targets >= 0.5).cpu().numpy()
+            if binarize:
+                resized_masks = (targets >= 0.5).cpu().numpy()
+            else:
+                resized_masks = targets.cpu().numpy()
         else:
             resized_masks = []
         return BitmapMasks(resized_masks, *out_shape)
@@ -633,12 +641,17 @@ def crop_and_resize(self,
                         out_shape,
                         inds,
                         device='cpu',
-                        interpolation='bilinear'):
+                        interpolation='bilinear',
+                        binarize=True):
         """see :func:`BaseInstanceMasks.crop_and_resize`"""
         out_h, out_w = out_shape
         if len(self.masks) == 0:
             return PolygonMasks([], out_h, out_w)
 
+        if not binarize:
+            raise ValueError('Polygons are always binary, '
+                             'setting binarize=False is unsupported')
+
         resized_masks = []
         for i in range(len(bboxes)):
             mask = self.masks[inds[i]]


Test patch: