Repository: Project-MONAI/MONAI

Issue:
Box transforms
**Is your feature request related to a problem? Please describe.**
Box pre- and post-processing transform functions are not in MONAI

**Describe the solution you'd like**
Add box transform functions

**Describe alternatives you've considered**
N/A

**Additional context**
1. convert input box with original mode to the box with standard mode
2. convert output box with standard mode to the box with original mode
3. Flip boxes with image as a transform



Discussion:

Resolving patch:
diff --git a/monai/apps/detection/transforms/array.py b/monai/apps/detection/transforms/array.py
--- a/monai/apps/detection/transforms/array.py
+++ b/monai/apps/detection/transforms/array.py
@@ -13,19 +13,36 @@
 https://github.com/Project-MONAI/MONAI/wiki/MONAI_Design
 """
 
-from typing import Optional, Sequence, Type, Union
+from copy import deepcopy
+from typing import Optional, Sequence, Tuple, Type, Union
 
 import numpy as np
+import torch
 
 from monai.config.type_definitions import NdarrayOrTensor
-from monai.data.box_utils import BoxMode, convert_box_mode, convert_box_to_standard_mode, get_spatial_dims
+from monai.data.box_utils import (
+    BoxMode,
+    clip_boxes_to_image,
+    convert_box_mode,
+    convert_box_to_standard_mode,
+    get_spatial_dims,
+)
 from monai.transforms.transform import Transform
 from monai.utils import ensure_tuple, ensure_tuple_rep, fall_back_tuple, look_up_option
 from monai.utils.enums import TransformBackends
+from monai.utils.type_conversion import convert_data_type, convert_to_dst_type
 
 from .box_ops import apply_affine_to_boxes, flip_boxes, resize_boxes, zoom_boxes
 
-__all__ = ["ConvertBoxToStandardMode", "ConvertBoxMode", "AffineBox", "ZoomBox", "ResizeBox", "FlipBox"]
+__all__ = [
+    "ConvertBoxToStandardMode",
+    "ConvertBoxMode",
+    "AffineBox",
+    "ZoomBox",
+    "ResizeBox",
+    "FlipBox",
+    "ClipBoxToImage",
+]
 
 
 class ConvertBoxMode(Transform):
@@ -296,3 +313,63 @@ def __call__(  # type: ignore
         """
 
         return flip_boxes(boxes, spatial_size=spatial_size, flip_axes=self.spatial_axis)
+
+
+class ClipBoxToImage(Transform):
+    """
+    Clip the bounding boxes and the associated labels/scores to makes sure they are within the image.
+    There might be multiple arryas of labels/scores associated with one array of boxes.
+
+    Args:
+        remove_empty: whether to remove the boxes and corresponding labels that are actually empty
+    """
+
+    backend = [TransformBackends.TORCH, TransformBackends.NUMPY]
+
+    def __init__(self, remove_empty: bool = False) -> None:
+        self.remove_empty = remove_empty
+
+    def __call__(  # type: ignore
+        self,
+        boxes: NdarrayOrTensor,
+        labels: Union[Sequence[NdarrayOrTensor], NdarrayOrTensor],
+        spatial_size: Union[Sequence[int], int],
+    ) -> Tuple[NdarrayOrTensor, Tuple]:
+        """
+        Args:
+            boxes: bounding boxes, Nx4 or Nx6 torch tensor or ndarray. The box mode is assumed to be ``StandardMode``
+            labels: Sequence of array. Each element represents classification labels or scores
+                corresponding to ``boxes``, sized (N,).
+            spatial_size: The spatial size of the image where the boxes are attached. len(spatial_size) should be in [2, 3].
+
+        Returns:
+            - clipped boxes, does not share memory with original boxes
+            - clipped labels, does not share memory with original labels
+
+        Example:
+            .. code-block:: python
+
+                box_clipper = ClipBoxToImage(remove_empty=True)
+                boxes = torch.ones(2, 6)
+                class_labels = torch.Tensor([0, 1])
+                pred_scores = torch.Tensor([[0.4,0.3,0.3], [0.5,0.1,0.4]])
+                labels = (class_labels, pred_scores)
+                spatial_size = [32, 32, 32]
+                boxes_clip, labels_clip_tuple = box_clipper(boxes, labels, spatial_size)
+        """
+        spatial_dims: int = get_spatial_dims(boxes=boxes)
+        spatial_size = ensure_tuple_rep(spatial_size, spatial_dims)  # match the spatial image dim
+
+        boxes_clip, keep = clip_boxes_to_image(boxes, spatial_size, self.remove_empty)
+
+        labels_tuple = ensure_tuple(labels)
+        labels_clip_list = []
+
+        keep_t: torch.Tensor = convert_data_type(keep, torch.Tensor)[0]
+        for i in range(len(labels_tuple)):
+            labels_t: torch.Tensor = convert_data_type(labels_tuple[i], torch.Tensor)[0]
+            if boxes.shape[0] != labels_t.shape[0]:
+                raise ValueError("boxes.shape[0] should be equal to the number of labels or scores.")
+            labels_t = deepcopy(labels_t[keep_t, ...])
+            labels_clip_list.append(convert_to_dst_type(src=labels_t, dst=labels_tuple[i])[0])
+        return boxes_clip, tuple(labels_clip_list)
diff --git a/monai/apps/detection/transforms/dictionary.py b/monai/apps/detection/transforms/dictionary.py
--- a/monai/apps/detection/transforms/dictionary.py
+++ b/monai/apps/detection/transforms/dictionary.py
@@ -22,7 +22,14 @@
 import numpy as np
 import torch
 
-from monai.apps.detection.transforms.array import AffineBox, ConvertBoxMode, ConvertBoxToStandardMode, FlipBox, ZoomBox
+from monai.apps.detection.transforms.array import (
+    AffineBox,
+    ClipBoxToImage,
+    ConvertBoxMode,
+    ConvertBoxToStandardMode,
+    FlipBox,
+    ZoomBox,
+)
 from monai.config import KeysCollection
 from monai.config.type_definitions import NdarrayOrTensor
 from monai.data.box_utils import BoxMode
@@ -56,6 +63,9 @@
     "RandFlipBoxd",
     "RandFlipBoxD",
     "RandFlipBoxDict",
+    "ClipBoxToImaged",
+    "ClipBoxToImageD",
+    "ClipBoxToImageDict",
 ]
 
 DEFAULT_POST_FIX = PostFix.meta()
@@ -687,6 +697,67 @@ def inverse(self, data: Mapping[Hashable, NdarrayOrTensor]) -> Dict[Hashable, Nd
         return d
 
 
+class ClipBoxToImaged(MapTransform):
+    """
+    Dictionary-based wrapper of :py:class:`monai.apps.detection.transforms.array.ClipBoxToImage`.
+
+    Clip the bounding boxes and the associated labels/scores to makes sure they are within the image.
+    There might be multiple keys of labels/scores associated with one key of boxes.
+
+    Args:
+        box_keys: The single key to pick box data for transformation. The box mode is assumed to be ``StandardMode``.
+        label_keys: Keys that represents the lables corresponding to the ``box_keys``. Multiple keys are allowed.
+        box_ref_image_keys: The single key that represents the reference image
+            to which ``box_keys`` and ``label_keys`` are attached.
+        remove_empty: whether to remove the boxes that are actually empty
+        allow_missing_keys: don't raise exception if key is missing.
+
+    Example:
+        .. code-block:: python
+
+            ClipBoxToImaged(
+                box_keys="boxes", box_ref_image_keys="image", label_keys=["labels", "scores"], remove_empty=True
+            )
+    """
+
+    def __init__(
+        self,
+        box_keys: KeysCollection,
+        label_keys: KeysCollection,
+        box_ref_image_keys: KeysCollection,
+        remove_empty: bool = True,
+        allow_missing_keys: bool = False,
+    ) -> None:
+        box_keys_tuple = ensure_tuple(box_keys)
+        if len(box_keys_tuple) != 1:
+            raise ValueError(
+                "Please provide a single key for box_keys.\
+                All label_keys are attached to this box_keys."
+            )
+        box_ref_image_keys_tuple = ensure_tuple(box_ref_image_keys)
+        if len(box_ref_image_keys_tuple) != 1:
+            raise ValueError(
+                "Please provide a single key for box_ref_image_keys.\
+                All box_keys and label_keys are attached to this box_ref_image_keys."
+            )
+        self.label_keys = ensure_tuple(label_keys)
+        super().__init__(box_keys_tuple, allow_missing_keys)
+
+        self.box_keys = box_keys_tuple[0]
+        self.box_ref_image_keys = box_ref_image_keys_tuple[0]
+        self.clipper = ClipBoxToImage(remove_empty=remove_empty)
+
+    def __call__(self, data: Mapping[Hashable, NdarrayOrTensor]) -> Dict[Hashable, NdarrayOrTensor]:
+        d = dict(data)
+        spatial_size = d[self.box_ref_image_keys].shape[1:]
+        labels = [d[label_key] for label_key in self.label_keys]  # could be multiple arrays
+        d[self.box_keys], clipped_labels = self.clipper(d[self.box_keys], labels, spatial_size)
+
+        for label_key, clipped_labels_i in zip(self.label_keys, clipped_labels):
+            d[label_key] = clipped_labels_i
+        return d
+
+
 ConvertBoxModeD = ConvertBoxModeDict = ConvertBoxModed
 ConvertBoxToStandardModeD = ConvertBoxToStandardModeDict = ConvertBoxToStandardModed
 ZoomBoxD = ZoomBoxDict = ZoomBoxd
@@ -694,3 +765,4 @@ def inverse(self, data: Mapping[Hashable, NdarrayOrTensor]) -> Dict[Hashable, Nd
 AffineBoxToImageCoordinateD = AffineBoxToImageCoordinateDict = AffineBoxToImageCoordinated
 FlipBoxD = FlipBoxDict = FlipBoxd
 RandFlipBoxD = RandFlipBoxDict = RandFlipBoxd
+ClipBoxToImageD = ClipBoxToImageDict = ClipBoxToImaged


Test patch:
diff --git a/tests/test_box_transform.py b/tests/test_box_transform.py
--- a/tests/test_box_transform.py
+++ b/tests/test_box_transform.py
@@ -17,6 +17,7 @@
 
 from monai.apps.detection.transforms.dictionary import (
     AffineBoxToImageCoordinated,
+    ClipBoxToImaged,
     ConvertBoxModed,
     FlipBoxd,
     RandFlipBoxd,
@@ -29,6 +30,8 @@
 TESTS = []
 
 boxes = [[0, 0, 0, 0, 0, 0], [0, 1, 0, 2, 3, 3], [0, 1, 1, 2, 3, 4]]
+labels = [1, 1, 0]
+scores = [[0.2, 0.8], [0.3, 0.7], [0.6, 0.4]]
 image_size = [1, 4, 6, 4]
 image = np.zeros(image_size)
 
@@ -36,7 +39,7 @@
     TESTS.append(
         [
             {"box_keys": "boxes", "dst_mode": "xyzwhd"},
-            {"boxes": p(boxes), "image": p(image)},
+            {"boxes": p(boxes), "image": p(image), "labels": p(labels), "scores": p(scores)},
             p([[0, 0, 0, 0, 0, 0], [0, 1, 0, 2, 2, 3], [0, 1, 1, 2, 2, 3]]),
             p([[0, 0, 0, 0, 0, 0], [0, 3, 0, 1, 9, 4.5], [0, 3, 1.5, 1, 9, 6]]),
             p([[1, -6, -1, 1, -6, -1], [1, -3, -1, 2, 3, 3.5], [1, -3, 0.5, 2, 3, 5]]),
@@ -156,6 +159,21 @@ def test_value(
                 assert_allclose(data_back["boxes"], data["boxes"], type_test=False, device_test=False, atol=1e-3)
                 assert_allclose(data_back["image"], data["image"], type_test=False, device_test=False, atol=1e-3)
 
+            # test ClipBoxToImaged
+            transform_clip = ClipBoxToImaged(
+                box_keys="boxes", box_ref_image_keys="image", label_keys=["labels", "scores"], remove_empty=True
+            )
+            clip_result = transform_clip(data)
+            assert_allclose(clip_result["boxes"], expected_clip_result, type_test=True, device_test=True, atol=1e-3)
+            assert_allclose(clip_result["labels"], data["labels"][1:], type_test=True, device_test=True, atol=1e-3)
+            assert_allclose(clip_result["scores"], data["scores"][1:], type_test=True, device_test=True, atol=1e-3)
+
+            transform_clip = ClipBoxToImaged(
+                box_keys="boxes", box_ref_image_keys="image", label_keys=[], remove_empty=True
+            )  # corner case when label_keys is empty
+            clip_result = transform_clip(data)
+            assert_allclose(clip_result["boxes"], expected_clip_result, type_test=True, device_test=True, atol=1e-3)
+
 
 if __name__ == "__main__":
     unittest.main()