Repository: Project-MONAI/MONAI

Issue:
Box evaluation metrics
**Is your feature request related to a problem? Please describe.**
No box evaluator

**Describe the solution you'd like**
Add box evaluation metrics and evaluator

**Describe alternatives you've considered**
N/A

**Additional context**
N/A



Discussion:

Resolving patch:
diff --git a/monai/apps/detection/metrics/__init__.py b/monai/apps/detection/metrics/__init__.py
new file mode 100644
--- /dev/null
+++ b/monai/apps/detection/metrics/__init__.py
@@ -0,0 +1,10 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/monai/apps/detection/metrics/coco.py b/monai/apps/detection/metrics/coco.py
new file mode 100644
--- /dev/null
+++ b/monai/apps/detection/metrics/coco.py
@@ -0,0 +1,547 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# =========================================================================
+# Adapted from https://github.com/MIC-DKFZ/nnDetection/blob/main/nndet/evaluator/detection/coco.py
+# which has the following license...
+# https://github.com/MIC-DKFZ/nnDetection/blob/main/LICENSE
+#
+# Copyright 2020 Division of Medical Image Computing, German Cancer Research Center (DKFZ), Heidelberg, Germany
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#    http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# =========================================================================
+# Adapted from https://github.com/cocodataset/cocoapi
+# which has the following license...
+# https://github.com/cocodataset/cocoapi/blob/master/license.txt
+
+# Copyright (c) 2014, Piotr Dollar and Tsung-Yi Lin
+# All rights reserved.
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+
+# 1. Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# The views and conclusions contained in the software and documentation are those
+# of the authors and should not be interpreted as representing official policies,
+# either expressed or implied, of the FreeBSD Project.
+
+"""
+This script is almost same with https://github.com/MIC-DKFZ/nnDetection/blob/main/nndet/evaluator/detection/coco.py
+The changes include 1) code reformatting, 2) docstrings.
+"""
+
+import logging as logger
+import time
+from typing import Dict, List, Sequence, Tuple, Union
+
+import numpy as np
+
+
+class COCOMetric:
+    def __init__(
+        self,
+        classes: Sequence[str],
+        iou_list: Sequence[float] = (0.1, 0.5, 0.75),
+        iou_range: Sequence[float] = (0.1, 0.5, 0.05),
+        max_detection: Sequence[int] = (1, 5, 100),
+        per_class: bool = True,
+        verbose: bool = True,
+    ):
+        """
+        Class to compute COCO metrics
+        Metrics computed includes,
+
+        - mAP over the IoU range specified by `iou_range` at last value of `max_detection`
+        - AP values at IoU thresholds specified by `iou_list` at last value of `max_detection`
+        - AR over max detections thresholds defined by `max_detection` (over iou range)
+
+        Args:
+            classes (Sequence[str]): name of each class (index needs to correspond to predicted class indices!)
+            iou_list (Sequence[float]): specific thresholds where ap is evaluated and saved
+            iou_range (Sequence[float]): (start, stop, step) for mAP iou thresholds
+            max_detection (Sequence[int]): maximum number of detections per image
+            verbose (bool): log time needed for evaluation
+
+        Example:
+
+            .. code-block:: python
+
+                from monai.data.box_utils import box_iou
+                from monai.apps.detection.metrics.coco import COCOMetric
+                from monai.apps.detection.metrics.matching import matching_batch
+                # 3D example outputs of one image from detector
+                val_outputs_all = [
+                        {"boxes": torch.tensor([[1,1,1,3,4,5]],dtype=torch.float16),
+                        "labels": torch.randint(3,(1,)),
+                        "scores": torch.randn((1,)).absolute()},
+                ]
+                val_targets_all = [
+                        {"boxes": torch.tensor([[1,1,1,2,6,4]],dtype=torch.float16),
+                        "labels": torch.randint(3,(1,))},
+                ]
+
+                coco_metric = COCOMetric(
+                    classes=['c0','c1','c2'], iou_list=[0.1], max_detection=[10]
+                )
+                results_metric = matching_batch(
+                    iou_fn=box_iou,
+                    iou_thresholds=coco_metric.iou_thresholds,
+                    pred_boxes=[val_data_i["boxes"].numpy() for val_data_i in val_outputs_all],
+                    pred_classes=[val_data_i["labels"].numpy() for val_data_i in val_outputs_all],
+                    pred_scores=[val_data_i["scores"].numpy() for val_data_i in val_outputs_all],
+                    gt_boxes=[val_data_i["boxes"].numpy() for val_data_i in val_targets_all],
+                    gt_classes=[val_data_i["labels"].numpy() for val_data_i in val_targets_all],
+                )
+                val_metric_dict = coco_metric(results_metric)
+                print(val_metric_dict)
+        """
+        self.verbose = verbose
+        self.classes = classes
+        self.per_class = per_class
+
+        iou_list_np = np.array(iou_list)
+        _iou_range = np.linspace(
+            iou_range[0], iou_range[1], int(np.round((iou_range[1] - iou_range[0]) / iou_range[2])) + 1, endpoint=True
+        )
+        self.iou_thresholds = np.union1d(iou_list_np, _iou_range)
+        self.iou_range = iou_range
+
+        # get indices of iou values of ious range and ious list for later evaluation
+        self.iou_list_idx = np.nonzero(iou_list_np[:, np.newaxis] == self.iou_thresholds[np.newaxis])[1]
+        self.iou_range_idx = np.nonzero(_iou_range[:, np.newaxis] == self.iou_thresholds[np.newaxis])[1]
+
+        if (
+            not (self.iou_thresholds[self.iou_list_idx] == iou_list_np).all()
+            or not (self.iou_thresholds[self.iou_range_idx] == _iou_range).all()
+        ):
+            raise ValueError(
+                "Require self.iou_thresholds[self.iou_list_idx] == iou_list_np and "
+                "self.iou_thresholds[self.iou_range_idx] == _iou_range."
+            )
+
+        self.recall_thresholds = np.linspace(0.0, 1.00, int(np.round((1.00 - 0.0) / 0.01)) + 1, endpoint=True)
+        self.max_detections = max_detection
+
+    def __call__(self, *args, **kwargs) -> Tuple[Dict[str, float], Union[Dict[str, np.ndarray], None]]:
+        """
+        Compute metric. See :func:`compute` for more information.
+
+        Args:
+            *args: positional arguments passed to :func:`compute`
+            **kwargs: keyword arguments passed to :func:`compute`
+
+        Returns:
+            Dict[str, float]: dictionary with scalar values for evaluation
+            Dict[str, np.ndarray]: dictionary with arrays, e.g. for visualization of graphs
+        """
+        return self.compute(*args, **kwargs)
+
+    def check_number_of_iou(self, *args) -> None:
+        """
+        Check if shape of input in first dimension is consistent with expected IoU values
+        (assumes IoU dimension is the first dimension)
+
+        Args:
+            args: array like inputs with shape function
+        """
+        num_ious = len(self.get_iou_thresholds())
+        for arg in args:
+            if arg.shape[0] != num_ious:
+                raise ValueError(
+                    f"Require arg.shape[0] == len(self.get_iou_thresholds()). Got arg.shape[0]={arg.shape[0]}, "
+                    f"self.get_iou_thresholds()={self.get_iou_thresholds()}."
+                )
+
+    def get_iou_thresholds(self) -> Sequence[float]:
+        """
+        Return IoU thresholds needed for this metric in an numpy array
+
+        Returns:
+            Sequence[float]: IoU thresholds [M], M is the number of thresholds
+        """
+        return list(self.iou_thresholds)
+
+    def compute(self, results_list: List[Dict[int, Dict[str, np.ndarray]]]) -> Tuple[Dict[str, float], None]:
+        """
+        Compute COCO metrics
+
+        Args:
+            results_list (List[Dict[int, Dict[str, np.ndarray]]]): list with results per image (in list)
+                per category (dict). Inner Dict contains multiple results obtained by :func:`box_matching_batch`.
+
+                - `dtMatches`: matched detections [T, D], where T = number of
+                  thresholds, D = number of detections
+                - `gtMatches`: matched ground truth boxes [T, G], where T = number
+                  of thresholds, G = number of ground truth
+                - `dtScores`: prediction scores [D] detection scores
+                - `gtIgnore`: ground truth boxes which should be ignored
+                  [G] indicate whether ground truth should be ignored
+                - `dtIgnore`: detections which should be ignored [T, D],
+                  indicate which detections should be ignored
+
+        Returns:
+            Dict[str, float], dictionary with coco metrics
+        """
+        if self.verbose:
+            logger.info("Start COCO metric computation...")
+            tic = time.time()
+
+        dataset_statistics = self._compute_statistics(results_list=results_list)  # Dict[str, Union[np.ndarray, List]]
+
+        if self.verbose:
+            toc = time.time()
+            logger.info(f"Statistics for COCO metrics finished (t={(toc - tic):0.2f}s).")
+
+        results = {}
+        results.update(self._compute_ap(dataset_statistics))
+        results.update(self._compute_ar(dataset_statistics))
+
+        if self.verbose:
+            toc = time.time()
+            logger.info(f"COCO metrics computed in t={(toc - tic):0.2f}s.")
+        return results, None
+
+    def _compute_ap(self, dataset_statistics: Dict[str, Union[np.ndarray, List]]) -> Dict[str, float]:
+        """
+        Compute AP metrics
+
+        Args:
+            dataset_statistics (List[Dict[int, Dict[str, np.ndarray]]]): list with result s per image (in list)
+                per category (dict). Inner Dict contains multiple results obtained by :func:`box_matching_batch`.
+
+                - `dtMatches`: matched detections [T, D], where T = number of
+                  thresholds, D = number of detections
+                - `gtMatches`: matched ground truth boxes [T, G], where T = number
+                  of thresholds, G = number of ground truth
+                - `dtScores`: prediction scores [D] detection scores
+                - `gtIgnore`: ground truth boxes which should be ignored
+                  [G] indicate whether ground truth should be ignored
+                - `dtIgnore`: detections which should be ignored [T, D],
+                  indicate which detections should be ignored
+        """
+        results = {}
+        if self.iou_range:  # mAP
+            key = (
+                f"mAP_IoU_{self.iou_range[0]:.2f}_{self.iou_range[1]:.2f}_{self.iou_range[2]:.2f}_"
+                f"MaxDet_{self.max_detections[-1]}"
+            )
+            results[key] = self._select_ap(dataset_statistics, iou_idx=self.iou_range_idx, max_det_idx=-1)
+
+            if self.per_class:
+                for cls_idx, cls_str in enumerate(self.classes):  # per class results
+                    key = (
+                        f"{cls_str}_"
+                        f"mAP_IoU_{self.iou_range[0]:.2f}_{self.iou_range[1]:.2f}_{self.iou_range[2]:.2f}_"
+                        f"MaxDet_{self.max_detections[-1]}"
+                    )
+                    results[key] = self._select_ap(
+                        dataset_statistics, iou_idx=self.iou_range_idx, cls_idx=cls_idx, max_det_idx=-1
+                    )
+
+        for idx in self.iou_list_idx:  # AP@IoU
+            key = f"AP_IoU_{self.iou_thresholds[idx]:.2f}_MaxDet_{self.max_detections[-1]}"
+            results[key] = self._select_ap(dataset_statistics, iou_idx=[idx], max_det_idx=-1)
+
+            if self.per_class:
+                for cls_idx, cls_str in enumerate(self.classes):  # per class results
+                    key = f"{cls_str}_" f"AP_IoU_{self.iou_thresholds[idx]:.2f}_" f"MaxDet_{self.max_detections[-1]}"
+                    results[key] = self._select_ap(dataset_statistics, iou_idx=[idx], cls_idx=cls_idx, max_det_idx=-1)
+        return results
+
+    def _compute_ar(self, dataset_statistics: Dict[str, Union[np.ndarray, List]]) -> Dict[str, float]:
+        """
+        Compute AR metrics
+
+        Args:
+            dataset_statistics (List[Dict[int, Dict[str, np.ndarray]]]): list with result s per image (in list)
+                per category (dict). Inner Dict contains multiple results obtained by :func:`box_matching_batch`.
+
+                - `dtMatches`: matched detections [T, D], where T = number of
+                  thresholds, D = number of detections
+                - `gtMatches`: matched ground truth boxes [T, G], where T = number
+                  of thresholds, G = number of ground truth
+                - `dtScores`: prediction scores [D] detection scores
+                - `gtIgnore`: ground truth boxes which should be ignored
+                  [G] indicate whether ground truth should be ignored
+                - `dtIgnore`: detections which should be ignored [T, D],
+                  indicate which detections should be ignored
+        """
+        results = {}
+        for max_det_idx, max_det in enumerate(self.max_detections):  # mAR
+            key = f"mAR_IoU_{self.iou_range[0]:.2f}_{self.iou_range[1]:.2f}_{self.iou_range[2]:.2f}_MaxDet_{max_det}"
+            results[key] = self._select_ar(dataset_statistics, max_det_idx=max_det_idx)
+
+            if self.per_class:
+                for cls_idx, cls_str in enumerate(self.classes):  # per class results
+                    key = (
+                        f"{cls_str}_"
+                        f"mAR_IoU_{self.iou_range[0]:.2f}_{self.iou_range[1]:.2f}_{self.iou_range[2]:.2f}_"
+                        f"MaxDet_{max_det}"
+                    )
+                    results[key] = self._select_ar(dataset_statistics, cls_idx=cls_idx, max_det_idx=max_det_idx)
+
+        for idx in self.iou_list_idx:  # AR@IoU
+            key = f"AR_IoU_{self.iou_thresholds[idx]:.2f}_MaxDet_{self.max_detections[-1]}"
+            results[key] = self._select_ar(dataset_statistics, iou_idx=idx, max_det_idx=-1)
+
+            if self.per_class:
+                for cls_idx, cls_str in enumerate(self.classes):  # per class results
+                    key = f"{cls_str}_" f"AR_IoU_{self.iou_thresholds[idx]:.2f}_" f"MaxDet_{self.max_detections[-1]}"
+                    results[key] = self._select_ar(dataset_statistics, iou_idx=idx, cls_idx=cls_idx, max_det_idx=-1)
+        return results
+
+    @staticmethod
+    def _select_ap(
+        dataset_statistics: dict,
+        iou_idx: Union[int, List[int], np.ndarray, None] = None,
+        cls_idx: Union[int, Sequence[int], None] = None,
+        max_det_idx: int = -1,
+    ) -> float:
+        """
+        Compute average precision
+
+        Args:
+            dataset_statistics (dict): computed statistics over dataset
+
+                - `counts`: Number of thresholds, Number recall thresholds, Number of classes, Number of max
+                  detection thresholds
+                - `recall`: Computed recall values [num_iou_th, num_classes, num_max_detections]
+                - `precision`: Precision values at specified recall thresholds
+                  [num_iou_th, num_recall_th, num_classes, num_max_detections]
+                - `scores`: Scores corresponding to specified recall thresholds
+                  [num_iou_th, num_recall_th, num_classes, num_max_detections]
+            iou_idx: index of IoU values to select for evaluation(if None, all values are used)
+            cls_idx: class indices to select, if None all classes will be selected
+            max_det_idx (int): index to select max detection threshold from data
+
+        Returns:
+            np.ndarray: AP value
+        """
+        prec = dataset_statistics["precision"]
+        if iou_idx is not None:
+            prec = prec[iou_idx]
+        if cls_idx is not None:
+            prec = prec[..., cls_idx, :]
+        prec = prec[..., max_det_idx]
+        return float(np.mean(prec))
+
+    @staticmethod
+    def _select_ar(
+        dataset_statistics: dict,
+        iou_idx: Union[int, Sequence[int], None] = None,
+        cls_idx: Union[int, Sequence[int], None] = None,
+        max_det_idx: int = -1,
+    ) -> float:
+        """
+        Compute average recall
+
+        Args:
+            dataset_statistics (dict): computed statistics over dataset
+
+                - `counts`: Number of thresholds, Number recall thresholds, Number of classes, Number of max
+                  detection thresholds
+                - `recall`: Computed recall values [num_iou_th, num_classes, num_max_detections]
+                - `precision`: Precision values at specified recall thresholds
+                  [num_iou_th, num_recall_th, num_classes, num_max_detections]
+                - `scores`: Scores corresponding to specified recall thresholds
+                  [num_iou_th, num_recall_th, num_classes, num_max_detections]
+            iou_idx: index of IoU values to select for evaluation(if None, all values are used)
+            cls_idx: class indices to select, if None all classes will be selected
+            max_det_idx (int): index to select max detection threshold from data
+
+        Returns:
+            np.ndarray: recall value
+        """
+        rec = dataset_statistics["recall"]
+        if iou_idx is not None:
+            rec = rec[iou_idx]
+        if cls_idx is not None:
+            rec = rec[..., cls_idx, :]
+        rec = rec[..., max_det_idx]
+
+        if len(rec[rec > -1]) == 0:
+            return -1.0
+
+        return float(np.mean(rec[rec > -1]))
+
+    def _compute_statistics(
+        self, results_list: List[Dict[int, Dict[str, np.ndarray]]]
+    ) -> Dict[str, Union[np.ndarray, List]]:
+        """
+        Compute statistics needed for COCO metrics (mAP, AP of individual classes, mAP@IoU_Thresholds, AR)
+        Adapted from https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocotools/cocoeval.py
+
+        Args:
+            results_list (List[Dict[int, Dict[str, np.ndarray]]]): list with result s per image (in list)
+                per cateory (dict). Inner Dict contains multiple results obtained by :func:`box_matching_batch`.
+
+                - `dtMatches`: matched detections [T, D], where T = number of
+                  thresholds, D = number of detections
+                - `gtMatches`: matched ground truth boxes [T, G], where T = number
+                  of thresholds, G = number of ground truth
+                - `dtScores`: prediction scores [D] detection scores
+                - `gtIgnore`: ground truth boxes which should be ignored
+                  [G] indicate whether ground truth should be ignored
+                - `dtIgnore`: detections which should be ignored [T, D],
+                  indicate which detections should be ignored
+
+        Returns:
+            dict: computed statistics over dataset
+                - `counts`: Number of thresholds, Number recall thresholds, Number of classes, Number of max
+                  detection thresholds
+                - `recall`: Computed recall values [num_iou_th, num_classes, num_max_detections]
+                - `precision`: Precision values at specified recall thresholds
+                  [num_iou_th, num_recall_th, num_classes, num_max_detections]
+                - `scores`: Scores corresponding to specified recall thresholds
+                  [num_iou_th, num_recall_th, num_classes, num_max_detections]
+        """
+        num_iou_th = len(self.iou_thresholds)
+        num_recall_th = len(self.recall_thresholds)
+        num_classes = len(self.classes)
+        num_max_detections = len(self.max_detections)
+
+        # -1 for the precision of absent categories
+        precision = -np.ones((num_iou_th, num_recall_th, num_classes, num_max_detections))
+        recall = -np.ones((num_iou_th, num_classes, num_max_detections))
+        scores = -np.ones((num_iou_th, num_recall_th, num_classes, num_max_detections))
+
+        for cls_idx, cls_i in enumerate(self.classes):  # for each class
+            for max_det_idx, max_det in enumerate(self.max_detections):  # for each maximum number of detections
+                results = [r[cls_idx] for r in results_list if cls_idx in r]  # len is num_images
+
+                if len(results) == 0:
+                    logger.warning(f"WARNING, no results found for coco metric for class {cls_i}")
+                    continue
+
+                dt_scores = np.concatenate([r["dtScores"][0:max_det] for r in results])
+                # different sorting method generates slightly different results.
+                # mergesort is used to be consistent as Matlab implementation.
+                inds = np.argsort(-dt_scores, kind="mergesort")
+                dt_scores_sorted = dt_scores[inds]
+
+                # r['dtMatches'] [T, R], where R = sum(all detections)
+                dt_matches = np.concatenate([r["dtMatches"][:, 0:max_det] for r in results], axis=1)[:, inds]
+                dt_ignores = np.concatenate([r["dtIgnore"][:, 0:max_det] for r in results], axis=1)[:, inds]
+                self.check_number_of_iou(dt_matches, dt_ignores)
+                gt_ignore = np.concatenate([r["gtIgnore"] for r in results])
+                num_gt = np.count_nonzero(gt_ignore == 0)  # number of ground truth boxes (non ignored)
+                if num_gt == 0:
+                    logger.warning(f"WARNING, no gt found for coco metric for class {cls_i}")
+                    continue
+
+                # ignore cases need to be handled differently for tp and fp
+                tps = np.logical_and(dt_matches, np.logical_not(dt_ignores))
+                fps = np.logical_and(np.logical_not(dt_matches), np.logical_not(dt_ignores))
+
+                tp_sum = np.cumsum(tps, axis=1).astype(dtype=np.float32)
+                fp_sum = np.cumsum(fps, axis=1).astype(dtype=np.float32)
+
+                for th_ind, (tp, fp) in enumerate(zip(tp_sum, fp_sum)):  # for each threshold th_ind
+                    tp, fp = np.array(tp), np.array(fp)
+                    r, p, s = _compute_stats_single_threshold(tp, fp, dt_scores_sorted, self.recall_thresholds, num_gt)
+                    recall[th_ind, cls_idx, max_det_idx] = r
+                    precision[th_ind, :, cls_idx, max_det_idx] = p
+                    # corresponding score thresholds for recall steps
+                    scores[th_ind, :, cls_idx, max_det_idx] = s
+
+        return {
+            "counts": [num_iou_th, num_recall_th, num_classes, num_max_detections],  # [4]
+            "recall": recall,  # [num_iou_th, num_classes, num_max_detections]
+            "precision": precision,  # [num_iou_th, num_recall_th, num_classes, num_max_detections]
+            "scores": scores,  # [num_iou_th, num_recall_th, num_classes, num_max_detections]
+        }
+
+
+def _compute_stats_single_threshold(
+    tp: np.ndarray,
+    fp: np.ndarray,
+    dt_scores_sorted: np.ndarray,
+    recall_thresholds: Union[np.ndarray, Sequence[float]],
+    num_gt: int,
+) -> Tuple[float, np.ndarray, np.ndarray]:
+    """
+    Compute recall value, precision curve and scores thresholds
+    Adapted from https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocotools/cocoeval.py
+
+    Args:
+        tp (np.ndarray): cumsum over true positives [R], R is the number of detections
+        fp (np.ndarray): cumsum over false positives [R], R is the number of detections
+        dt_scores_sorted (np.ndarray): sorted (descending) scores [R], R is the number of detections
+        recall_thresholds (Sequence[float]): recall thresholds which should be evaluated
+        num_gt (int): number of ground truth bounding boxes (excluding boxes which are ignored)
+
+    Returns:
+        - float, overall recall for given IoU value
+        - np.ndarray, precision values at defined recall values
+          [RTH], where RTH is the number of recall thresholds
+        - np.ndarray, prediction scores corresponding to recall values
+          [RTH], where RTH is the number of recall thresholds
+    """
+    num_recall_th = len(recall_thresholds)
+
+    rc = tp / num_gt
+    # np.spacing(1) is the smallest representable epsilon with float
+    pr = tp / (fp + tp + np.spacing(1))
+
+    if len(tp):
+        recall = rc[-1]
+    else:
+        # no prediction
+        recall = 0
+
+    # array where precision values nearest to given recall th are saved
+    precision = np.zeros((num_recall_th,))
+    # save scores for corresponding recall value in here
+    th_scores = np.zeros((num_recall_th,))
+    # numpy is slow without cython optimization for accessing elements
+    # use python array gets significant speed improvement
+    pr = pr.tolist()
+    precision = precision.tolist()
+
+    # smooth precision curve (create box shape)
+    for i in range(len(tp) - 1, 0, -1):
+        if pr[i] > pr[i - 1]:
+            pr[i - 1] = pr[i]
+
+    # get indices to nearest given recall threshold (nn interpolation!)
+    inds = np.searchsorted(rc, recall_thresholds, side="left")
+    try:
+        for save_idx, array_index in enumerate(inds):
+            precision[save_idx] = pr[array_index]
+            th_scores[save_idx] = dt_scores_sorted[array_index]
+    except BaseException:
+        pass
+
+    return recall, np.array(precision), np.array(th_scores)
diff --git a/monai/apps/detection/metrics/matching.py b/monai/apps/detection/metrics/matching.py
new file mode 100644
--- /dev/null
+++ b/monai/apps/detection/metrics/matching.py
@@ -0,0 +1,367 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# =========================================================================
+# Adapted from https://github.com/MIC-DKFZ/nnDetection/blob/main/nndet/evaluator/detection/matching.py
+# which has the following license...
+# https://github.com/MIC-DKFZ/nnDetection/blob/main/LICENSE
+#
+# Copyright 2020 Division of Medical Image Computing, German Cancer Research Center (DKFZ), Heidelberg, Germany
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#    http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# =========================================================================
+# Adapted from https://github.com/cocodataset/cocoapi
+# which has the following license...
+# https://github.com/cocodataset/cocoapi/blob/master/license.txt
+
+# Copyright (c) 2014, Piotr Dollar and Tsung-Yi Lin
+# All rights reserved.
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+
+# 1. Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# The views and conclusions contained in the software and documentation are those
+# of the authors and should not be interpreted as representing official policies,
+# either expressed or implied, of the FreeBSD Project.
+
+"""
+This script is almost same with https://github.com/MIC-DKFZ/nnDetection/blob/main/nndet/evaluator/detection/matching.py
+The changes include 1) code reformatting, 2) docstrings,
+3) allow input args gt_ignore to be optional. (If so, no GT boxes will be ignored.)
+"""
+
+from typing import Callable, Dict, List, Sequence, Union
+
+import numpy as np
+
+__all__ = ["matching_batch"]
+
+
+def matching_batch(
+    iou_fn: Callable[[np.ndarray, np.ndarray], np.ndarray],
+    iou_thresholds: Sequence[float],
+    pred_boxes: Sequence[np.ndarray],
+    pred_classes: Sequence[np.ndarray],
+    pred_scores: Sequence[np.ndarray],
+    gt_boxes: Sequence[np.ndarray],
+    gt_classes: Sequence[np.ndarray],
+    gt_ignore: Union[Sequence[Sequence[bool]], Sequence[np.ndarray], None] = None,
+    max_detections: int = 100,
+) -> List[Dict[int, Dict[str, np.ndarray]]]:
+    """
+    Match boxes of a batch to corresponding ground truth for each category
+    independently.
+
+    Args:
+        iou_fn: compute overlap for each pair
+        iou_thresholds: defined which IoU thresholds should be evaluated
+        pred_boxes: predicted boxes from single batch; List[[D, dim * 2]],
+            D number of predictions
+        pred_classes: predicted classes from a single batch; List[[D]],
+            D number of predictions
+        pred_scores: predicted score for each bounding box; List[[D]],
+            D number of predictions
+        gt_boxes: ground truth boxes; List[[G, dim * 2]], G number of ground
+            truth
+        gt_classes: ground truth classes; List[[G]], G number of ground truth
+        gt_ignore: specified if which ground truth boxes are not counted as
+            true positives. If not given, when use all the gt_boxes.
+            (detections which match theses boxes are not counted as false
+            positives either); List[[G]], G number of ground truth
+        max_detections: maximum number of detections which should be evaluated
+
+    Returns:
+        List[Dict[int, Dict[str, np.ndarray]]], each Dict[str, np.ndarray] corresponds to an image.
+        Dict has the following keys.
+
+        - `dtMatches`: matched detections [T, D], where T = number of
+          thresholds, D = number of detections
+        - `gtMatches`: matched ground truth boxes [T, G], where T = number
+          of thresholds, G = number of ground truth
+        - `dtScores`: prediction scores [D] detection scores
+        - `gtIgnore`: ground truth boxes which should be ignored
+          [G] indicate whether ground truth should be ignored
+        - `dtIgnore`: detections which should be ignored [T, D],
+          indicate which detections should be ignored
+
+    Example:
+
+        .. code-block:: python
+
+            from monai.data.box_utils import box_iou
+            from monai.apps.detection.metrics.coco import COCOMetric
+            from monai.apps.detection.metrics.matching import matching_batch
+            # 3D example outputs of one image from detector
+            val_outputs_all = [
+                    {"boxes": torch.tensor([[1,1,1,3,4,5]],dtype=torch.float16),
+                    "labels": torch.randint(3,(1,)),
+                    "scores": torch.randn((1,)).absolute()},
+            ]
+            val_targets_all = [
+                    {"boxes": torch.tensor([[1,1,1,2,6,4]],dtype=torch.float16),
+                    "labels": torch.randint(3,(1,))},
+            ]
+
+            coco_metric = COCOMetric(
+                classes=['c0','c1','c2'], iou_list=[0.1], max_detection=[10]
+            )
+            results_metric = matching_batch(
+                iou_fn=box_iou,
+                iou_thresholds=coco_metric.iou_thresholds,
+                pred_boxes=[val_data_i["boxes"].numpy() for val_data_i in val_outputs_all],
+                pred_classes=[val_data_i["labels"].numpy() for val_data_i in val_outputs_all],
+                pred_scores=[val_data_i["scores"].numpy() for val_data_i in val_outputs_all],
+                gt_boxes=[val_data_i["boxes"].numpy() for val_data_i in val_targets_all],
+                gt_classes=[val_data_i["labels"].numpy() for val_data_i in val_targets_all],
+            )
+            val_metric_dict = coco_metric(results_metric)
+            print(val_metric_dict)
+    """
+    results = []
+    if gt_ignore is None:
+        gt_ignore = [np.full_like(gt_c, False) for gt_c in gt_classes]
+    # iterate over images/batches
+    for pboxes, pclasses, pscores, gboxes, gclasses, gignore in zip(
+        pred_boxes, pred_classes, pred_scores, gt_boxes, gt_classes, gt_ignore
+    ):
+        # for each image
+        img_classes = np.union1d(pclasses, gclasses)  # possible class labels
+        result = {}  # dict contains results for each class in one image
+        for c in img_classes:
+            pred_mask = pclasses == c  # bool mask predictions with current class
+            gt_mask = gclasses == c  # nool mask ground trtuh with current class
+
+            if not np.any(gt_mask):  # no ground truth
+                result[c] = _matching_no_gt(
+                    iou_thresholds=iou_thresholds, pred_scores=pscores[pred_mask], max_detections=max_detections
+                )
+            elif not np.any(pred_mask):  # no predictions
+                result[c] = _matching_no_pred(iou_thresholds=iou_thresholds, gt_ignore=gignore[gt_mask])
+            else:  # at least one prediction and one ground truth
+                result[c] = _matching_single_image_single_class(
+                    iou_fn=iou_fn,
+                    pred_boxes=pboxes[pred_mask],
+                    pred_scores=pscores[pred_mask],
+                    gt_boxes=gboxes[gt_mask],
+                    gt_ignore=gignore[gt_mask],
+                    max_detections=max_detections,
+                    iou_thresholds=iou_thresholds,
+                )
+        results.append(result)
+    return results
+
+
+def _matching_no_gt(
+    iou_thresholds: Sequence[float], pred_scores: np.ndarray, max_detections: int
+) -> Dict[str, np.ndarray]:
+    """
+    Matching result with not ground truth in image
+
+    Args:
+        iou_thresholds: defined which IoU thresholds should be evaluated
+        dt_scores: predicted scores
+        max_detections: maximum number of allowed detections per image.
+            This functions uses this parameter to stay consistent with
+            the actual matching function which needs this limit.
+
+    Returns:
+        computed matching, a Dict[str, np.ndarray]
+
+        - `dtMatches`: matched detections [T, D], where T = number of
+          thresholds, D = number of detections
+        - `gtMatches`: matched ground truth boxes [T, G], where T = number
+          of thresholds, G = number of ground truth
+        - `dtScores`: prediction scores [D] detection scores
+        - `gtIgnore`: ground truth boxes which should be ignored
+          [G] indicate whether ground truth should be ignored
+        - `dtIgnore`: detections which should be ignored [T, D],
+          indicate which detections should be ignored
+    """
+    dt_ind = np.argsort(-pred_scores, kind="mergesort")
+    dt_ind = dt_ind[:max_detections]
+    dt_scores = pred_scores[dt_ind]
+
+    num_preds = len(dt_scores)
+
+    gt_match: np.ndarray = np.array([[]] * len(iou_thresholds))
+    dt_match: np.ndarray = np.zeros((len(iou_thresholds), num_preds))
+    dt_ignore: np.ndarray = np.zeros((len(iou_thresholds), num_preds))
+
+    return {
+        "dtMatches": dt_match,  # [T, D], where T = number of thresholds, D = number of detections
+        "gtMatches": gt_match,  # [T, G], where T = number of thresholds, G = number of ground truth
+        "dtScores": dt_scores,  # [D] detection scores
+        "gtIgnore": np.array([]).reshape(-1),  # [G] indicate whether ground truth should be ignored
+        "dtIgnore": dt_ignore,  # [T, D], indicate which detections should be ignored
+    }
+
+
+def _matching_no_pred(iou_thresholds: Sequence[float], gt_ignore: np.ndarray) -> Dict[str, np.ndarray]:
+    """
+    Matching result with no predictions
+
+    Args:
+        iou_thresholds: defined which IoU thresholds should be evaluated
+        gt_ignore: specified if which ground truth boxes are not counted as
+            true positives (detections which match theses boxes are not
+            counted as false positives either); [G], G number of ground truth
+
+    Returns:
+        dict: computed matching
+
+        - `dtMatches`: matched detections [T, D], where T = number of
+          thresholds, D = number of detections
+        - `gtMatches`: matched ground truth boxes [T, G], where T = number
+          of thresholds, G = number of ground truth
+        - `dtScores`: prediction scores [D] detection scores
+        - `gtIgnore`: ground truth boxes which should be ignored
+          [G] indicate whether ground truth should be ignored
+        - `dtIgnore`: detections which should be ignored [T, D],
+          indicate which detections should be ignored
+    """
+    dt_scores: np.ndarray = np.array([])
+    dt_match: np.ndarray = np.array([[]] * len(iou_thresholds))
+    dt_ignore: np.ndarray = np.array([[]] * len(iou_thresholds))
+
+    n_gt = 0 if gt_ignore.size == 0 else gt_ignore.shape[0]
+    gt_match = np.zeros((len(iou_thresholds), n_gt))
+
+    return {
+        "dtMatches": dt_match,  # [T, D], where T = number of thresholds, D = number of detections
+        "gtMatches": gt_match,  # [T, G], where T = number of thresholds, G = number of ground truth
+        "dtScores": dt_scores,  # [D] detection scores
+        "gtIgnore": gt_ignore.reshape(-1),  # [G] indicate whether ground truth should be ignored
+        "dtIgnore": dt_ignore,  # [T, D], indicate which detections should be ignored
+    }
+
+
+def _matching_single_image_single_class(
+    iou_fn: Callable[[np.ndarray, np.ndarray], np.ndarray],
+    pred_boxes: np.ndarray,
+    pred_scores: np.ndarray,
+    gt_boxes: np.ndarray,
+    gt_ignore: np.ndarray,
+    max_detections: int,
+    iou_thresholds: Sequence[float],
+) -> Dict[str, np.ndarray]:
+    """
+    Adapted from https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocotools/cocoeval.py
+
+    Args:
+        iou_fn: compute overlap for each pair
+        iou_thresholds: defined which IoU thresholds should be evaluated
+        pred_boxes: predicted boxes from single batch; [D, dim * 2], D number
+            of predictions
+        pred_scores: predicted score for each bounding box; [D], D number of
+            predictions
+        gt_boxes: ground truth boxes; [G, dim * 2], G number of ground truth
+        gt_ignore: specified if which ground truth boxes are not counted as
+            true positives (detections which match theses boxes are not
+            counted as false positives either); [G], G number of ground truth
+        max_detections: maximum number of detections which should be evaluated
+
+    Returns:
+        dict: computed matching
+
+        - `dtMatches`: matched detections [T, D], where T = number of
+          thresholds, D = number of detections
+        - `gtMatches`: matched ground truth boxes [T, G], where T = number
+          of thresholds, G = number of ground truth
+        - `dtScores`: prediction scores [D] detection scores
+        - `gtIgnore`: ground truth boxes which should be ignored
+          [G] indicate whether ground truth should be ignored
+        - `dtIgnore`: detections which should be ignored [T, D],
+          indicate which detections should be ignored
+    """
+    # filter for max_detections highest scoring predictions to speed up computation
+    dt_ind = np.argsort(-pred_scores, kind="mergesort")
+    dt_ind = dt_ind[:max_detections]
+
+    pred_boxes = pred_boxes[dt_ind]
+    pred_scores = pred_scores[dt_ind]
+
+    # sort ignored ground truth to last positions
+    gt_ind = np.argsort(gt_ignore, kind="mergesort")
+    gt_boxes = gt_boxes[gt_ind]
+    gt_ignore = gt_ignore[gt_ind]
+
+    # ious between sorted(!) predictions and ground truth
+    ious = iou_fn(pred_boxes, gt_boxes)  # array sized (num_preds, num_gts)
+
+    num_preds, num_gts = ious.shape[0], ious.shape[1]
+    gt_match = np.zeros((len(iou_thresholds), num_gts))
+    dt_match = np.zeros((len(iou_thresholds), num_preds))
+    dt_ignore = np.zeros((len(iou_thresholds), num_preds))
+
+    for tind, t in enumerate(iou_thresholds):
+        for dind, _d in enumerate(pred_boxes):  # iterate detections starting from highest scoring one
+            # information about best match so far (m=-1 -> unmatched)
+            iou = min([t, 1 - 1e-10])
+            m = -1
+
+            for gind, _g in enumerate(gt_boxes):  # iterate ground truth
+                # if this gt already matched, continue
+                if gt_match[tind, gind] > 0:
+                    continue
+
+                # if dt matched to reg gt, and on ignore gt, stop
+                if m > -1 and gt_ignore[m] == 0 and gt_ignore[gind] == 1:
+                    break
+
+                # continue to next gt unless better match made
+                if ious[dind, gind] < iou:
+                    continue
+
+                # if match successful and best so far, store appropriately
+                iou = ious[dind, gind]
+                m = gind
+
+            # if match made, store id of match for both dt and gt
+            if m == -1:
+                continue
+            else:
+                dt_ignore[tind, dind] = int(gt_ignore[m])
+                dt_match[tind, dind] = 1
+                gt_match[tind, m] = 1
+
+    # store results for given image and category
+    return {
+        "dtMatches": dt_match,  # [T, D], where T = number of thresholds, D = number of detections
+        "gtMatches": gt_match,  # [T, G], where T = number of thresholds, G = number of ground truth
+        "dtScores": pred_scores,  # [D] detection scores
+        "gtIgnore": gt_ignore.reshape(-1),  # [G] indicate whether ground truth should be ignored
+        "dtIgnore": dt_ignore,  # [T, D], indicate which detections should be ignored
+    }


Test patch:
diff --git a/tests/test_detection_coco_metrics.py b/tests/test_detection_coco_metrics.py
new file mode 100644
--- /dev/null
+++ b/tests/test_detection_coco_metrics.py
@@ -0,0 +1,66 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import random
+import unittest
+
+import numpy as np
+import torch
+
+from monai.apps.detection.metrics.coco import COCOMetric
+from monai.apps.detection.metrics.matching import matching_batch
+from monai.data.box_utils import box_iou
+
+
+class TestCOCOMetrics(unittest.TestCase):
+    def test_coco_run(self):
+        coco_metric = COCOMetric(classes=["c0", "c1", "c2"], iou_list=[0.1], max_detection=[10])
+
+        num_images = 10
+
+        val_outputs_all = []
+        val_targets_all = []
+        for _ in range(num_images):
+            # randomly generate gt boxes and pred boxes
+            num_gt_boxes = random.randint(1, 3)
+            num_pred_boxes = random.randint(0, 3)
+
+            box_start = torch.randint(3, (num_pred_boxes, 3))
+            box_stop = box_start + torch.randint(1, 32, (num_pred_boxes, 3))
+            boxes = torch.cat((box_start, box_stop), dim=1).to(torch.float16)
+            val_outputs_all.append(
+                {
+                    "boxes": boxes,
+                    "labels": torch.randint(3, (num_pred_boxes,)),
+                    "scores": torch.randn((num_pred_boxes,)).absolute(),
+                }
+            )
+
+            box_start = torch.randint(3, (num_gt_boxes, 3))
+            box_stop = box_start + torch.randint(1, 32, (num_gt_boxes, 3))
+            boxes = torch.cat((box_start, box_stop), dim=1).to(torch.float16)
+            val_targets_all.append({"boxes": boxes, "labels": torch.randint(3, (num_gt_boxes,))})
+
+        results_metric = matching_batch(
+            iou_fn=box_iou,
+            iou_thresholds=coco_metric.iou_thresholds,
+            pred_boxes=[val_data_i["boxes"].numpy() for val_data_i in val_outputs_all],
+            pred_classes=[val_data_i["labels"].numpy() for val_data_i in val_outputs_all],
+            pred_scores=[val_data_i["scores"].numpy() for val_data_i in val_outputs_all],
+            gt_boxes=[val_data_i["boxes"].numpy() for val_data_i in val_targets_all],
+            gt_classes=[val_data_i["labels"].numpy() for val_data_i in val_targets_all],
+        )
+        val_epoch_metric_dict = coco_metric(results_metric)[0]
+        np.testing.assert_array_less([-0.01], [sum(val_epoch_metric_dict.values())])
+
+
+if __name__ == "__main__":
+    unittest.main()