Repository: Project-MONAI/MONAI

Issue:
MetricsReloaded Wrapper as a monai handler
Follow up of https://github.com/Project-MONAI/MONAI/pull/5741
the MetricsReloaded Wrapper could be made available as a component in `monai.handlers`, 
and so that it can work with monai.engine and model bundles written with workflows.

cc @brudfors @Nic-Ma 


Discussion:
Hi @yiheng-wang-nv ,

Could you please help on this ticket?

Thanks in advance.

Ni @Nic-Ma, For your info, I am meeting with Carole tomorrow to discuss the MetricsReloaded API, so should be able to work on this shortly thereafter.
> Ni @Nic-Ma, For your info, I am meeting with Carole tomorrow to discuss the MetricsReloaded API, so should be able to work on this shortly thereafter.

Looks great. Don't worry, I think @yiheng-wang-nv may already implemented the handler for `MetricsReloaded`, it's not a big task.

Thanks.


Resolving patch:
diff --git a/monai/handlers/__init__.py b/monai/handlers/__init__.py
--- a/monai/handlers/__init__.py
+++ b/monai/handlers/__init__.py
@@ -25,6 +25,7 @@
 from .mean_dice import MeanDice
 from .mean_iou import MeanIoUHandler
 from .metric_logger import MetricLogger, MetricLoggerKeys
+from .metrics_reloaded_handler import MetricsReloadedBinaryHandler, MetricsReloadedCategoricalHandler
 from .metrics_saver import MetricsSaver
 from .mlflow_handler import MLFlowHandler
 from .nvtx_handlers import MarkHandler, RangeHandler, RangePopHandler, RangePushHandler
diff --git a/monai/handlers/metrics_reloaded_handler.py b/monai/handlers/metrics_reloaded_handler.py
new file mode 100644
--- /dev/null
+++ b/monai/handlers/metrics_reloaded_handler.py
@@ -0,0 +1,115 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+from monai.handlers.ignite_metric import IgniteMetric
+from monai.metrics import MetricsReloadedBinary, MetricsReloadedCategorical
+from monai.utils.enums import MetricReduction
+
+
+class MetricsReloadedBinaryHandler(IgniteMetric):
+    """
+    Handler of MetricsReloadedBinary, which wraps the binary pairwise metrics of MetricsReloaded.
+    """
+
+    def __init__(
+        self,
+        metric_name: str,
+        include_background: bool = True,
+        reduction: MetricReduction | str = MetricReduction.MEAN,
+        get_not_nans: bool = False,
+        output_transform: Callable = lambda x: x,
+        save_details: bool = True,
+    ) -> None:
+        """
+
+        Args:
+            metric_name: Name of a binary metric from the MetricsReloaded package.
+            include_background: whether to skip computation on the first channel of
+                the predicted output. Defaults to ``True``.
+            reduction: define mode of reduction to the metrics, will only apply reduction on `not-nan` values,
+                available reduction modes: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``,
+                ``"mean_channel"``, ``"sum_channel"``}, default to ``"mean"``. if "none", will not do reduction.
+            get_not_nans: whether to return the `not_nans` count, if True, aggregate() returns (metric, not_nans).
+                Here `not_nans` count the number of not nans for the metric,
+                thus its shape equals to the shape of the metric.
+            output_transform: callable to extract `y_pred` and `y` from `ignite.engine.state.output` then
+                construct `(y_pred, y)` pair, where `y_pred` and `y` can be `batch-first` Tensors or
+                lists of `channel-first` Tensors. the form of `(y_pred, y)` is required by the `update()`.
+                `engine.state` and `output_transform` inherit from the ignite concept:
+                https://pytorch.org/ignite/concepts.html#state, explanation and usage example are in the tutorial:
+                https://github.com/Project-MONAI/tutorials/blob/master/modules/batch_output_transform.ipynb.
+            save_details: whether to save metric computation details per image, for example: TP/TN/FP/FN of every image.
+                default to True, will save to `engine.state.metric_details` dict with the metric name as key.
+
+        See also:
+            :py:meth:`monai.metrics.wrapper`
+        """
+        metric_fn = MetricsReloadedBinary(
+            metric_name=metric_name,
+            include_background=include_background,
+            reduction=reduction,
+            get_not_nans=get_not_nans,
+        )
+        super().__init__(metric_fn=metric_fn, output_transform=output_transform, save_details=save_details)
+
+
+class MetricsReloadedCategoricalHandler(IgniteMetric):
+    """
+    Handler of MetricsReloadedCategorical, which wraps the categorical pairwise metrics of MetricsReloaded.
+    """
+
+    def __init__(
+        self,
+        metric_name: str,
+        include_background: bool = True,
+        reduction: MetricReduction | str = MetricReduction.MEAN,
+        get_not_nans: bool = False,
+        smooth_dr: float = 1e-5,
+        output_transform: Callable = lambda x: x,
+        save_details: bool = True,
+    ) -> None:
+        """
+
+        Args:
+            metric_name: Name of a categorical metric from the MetricsReloaded package.
+            include_background: whether to skip computation on the first channel of
+                the predicted output. Defaults to ``True``.
+            reduction: define mode of reduction to the metrics, will only apply reduction on `not-nan` values,
+                available reduction modes: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``,
+                ``"mean_channel"``, ``"sum_channel"``}, default to ``"mean"``. if "none", will not do reduction.
+            get_not_nans: whether to return the `not_nans` count, if True, aggregate() returns (metric, not_nans).
+                Here `not_nans` count the number of not nans for the metric,
+                thus its shape equals to the shape of the metric.
+            smooth_dr: a small constant added to the denominator to avoid nan. OBS: should be greater than zero.
+            output_transform: callable to extract `y_pred` and `y` from `ignite.engine.state.output` then
+                construct `(y_pred, y)` pair, where `y_pred` and `y` can be `batch-first` Tensors or
+                lists of `channel-first` Tensors. the form of `(y_pred, y)` is required by the `update()`.
+                `engine.state` and `output_transform` inherit from the ignite concept:
+                https://pytorch.org/ignite/concepts.html#state, explanation and usage example are in the tutorial:
+                https://github.com/Project-MONAI/tutorials/blob/master/modules/batch_output_transform.ipynb.
+            save_details: whether to save metric computation details per image, for example: TP/TN/FP/FN of every image.
+                default to True, will save to `engine.state.metric_details` dict with the metric name as key.
+
+        See also:
+            :py:meth:`monai.metrics.wrapper`
+        """
+        metric_fn = MetricsReloadedCategorical(
+            metric_name=metric_name,
+            include_background=include_background,
+            reduction=reduction,
+            get_not_nans=get_not_nans,
+            smooth_dr=smooth_dr,
+        )
+        super().__init__(metric_fn=metric_fn, output_transform=output_transform, save_details=save_details)
diff --git a/monai/metrics/wrapper.py b/monai/metrics/wrapper.py
--- a/monai/metrics/wrapper.py
+++ b/monai/metrics/wrapper.py
@@ -30,8 +30,8 @@ class MetricsReloadedWrapper(CumulativeIterationMetric):
     """Base class for defining MetricsReloaded metrics as a CumulativeIterationMetric.
 
     Args:
-        metric_name: Name of a binary metric from the MetricsReloaded package.
-        include_background: whether to skip Dice computation on the first channel of
+        metric_name: Name of a metric from the MetricsReloaded package.
+        include_background: whether to skip computation on the first channel of
             the predicted output. Defaults to ``True``.
         reduction: define mode of reduction to the metrics, will only apply reduction on `not-nan` values,
             available reduction modes: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``,
@@ -80,7 +80,7 @@ class MetricsReloadedBinary(MetricsReloadedWrapper):
 
     Args:
         metric_name: Name of a binary metric from the MetricsReloaded package.
-        include_background: whether to skip Dice computation on the first channel of
+        include_background: whether to skip computation on the first channel of
             the predicted output. Defaults to ``True``.
         reduction: define mode of reduction to the metrics, will only apply reduction on `not-nan` values,
             available reduction modes: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``,
@@ -185,7 +185,7 @@ class MetricsReloadedCategorical(MetricsReloadedWrapper):
 
     Args:
         metric_name: Name of a categorical metric from the MetricsReloaded package.
-        include_background: whether to skip Dice computation on the first channel of
+        include_background: whether to skip computation on the first channel of
             the predicted output. Defaults to ``True``.
         reduction: define mode of reduction to the metrics, will only apply reduction on `not-nan` values,
             available reduction modes: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``,


Test patch:
diff --git a/tests/min_tests.py b/tests/min_tests.py
--- a/tests/min_tests.py
+++ b/tests/min_tests.py
@@ -67,6 +67,7 @@ def run_testsuit():
         "test_global_mutual_information_loss",
         "test_grid_patch",
         "test_gmm",
+        "test_handler_metrics_reloaded",
         "test_handler_checkpoint_loader",
         "test_handler_checkpoint_saver",
         "test_handler_classification_saver",
diff --git a/tests/test_handler_metrics_reloaded.py b/tests/test_handler_metrics_reloaded.py
new file mode 100644
--- /dev/null
+++ b/tests/test_handler_metrics_reloaded.py
@@ -0,0 +1,149 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import unittest
+
+import torch
+from ignite.engine import Engine, Events
+from parameterized import parameterized
+
+from monai.handlers import MetricsReloadedBinaryHandler, MetricsReloadedCategoricalHandler, from_engine
+from monai.utils import optional_import
+from tests.utils import assert_allclose
+
+_, has_metrics = optional_import("MetricsReloaded")
+
+TEST_CASE_BIN_1 = [
+    {"metric_name": "Volume Difference"},
+    [torch.tensor([[[1.0, 0.0], [0.0, 1.0]]]), torch.tensor([[[1.0, 0.0], [0.0, 1.0]]])],
+    [torch.tensor([[[1.0, 0.0], [1.0, 1.0]]]), torch.tensor([[[1.0, 0.0], [1.0, 1.0]]])],
+    0.3333,
+]
+
+TEST_CASE_BIN_2 = [
+    {"metric_name": "Boundary IoU"},
+    [torch.tensor([[[1.0, 0.0], [0.0, 1.0]]]), torch.tensor([[[1.0, 0.0], [0.0, 1.0]]])],
+    [torch.tensor([[[1.0, 0.0], [1.0, 1.0]]]), torch.tensor([[[1.0, 0.0], [1.0, 1.0]]])],
+    0.6667,
+]
+
+TEST_CASE_BIN_3 = [
+    {"metric_name": "xTh Percentile Hausdorff Distance"},
+    [torch.tensor([[[1.0, 0.0], [0.0, 1.0]]]), torch.tensor([[[1.0, 0.0], [0.0, 1.0]]])],
+    [torch.tensor([[[1.0, 0.0], [1.0, 1.0]]]), torch.tensor([[[1.0, 0.0], [1.0, 1.0]]])],
+    0.9,
+]
+
+TEST_CASE_CAT_1 = [
+    {"metric_name": "Weighted Cohens Kappa"},
+    [
+        torch.tensor([[[0, 0], [0, 1]], [[0, 0], [0, 0]], [[1, 1], [1, 0]]]),
+        torch.tensor([[[0, 0], [0, 1]], [[0, 0], [0, 0]], [[1, 1], [1, 0]]]),
+    ],
+    [
+        torch.tensor([[[1, 0], [0, 1]], [[0, 1], [0, 0]], [[0, 0], [1, 0]]]),
+        torch.tensor([[[1, 0], [0, 1]], [[0, 1], [0, 0]], [[0, 0], [1, 0]]]),
+    ],
+    0.272727,
+]
+
+TEST_CASE_CAT_2 = [
+    {"metric_name": "Matthews Correlation Coefficient"},
+    [
+        torch.tensor([[[0, 0], [0, 1]], [[0, 0], [0, 0]], [[1, 1], [1, 0]]]),
+        torch.tensor([[[0, 0], [0, 1]], [[0, 0], [0, 0]], [[1, 1], [1, 0]]]),
+    ],
+    [
+        torch.tensor([[[1, 0], [0, 1]], [[0, 1], [0, 0]], [[0, 0], [1, 0]]]),
+        torch.tensor([[[1, 0], [0, 1]], [[0, 1], [0, 0]], [[0, 0], [1, 0]]]),
+    ],
+    0.387298,
+]
+
+
+@unittest.skipIf(not has_metrics, "MetricsReloaded not available.")
+class TestHandlerMetricsReloadedBinary(unittest.TestCase):
+    @parameterized.expand([TEST_CASE_BIN_1, TEST_CASE_BIN_2, TEST_CASE_BIN_3])
+    def test_compute(self, input_params, y_pred, y, expected_value):
+        input_params["output_transform"] = from_engine(["pred", "label"])
+        metric = MetricsReloadedBinaryHandler(**input_params)
+
+        # set up engine
+
+        def _val_func(engine, batch):
+            pass
+
+        engine = Engine(_val_func)
+        metric.attach(engine=engine, name=input_params["metric_name"])
+        engine.state.output = {"pred": y_pred, "label": y}
+        engine.fire_event(Events.ITERATION_COMPLETED)
+
+        engine.state.output = {"pred": y_pred, "label": y}
+        engine.fire_event(Events.ITERATION_COMPLETED)
+
+        engine.fire_event(Events.EPOCH_COMPLETED)
+        assert_allclose(
+            engine.state.metrics[input_params["metric_name"]], expected_value, atol=1e-4, rtol=1e-4, type_test=False
+        )
+
+    @parameterized.expand([TEST_CASE_BIN_1, TEST_CASE_BIN_2, TEST_CASE_BIN_3])
+    def test_shape_mismatch(self, input_params, _y_pred, _y, _expected_value):
+        input_params["output_transform"] = from_engine(["pred", "label"])
+        metric = MetricsReloadedBinaryHandler(**input_params)
+        with self.assertRaises((AssertionError, ValueError)):
+            y_pred = torch.Tensor([[0, 1], [1, 0]])
+            y = torch.ones((2, 3))
+            metric.update([y_pred, y])
+
+        with self.assertRaises((AssertionError, ValueError)):
+            y_pred = [torch.ones((2, 1, 1)), torch.ones((1, 1, 1))]
+            y = [torch.ones((2, 1, 1)), torch.ones((1, 1, 1))]
+            metric.update([y_pred, y])
+
+
+@unittest.skipIf(not has_metrics, "MetricsReloaded not available.")
+class TestMetricsReloadedCategorical(unittest.TestCase):
+    @parameterized.expand([TEST_CASE_CAT_1, TEST_CASE_CAT_2])
+    def test_compute(self, input_params, y_pred, y, expected_value):
+        input_params["output_transform"] = from_engine(["pred", "label"])
+        metric = MetricsReloadedCategoricalHandler(**input_params)
+
+        # set up engine
+
+        def _val_func(engine, batch):
+            pass
+
+        engine = Engine(_val_func)
+        metric.attach(engine=engine, name=input_params["metric_name"])
+        engine.state.output = {"pred": y_pred, "label": y}
+        engine.fire_event(Events.ITERATION_COMPLETED)
+
+        engine.state.output = {"pred": y_pred, "label": y}
+        engine.fire_event(Events.ITERATION_COMPLETED)
+
+        engine.fire_event(Events.EPOCH_COMPLETED)
+        assert_allclose(
+            engine.state.metrics[input_params["metric_name"]], expected_value, atol=1e-4, rtol=1e-4, type_test=False
+        )
+
+    @parameterized.expand([TEST_CASE_CAT_1, TEST_CASE_CAT_2])
+    def test_shape_mismatch(self, input_params, y_pred, y, _expected_value):
+        input_params["output_transform"] = from_engine(["pred", "label"])
+        metric = MetricsReloadedCategoricalHandler(**input_params)
+        with self.assertRaises((AssertionError, ValueError)):
+            y_pred[0] = torch.zeros([3, 2, 1])
+            metric.update([y_pred, y])
+
+
+if __name__ == "__main__":
+    unittest.main()