Repository: Project-MONAI/MONAI

Issue:
Add different reductions in metric handlers
**Is your feature request related to a problem? Please describe.**
Currently, all the metric handlers set `reduction=MEAN`.
It would be nice to add reduction support for metric handlers, and if the metric is key metric to compare, must use "MEAN" reduction.




Discussion:

Patch:
diff --git a/monai/engines/workflow.py b/monai/engines/workflow.py
--- a/monai/engines/workflow.py
+++ b/monai/engines/workflow.py
@@ -20,7 +20,7 @@
 from monai.config import IgniteInfo
 from monai.engines.utils import IterationEvents, default_metric_cmp_fn, default_prepare_batch
 from monai.transforms import Decollated, Transform
-from monai.utils import ensure_tuple, min_version, optional_import
+from monai.utils import ensure_tuple, is_scalar, min_version, optional_import
 
 from .utils import engine_apply_transform
 
@@ -229,6 +229,13 @@ def _compare_metrics(engine: Engine) -> None:
             key_metric_name = engine.state.key_metric_name  # type: ignore
             if key_metric_name is not None:
                 current_val_metric = engine.state.metrics[key_metric_name]
+                if not is_scalar(current_val_metric):
+                    warnings.warn(
+                        "key metric is not a scalar value, skip the metric comaprison with best metric."
+                        "please use other metrics as key metric, or change the `reduction` mode to 'mean'."
+                    )
+                    return
+
                 if self.metric_cmp_fn(current_val_metric, engine.state.best_metric):  # type: ignore
                     self.logger.info(f"Got new best metric of {key_metric_name}: {current_val_metric}")
                     engine.state.best_metric = current_val_metric  # type: ignore
diff --git a/monai/handlers/checkpoint_saver.py b/monai/handlers/checkpoint_saver.py
--- a/monai/handlers/checkpoint_saver.py
+++ b/monai/handlers/checkpoint_saver.py
@@ -14,7 +14,7 @@
 from typing import TYPE_CHECKING, Dict, Mapping, Optional
 
 from monai.config import IgniteInfo
-from monai.utils import min_version, optional_import
+from monai.utils import is_scalar, min_version, optional_import
 
 Events, _ = optional_import("ignite.engine", IgniteInfo.OPT_IMPORT_VERSION, min_version, "Events")
 Checkpoint, _ = optional_import("ignite.handlers", IgniteInfo.OPT_IMPORT_VERSION, min_version, "Checkpoint")
@@ -160,8 +160,14 @@ def _score_func(engine: Engine):
                     raise ValueError(
                         f"Incompatible values: save_key_metric=True and key_metric_name={key_metric_name}."
                     )
-
-                return (-1 if key_metric_negative_sign else 1) * engine.state.metrics[metric_name]
+                metric = engine.state.metrics[metric_name]
+                if not is_scalar(metric):
+                    warnings.warn(
+                        "key metric is not a scalar value, skip metric comaprison and don't save a model."
+                        "please use other metrics as key metric, or change the `reduction` mode to 'mean'."
+                    )
+                    return -1
+                return (-1 if key_metric_negative_sign else 1) * metric
 
             if key_metric_filename is not None and key_metric_n_saved > 1:
                 raise ValueError("if using fixed filename to save the best metric model, we should only save 1 model.")
diff --git a/monai/handlers/confusion_matrix.py b/monai/handlers/confusion_matrix.py
--- a/monai/handlers/confusion_matrix.py
+++ b/monai/handlers/confusion_matrix.py
@@ -9,7 +9,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Callable
+from typing import Callable, Union
 
 from monai.handlers.ignite_metric import IgniteMetric
 from monai.metrics import ConfusionMatrixMetric
@@ -25,6 +25,8 @@ def __init__(
         self,
         include_background: bool = True,
         metric_name: str = "hit_rate",
+        compute_sample: bool = False,
+        reduction: Union[MetricReduction, str] = MetricReduction.MEAN,
         output_transform: Callable = lambda x: x,
         save_details: bool = True,
     ) -> None:
@@ -40,6 +42,10 @@ def __init__(
                 ``"informedness"``, ``"markedness"``]
                 Some of the metrics have multiple aliases (as shown in the wikipedia page aforementioned),
                 and you can also input those names instead.
+            compute_sample: when reducing, if ``True``, each sample's metric will be computed based on each confusion matrix first.
+                if ``False``, compute reduction on the confusion matrices first, defaults to ``False``.
+            reduction: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``,
+                ``"mean_channel"``, ``"sum_channel"``}
             output_transform: callable to extract `y_pred` and `y` from `ignite.engine.state.output` then
                 construct `(y_pred, y)` pair, where `y_pred` and `y` can be `batch-first` Tensors or
                 lists of `channel-first` Tensors. the form of `(y_pred, y)` is required by the `update()`.
@@ -55,8 +61,8 @@ def __init__(
         metric_fn = ConfusionMatrixMetric(
             include_background=include_background,
             metric_name=metric_name,
-            compute_sample=False,
-            reduction=MetricReduction.MEAN,
+            compute_sample=compute_sample,
+            reduction=reduction,
         )
         self.metric_name = metric_name
         super().__init__(metric_fn=metric_fn, output_transform=output_transform, save_details=save_details)
diff --git a/monai/handlers/hausdorff_distance.py b/monai/handlers/hausdorff_distance.py
--- a/monai/handlers/hausdorff_distance.py
+++ b/monai/handlers/hausdorff_distance.py
@@ -9,7 +9,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Callable, Optional
+from typing import Callable, Optional, Union
 
 from monai.handlers.ignite_metric import IgniteMetric
 from monai.metrics import HausdorffDistanceMetric
@@ -27,6 +27,7 @@ def __init__(
         distance_metric: str = "euclidean",
         percentile: Optional[float] = None,
         directed: bool = False,
+        reduction: Union[MetricReduction, str] = MetricReduction.MEAN,
         output_transform: Callable = lambda x: x,
         save_details: bool = True,
     ) -> None:
@@ -41,6 +42,9 @@ def __init__(
                 percentile of the Hausdorff Distance rather than the maximum result will be achieved.
                 Defaults to ``None``.
             directed: whether to calculate directed Hausdorff distance. Defaults to ``False``.
+            reduction: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``,
+                ``"mean_channel"``, ``"sum_channel"``}
+                Define the mode to reduce computation result. Defaults to ``"mean"``.
             output_transform: callable to extract `y_pred` and `y` from `ignite.engine.state.output` then
                 construct `(y_pred, y)` pair, where `y_pred` and `y` can be `batch-first` Tensors or
                 lists of `channel-first` Tensors. the form of `(y_pred, y)` is required by the `update()`.
@@ -56,6 +60,6 @@ def __init__(
             distance_metric=distance_metric,
             percentile=percentile,
             directed=directed,
-            reduction=MetricReduction.MEAN,
+            reduction=reduction,
         )
         super().__init__(metric_fn=metric_fn, output_transform=output_transform, save_details=save_details)
diff --git a/monai/handlers/ignite_metric.py b/monai/handlers/ignite_metric.py
--- a/monai/handlers/ignite_metric.py
+++ b/monai/handlers/ignite_metric.py
@@ -101,7 +101,11 @@ def compute(self) -> Any:
                 raise RuntimeError("please call the attach() function to connect expected engine first.")
             self._engine.state.metric_details[self._name] = self.metric_fn.get_buffer()  # type: ignore
 
-        return result.item() if isinstance(result, torch.Tensor) else result
+        if isinstance(result, torch.Tensor):
+            result = result.squeeze()
+            if result.ndim == 0:
+                result = result.item()
+        return result
 
     def attach(self, engine: Engine, name: str) -> None:
         """
diff --git a/monai/handlers/mean_dice.py b/monai/handlers/mean_dice.py
--- a/monai/handlers/mean_dice.py
+++ b/monai/handlers/mean_dice.py
@@ -9,7 +9,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Callable
+from typing import Callable, Union
 
 from monai.handlers.ignite_metric import IgniteMetric
 from monai.metrics import DiceMetric
@@ -22,13 +22,20 @@ class MeanDice(IgniteMetric):
     """
 
     def __init__(
-        self, include_background: bool = True, output_transform: Callable = lambda x: x, save_details: bool = True
+        self,
+        include_background: bool = True,
+        reduction: Union[MetricReduction, str] = MetricReduction.MEAN,
+        output_transform: Callable = lambda x: x,
+        save_details: bool = True,
     ) -> None:
         """
 
         Args:
             include_background: whether to include dice computation on the first channel of the predicted output.
                 Defaults to True.
+            reduction: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``,
+                ``"mean_channel"``, ``"sum_channel"``}
+            Define the mode to reduce computation result. Defaults to ``"mean"``.
             output_transform: callable to extract `y_pred` and `y` from `ignite.engine.state.output` then
                 construct `(y_pred, y)` pair, where `y_pred` and `y` can be `batch-first` Tensors or
                 lists of `channel-first` Tensors. the form of `(y_pred, y)` is required by the `update()`.
@@ -41,5 +48,5 @@ def __init__(
         See also:
             :py:meth:`monai.metrics.meandice.compute_meandice`
         """
-        metric_fn = DiceMetric(include_background=include_background, reduction=MetricReduction.MEAN)
+        metric_fn = DiceMetric(include_background=include_background, reduction=reduction)
         super().__init__(metric_fn=metric_fn, output_transform=output_transform, save_details=save_details)
diff --git a/monai/handlers/regression_metrics.py b/monai/handlers/regression_metrics.py
--- a/monai/handlers/regression_metrics.py
+++ b/monai/handlers/regression_metrics.py
@@ -21,10 +21,18 @@ class MeanSquaredError(IgniteMetric):
     Computes Mean Squared Error from full size Tensor and collects average over batch, iterations.
     """
 
-    def __init__(self, output_transform: Callable = lambda x: x, save_details: bool = True) -> None:
+    def __init__(
+        self,
+        reduction: Union[MetricReduction, str] = MetricReduction.MEAN,
+        output_transform: Callable = lambda x: x,
+        save_details: bool = True,
+    ) -> None:
         """
 
         Args:
+            reduction: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``,
+                ``"mean_channel"``, ``"sum_channel"``}
+                Define the mode to reduce computation result. Defaults to ``"mean"``.
             output_transform: callable to extract `y_pred` and `y` from `ignite.engine.state.output` then
                 construct `(y_pred, y)` pair, where `y_pred` and `y` can be `batch-first` Tensors or
                 lists of `channel-first` Tensors. the form of `(y_pred, y)` is required by the `update()`.
@@ -37,7 +45,7 @@ def __init__(self, output_transform: Callable = lambda x: x, save_details: bool
         See also:
             :py:class:`monai.metrics.MSEMetric`
         """
-        metric_fn = MSEMetric(reduction=MetricReduction.MEAN)
+        metric_fn = MSEMetric(reduction=reduction)
         super().__init__(metric_fn=metric_fn, output_transform=output_transform, save_details=save_details)
 
 
@@ -46,10 +54,18 @@ class MeanAbsoluteError(IgniteMetric):
     Computes Mean Absolute Error from full size Tensor and collects average over batch, iterations.
     """
 
-    def __init__(self, output_transform: Callable = lambda x: x, save_details: bool = True) -> None:
+    def __init__(
+        self,
+        reduction: Union[MetricReduction, str] = MetricReduction.MEAN,
+        output_transform: Callable = lambda x: x,
+        save_details: bool = True,
+    ) -> None:
         """
 
         Args:
+            reduction: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``,
+                ``"mean_channel"``, ``"sum_channel"``}
+                Define the mode to reduce computation result. Defaults to ``"mean"``.
             output_transform: callable to extract `y_pred` and `y` from `ignite.engine.state.output` then
                 construct `(y_pred, y)` pair, where `y_pred` and `y` can be `batch-first` Tensors or
                 lists of `channel-first` Tensors. the form of `(y_pred, y)` is required by the `update()`.
@@ -62,7 +78,7 @@ def __init__(self, output_transform: Callable = lambda x: x, save_details: bool
         See also:
             :py:class:`monai.metrics.MAEMetric`
         """
-        metric_fn = MAEMetric(reduction=MetricReduction.MEAN)
+        metric_fn = MAEMetric(reduction=reduction)
         super().__init__(metric_fn=metric_fn, output_transform=output_transform, save_details=save_details)
 
 
@@ -71,10 +87,18 @@ class RootMeanSquaredError(IgniteMetric):
     Computes Root Mean Squared Error from full size Tensor and collects average over batch, iterations.
     """
 
-    def __init__(self, output_transform: Callable = lambda x: x, save_details: bool = True) -> None:
+    def __init__(
+        self,
+        reduction: Union[MetricReduction, str] = MetricReduction.MEAN,
+        output_transform: Callable = lambda x: x,
+        save_details: bool = True,
+    ) -> None:
         """
 
         Args:
+            reduction: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``,
+                ``"mean_channel"``, ``"sum_channel"``}
+                Define the mode to reduce computation result. Defaults to ``"mean"``.
             output_transform: callable to extract `y_pred` and `y` from `ignite.engine.state.output` then
                 construct `(y_pred, y)` pair, where `y_pred` and `y` can be `batch-first` Tensors or
                 lists of `channel-first` Tensors. the form of `(y_pred, y)` is required by the `update()`.
@@ -87,7 +111,7 @@ def __init__(self, output_transform: Callable = lambda x: x, save_details: bool
         See also:
             :py:class:`monai.metrics.RMSEMetric`
         """
-        metric_fn = RMSEMetric(reduction=MetricReduction.MEAN)
+        metric_fn = RMSEMetric(reduction=reduction)
         super().__init__(metric_fn=metric_fn, output_transform=output_transform, save_details=save_details)
 
 
@@ -97,13 +121,20 @@ class PeakSignalToNoiseRatio(IgniteMetric):
     """
 
     def __init__(
-        self, max_val: Union[int, float], output_transform: Callable = lambda x: x, save_details: bool = True
+        self,
+        max_val: Union[int, float],
+        reduction: Union[MetricReduction, str] = MetricReduction.MEAN,
+        output_transform: Callable = lambda x: x,
+        save_details: bool = True,
     ) -> None:
         """
 
         Args:
             max_val: The dynamic range of the images/volumes (i.e., the difference between the
                 maximum and the minimum allowed values e.g. 255 for a uint8 image).
+            reduction: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``,
+                ``"mean_channel"``, ``"sum_channel"``}
+                Define the mode to reduce computation result. Defaults to ``"mean"``.
             output_transform: callable to extract `y_pred` and `y` from `ignite.engine.state.output` then
                 construct `(y_pred, y)` pair, where `y_pred` and `y` can be `batch-first` Tensors or
                 lists of `channel-first` Tensors. the form of `(y_pred, y)` is required by the `update()`.
@@ -117,5 +148,5 @@ def __init__(
         See also:
             :py:class:`monai.metrics.PSNRMetric`
         """
-        metric_fn = PSNRMetric(max_val=max_val, reduction=MetricReduction.MEAN)
+        metric_fn = PSNRMetric(max_val=max_val, reduction=reduction)
         super().__init__(metric_fn=metric_fn, output_transform=output_transform, save_details=save_details)
diff --git a/monai/handlers/stats_handler.py b/monai/handlers/stats_handler.py
--- a/monai/handlers/stats_handler.py
+++ b/monai/handlers/stats_handler.py
@@ -174,7 +174,7 @@ def _default_epoch_print(self, engine: Engine) -> None:
             out_str = f"Epoch[{current_epoch}] Metrics -- "
             for name in sorted(prints_dict):
                 value = prints_dict[name]
-                out_str += self.key_var_format.format(name, value)
+                out_str += self.key_var_format.format(name, value) if is_scalar(value) else f"{name}: {str(value)}"
             self.logger.info(out_str)
 
         if (
diff --git a/monai/handlers/surface_distance.py b/monai/handlers/surface_distance.py
--- a/monai/handlers/surface_distance.py
+++ b/monai/handlers/surface_distance.py
@@ -9,7 +9,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Callable
+from typing import Callable, Union
 
 from monai.handlers.ignite_metric import IgniteMetric
 from monai.metrics import SurfaceDistanceMetric
@@ -26,6 +26,7 @@ def __init__(
         include_background: bool = False,
         symmetric: bool = False,
         distance_metric: str = "euclidean",
+        reduction: Union[MetricReduction, str] = MetricReduction.MEAN,
         output_transform: Callable = lambda x: x,
         save_details: bool = True,
     ) -> None:
@@ -38,6 +39,9 @@ def __init__(
                 `seg_pred` and `seg_gt`. Defaults to ``False``.
             distance_metric: : [``"euclidean"``, ``"chessboard"``, ``"taxicab"``]
                 the metric used to compute surface distance. Defaults to ``"euclidean"``.
+            reduction: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``,
+                ``"mean_channel"``, ``"sum_channel"``}
+                Define the mode to reduce computation result. Defaults to ``"mean"``.
             output_transform: callable to extract `y_pred` and `y` from `ignite.engine.state.output` then
                 construct `(y_pred, y)` pair, where `y_pred` and `y` can be `batch-first` Tensors or
                 lists of `channel-first` Tensors. the form of `(y_pred, y)` is required by the `update()`.
@@ -52,6 +56,6 @@ def __init__(
             include_background=include_background,
             symmetric=symmetric,
             distance_metric=distance_metric,
-            reduction=MetricReduction.MEAN,
+            reduction=reduction,
         )
         super().__init__(metric_fn=metric_fn, output_transform=output_transform, save_details=save_details)
diff --git a/monai/handlers/tensorboard_handlers.py b/monai/handlers/tensorboard_handlers.py
--- a/monai/handlers/tensorboard_handlers.py
+++ b/monai/handlers/tensorboard_handlers.py
@@ -186,7 +186,8 @@ def _default_epoch_writer(self, engine: Engine, writer: SummaryWriter) -> None:
         current_epoch = self.global_epoch_transform(engine.state.epoch)
         summary_dict = engine.state.metrics
         for name, value in summary_dict.items():
-            writer.add_scalar(name, value, current_epoch)
+            if is_scalar(value):
+                writer.add_scalar(name, value, current_epoch)
 
         if self.state_attributes is not None:
             for attr in self.state_attributes:


Test patch:
diff --git a/tests/test_handler_confusion_matrix.py b/tests/test_handler_confusion_matrix.py
--- a/tests/test_handler_confusion_matrix.py
+++ b/tests/test_handler_confusion_matrix.py
@@ -20,7 +20,7 @@
 
 TEST_CASE_1 = [{"include_background": True, "save_details": False, "metric_name": "f1"}, 0.75]
 TEST_CASE_2 = [{"include_background": False, "save_details": False, "metric_name": "ppv"}, 1.0]
-
+TEST_CASE_3 = [{"save_details": False, "metric_name": "f1", "reduction": "mean_batch"}, torch.tensor([0.6667, 0.8000])]
 TEST_CASE_SEG_1 = [{"include_background": True, "metric_name": "tpr"}, 0.7]
 
 data_1: Dict[Any, Any] = {
@@ -47,7 +47,7 @@
 class TestHandlerConfusionMatrix(unittest.TestCase):
     # TODO test multi node averaged confusion matrix
 
-    @parameterized.expand([TEST_CASE_1, TEST_CASE_2])
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3])
     def test_compute(self, input_params, expected_avg):
         metric = ConfusionMatrix(**input_params)
         # test input a list of channel-first tensor
@@ -60,7 +60,7 @@ def test_compute(self, input_params, expected_avg):
         metric.update([y_pred, y])
 
         avg_metric = metric.compute()
-        self.assertAlmostEqual(avg_metric, expected_avg, places=4)
+        torch.testing.assert_allclose(avg_metric, expected_avg)
 
     @parameterized.expand([TEST_CASE_SEG_1])
     def test_compute_seg(self, input_params, expected_avg):
diff --git a/tests/test_handler_hausdorff_distance.py b/tests/test_handler_hausdorff_distance.py
--- a/tests/test_handler_hausdorff_distance.py
+++ b/tests/test_handler_hausdorff_distance.py
@@ -89,6 +89,21 @@ def test_shape_mismatch(self):
             y = torch.ones((1, 1, 10, 10, 10))
             hd_metric.update([y_pred, y])
 
+    def test_reduction(self):
+        hd_metric = HausdorffDistance(include_background=True, reduction="mean_channel")
+
+        def _val_func(engine, batch):
+            pass
+
+        engine = Engine(_val_func)
+        hd_metric.attach(engine, "hausdorff_distance")
+
+        y_pred, y = TEST_SAMPLE_1
+        hd_metric.update([y_pred, y])
+        y_pred, y = TEST_SAMPLE_2
+        hd_metric.update([y_pred, y])
+        torch.testing.assert_allclose(hd_metric.compute().float(), torch.tensor([10.0, 0.0]))
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_handler_mean_dice.py b/tests/test_handler_mean_dice.py
--- a/tests/test_handler_mean_dice.py
+++ b/tests/test_handler_mean_dice.py
@@ -19,12 +19,17 @@
 
 TEST_CASE_1 = [{"include_background": True, "output_transform": from_engine(["pred", "label"])}, 0.75, (4, 2)]
 TEST_CASE_2 = [{"include_background": False, "output_transform": from_engine(["pred", "label"])}, 0.66666, (4, 1)]
+TEST_CASE_3 = [
+    {"reduction": "mean_channel", "output_transform": from_engine(["pred", "label"])},
+    torch.Tensor([1.0, 0.0, 1.0, 1.0]),
+    (4, 2),
+]
 
 
 class TestHandlerMeanDice(unittest.TestCase):
     # TODO test multi node averaged dice
 
-    @parameterized.expand([TEST_CASE_1, TEST_CASE_2])
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3])
     def test_compute(self, input_params, expected_avg, details_shape):
         dice_metric = MeanDice(**input_params)
         # set up engine
@@ -46,8 +51,7 @@ def _val_func(engine, batch):
         engine.fire_event(Events.ITERATION_COMPLETED)
 
         engine.fire_event(Events.EPOCH_COMPLETED)
-
-        self.assertAlmostEqual(engine.state.metrics["mean_dice"], expected_avg, places=4)
+        torch.testing.assert_allclose(engine.state.metrics["mean_dice"], expected_avg)
         self.assertTupleEqual(tuple(engine.state.metric_details["mean_dice"].shape), details_shape)
 
     @parameterized.expand([TEST_CASE_1, TEST_CASE_2])
diff --git a/tests/test_handler_surface_distance.py b/tests/test_handler_surface_distance.py
--- a/tests/test_handler_surface_distance.py
+++ b/tests/test_handler_surface_distance.py
@@ -89,6 +89,21 @@ def test_shape_mismatch(self):
             y = torch.ones((1, 1, 10, 10, 10))
             sur_metric.update([y_pred, y])
 
+    def test_reduction(self):
+        sur_metric = SurfaceDistance(include_background=True, reduction="mean_channel")
+
+        def _val_func(engine, batch):
+            pass
+
+        engine = Engine(_val_func)
+        sur_metric.attach(engine, "surface_distance")
+
+        y_pred, y = TEST_SAMPLE_1
+        sur_metric.update([y_pred, y])
+        y_pred, y = TEST_SAMPLE_2
+        sur_metric.update([y_pred, y])
+        torch.testing.assert_allclose(sur_metric.compute().float(), torch.tensor([4.1713, 0.0000]))
+
 
 if __name__ == "__main__":
     unittest.main()