Repository: Project-MONAI/MONAI

Issue:
add StatsTransform to debug data in transform chain
**Is your feature request related to a problem? Please describe.**
Currently, our transforms compose a chain and hard to know the data after every transform and debug, need to add StatsTransform which can be inserted into the chain to print expected info.


Discussion:
hjmjohnson: @Nic-Ma Exceptional addition!!!   I was putting just this type of debugging in by hand.
Nic-Ma: > @Nic-Ma Exceptional addition!!! I was putting just this type of debugging in by hand.

Cool, glad to see it's helpful. I will try to complete it ASAP.
Thanks.
Nic-Ma: Hi @ericspod and @wyli ,

I already rebased this PR. Could you please help review it?
As there are many args in this transform, so for dictionary-based `DataStatsd`, I didn't implement tuple format for every arg. Instead, if users want to print different content for different keys, suggest using this transform multiple times.
For example:
```py
{
    ... ...
    DataStatsd(keys=('image', 'label'), prefix='image and label', data_shape=True, ...),
    DataStatsd(keys='affine', prefix='affine matrix', data_value=True, ...)
    ... ...
}
Thanks.
hjmjohnson: @Nic-Ma I would recommend that we drop python 3.5.  

It appears that PyTorch is well on the way to removing python 3.5 support already.  The main readme list 3.6, 3.7, 3.8 in its table of supported versions.

https://github.com/pytorch/pytorch


ericspod: I encountered an issue with 3.5 as well so I agree it's time to drop it.
Nic-Ma: Thanks for your comments guys.
OK, I will try to update the code and remove python 3.5 dependency.
Thanks.
Nic-Ma: Hi @ericspod ,

I updated the PR with "f string", could you please help review it again?
Thanks.
wyli: /black

Patch:
diff --git a/docs/source/transforms.rst b/docs/source/transforms.rst
index 4e32153a..4c766996 100644
--- a/docs/source/transforms.rst
+++ b/docs/source/transforms.rst
@@ -303,6 +303,12 @@ Vanilla Transforms
     :members:
     :special-members: __call__
 
+`DataStats`
+~~~~~~~~~~~
+.. autoclass:: DataStats
+    :members:
+    :special-members: __call__
+
 `SimulateDelay`
 ~~~~~~~~~~~~~~~
 .. autoclass:: SimulateDelay
@@ -555,6 +561,11 @@ Dictionary-based Transforms
     :members:
     :special-members: __call__
 
+`DataStatsd`
+~~~~~~~~~~~~
+.. autoclass:: DataStatsd
+    :members:
+    :special-members: __call__
 
 `SimulateDelayd`
 ~~~~~~~~~~~~~~~~
diff --git a/monai/transforms/utility/array.py b/monai/transforms/utility/array.py
index 5583bd74..73853010 100644
--- a/monai/transforms/utility/array.py
+++ b/monai/transforms/utility/array.py
@@ -14,6 +14,8 @@ https://github.com/Project-MONAI/MONAI/wiki/MONAI_Design
 """
 import time
 
+from typing import Callable
+import logging
 import numpy as np
 import torch
 
@@ -166,6 +168,64 @@ class SqueezeDim(Transform):
         return np.squeeze(img, self.dim)
 
 
+class DataStats(Transform):
+    """
+    Utility transform to show the statistics of data for debug or analysis.
+    It can be inserted into any place of a transform chain and check results of previous transforms.
+    """
+
+    def __init__(
+        self,
+        prefix="Data",
+        data_shape=True,
+        intensity_range=True,
+        data_value=False,
+        additional_info: Callable = None,
+        logger_handler=None,
+    ):
+        """
+        Args:
+            prefix (string): will be printed in format: "{prefix} statistics".
+            data_shape (bool): whether to show the shape of input data.
+            intensity_range (bool): whether to show the intensity value range of input data.
+            data_value (bool): whether to show the raw value of input data.
+                a typical example is to print some properties of Nifti image: affine, pixdim, etc.
+            additional_info (Callable): user can define callable function to extract additional info from input data.
+            logger_handler (logging.handler): add additional handler to output data: save to file, etc.
+                add existing python logging handlers: https://docs.python.org/3/library/logging.handlers.html
+        """
+        assert isinstance(prefix, str), "prefix must be a string."
+        self.prefix = prefix
+        self.data_shape = data_shape
+        self.intensity_range = intensity_range
+        self.data_value = data_value
+        if additional_info is not None:
+            assert isinstance(additional_info, Callable), "additional_info must be a Callable function."
+        self.additional_info = additional_info
+        self.output = None
+        logging.basicConfig(level=logging.NOTSET)
+        self._logger = logging.getLogger("DataStats")
+        if logger_handler is not None:
+            self._logger.addHandler(logger_handler)
+
+    def __call__(self, img):
+        lines = [f"{self.prefix} statistics:"]
+
+        if self.data_shape:
+            lines.append(f"Shape: {img.shape}")
+        if self.intensity_range:
+            lines.append(f"Intensity range: ({np.min(img)}, {np.max(img)})")
+        if self.data_value:
+            lines.append(f"Value: {img}")
+        if self.additional_info:
+            lines.append(f"Additional info: {self.additional_info(img)}")
+        separator = "\n"
+        self.output = f"{separator.join(lines)}"
+        self._logger.debug(self.output)
+
+        return img
+
+
 class SimulateDelay(Transform):
     """
     This is a pass through transform to be used for testing purposes. It allows
diff --git a/monai/transforms/utility/dictionary.py b/monai/transforms/utility/dictionary.py
index 4c420829..4a7a1a3d 100644
--- a/monai/transforms/utility/dictionary.py
+++ b/monai/transforms/utility/dictionary.py
@@ -26,6 +26,7 @@ from monai.transforms.utility.array import (
     CastToType,
     RepeatChannel,
     SqueezeDim,
+    DataStats,
     SimulateDelay,
 )
 
@@ -201,6 +202,44 @@ class SqueezeDimd(MapTransform):
         return d
 
 
+class DataStatsd(MapTransform):
+    """
+    Dictionary-based wrapper of :py:class:`monai.transforms.DataStats`.
+    """
+
+    def __init__(
+        self,
+        keys,
+        prefix="Data",
+        data_shape=True,
+        intensity_range=True,
+        data_value=False,
+        additional_info=None,
+        logger_handler=None,
+    ):
+        """
+        Args:
+            keys (hashable items): keys of the corresponding items to be transformed.
+                See also: :py:class:`monai.transforms.compose.MapTransform`
+            prefix (string): will be printed in format: "{prefix} statistics".
+            data_shape (bool): whether to show the shape of input data.
+            intensity_range (bool): whether to show the intensity value range of input data.
+            data_value (bool): whether to show the raw value of input data.
+                a typical example is to print some properties of Nifti image: affine, pixdim, etc.
+            additional_info (Callable): user can define callable function to extract additional info from input data.
+            logger_handler (logging.handler): add additional handler to output data: save to file, etc.
+                add existing python logging handlers: https://docs.python.org/3/library/logging.handlers.html
+        """
+        super().__init__(keys)
+        self.printer = DataStats(prefix, data_shape, intensity_range, data_value, additional_info, logger_handler)
+
+    def __call__(self, data):
+        d = dict(data)
+        for key in self.keys:
+            d[key] = self.printer(d[key])
+        return d
+
+
 class SimulateDelayd(MapTransform):
     """
     dictionary-based wrapper of :py:class:monai.transforms.utility.array.SimulateDelay.
@@ -238,4 +277,5 @@ CastToTypeD = CastToTypeDict = CastToTyped
 ToTensorD = ToTensorDict = ToTensord
 DeleteKeysD = DeleteKeysDict = DeleteKeysd
 SqueezeDimD = SqueezeDimDict = SqueezeDimd
+DataStatsD = DataStatsDict = DataStatsd
 SimulateDelayD = SimulateDelayDict = SimulateDelayd


Test patch:
diff --git a/tests/test_data_stats.py b/tests/test_data_stats.py
new file mode 100644
index 00000000..ace79c6e
--- /dev/null
+++ b/tests/test_data_stats.py
@@ -0,0 +1,122 @@
+# Copyright 2020 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+import os
+import logging
+import tempfile
+import numpy as np
+from parameterized import parameterized
+from monai.transforms import DataStats
+
+TEST_CASE_1 = [
+    {
+        "prefix": "test data",
+        "data_shape": False,
+        "intensity_range": False,
+        "data_value": False,
+        "additional_info": None,
+        "logger_handler": None,
+    },
+    np.array([[0, 1], [1, 2]]),
+    "test data statistics:",
+]
+
+TEST_CASE_2 = [
+    {
+        "prefix": "test data",
+        "data_shape": True,
+        "intensity_range": False,
+        "data_value": False,
+        "additional_info": None,
+        "logger_handler": None,
+    },
+    np.array([[0, 1], [1, 2]]),
+    "test data statistics:\nShape: (2, 2)",
+]
+
+TEST_CASE_3 = [
+    {
+        "prefix": "test data",
+        "data_shape": True,
+        "intensity_range": True,
+        "data_value": False,
+        "additional_info": None,
+        "logger_handler": None,
+    },
+    np.array([[0, 1], [1, 2]]),
+    "test data statistics:\nShape: (2, 2)\nIntensity range: (0, 2)",
+]
+
+TEST_CASE_4 = [
+    {
+        "prefix": "test data",
+        "data_shape": True,
+        "intensity_range": True,
+        "data_value": True,
+        "additional_info": None,
+        "logger_handler": None,
+    },
+    np.array([[0, 1], [1, 2]]),
+    "test data statistics:\nShape: (2, 2)\nIntensity range: (0, 2)\nValue: [[0 1]\n [1 2]]",
+]
+
+TEST_CASE_5 = [
+    {
+        "prefix": "test data",
+        "data_shape": True,
+        "intensity_range": True,
+        "data_value": True,
+        "additional_info": lambda x: np.mean(x),
+        "logger_handler": None,
+    },
+    np.array([[0, 1], [1, 2]]),
+    "test data statistics:\nShape: (2, 2)\nIntensity range: (0, 2)\nValue: [[0 1]\n [1 2]]\nAdditional info: 1.0",
+]
+
+TEST_CASE_6 = [
+    np.array([[0, 1], [1, 2]]),
+    "test data statistics:\nShape: (2, 2)\nIntensity range: (0, 2)\nValue: [[0 1]\n [1 2]]\nAdditional info: 1.0\n",
+]
+
+
+class TestDataStats(unittest.TestCase):
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5])
+    def test_value(self, input_param, input_data, expected_print):
+        transform = DataStats(**input_param)
+        _ = transform(input_data)
+        self.assertEqual(transform.output, expected_print)
+
+    @parameterized.expand([TEST_CASE_6])
+    def test_file(self, input_data, expected_print):
+        with tempfile.TemporaryDirectory() as tempdir:
+            filename = os.path.join(tempdir, "test_stats.log")
+            handler = logging.FileHandler(filename, mode="w")
+            input_param = {
+                "prefix": "test data",
+                "data_shape": True,
+                "intensity_range": True,
+                "data_value": True,
+                "additional_info": lambda x: np.mean(x),
+                "logger_handler": handler,
+            }
+            transform = DataStats(**input_param)
+            _ = transform(input_data)
+            handler.stream.close()
+            transform._logger.removeHandler(handler)
+            with open(filename, "r") as f:
+                content = f.read()
+                self.assertEqual(content, expected_print)
+            os.remove(filename)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_data_statsd.py b/tests/test_data_statsd.py
new file mode 100644
index 00000000..9466584a
--- /dev/null
+++ b/tests/test_data_statsd.py
@@ -0,0 +1,123 @@
+# Copyright 2020 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+import os
+import logging
+import tempfile
+import numpy as np
+from parameterized import parameterized
+from monai.transforms import DataStatsd
+
+TEST_CASE_1 = [
+    {
+        "keys": "img",
+        "prefix": "test data",
+        "data_shape": False,
+        "intensity_range": False,
+        "data_value": False,
+        "additional_info": None,
+    },
+    {"img": np.array([[0, 1], [1, 2]])},
+    "test data statistics:",
+]
+
+TEST_CASE_2 = [
+    {
+        "keys": "img",
+        "prefix": "test data",
+        "data_shape": True,
+        "intensity_range": False,
+        "data_value": False,
+        "additional_info": None,
+    },
+    {"img": np.array([[0, 1], [1, 2]])},
+    "test data statistics:\nShape: (2, 2)",
+]
+
+TEST_CASE_3 = [
+    {
+        "keys": "img",
+        "prefix": "test data",
+        "data_shape": True,
+        "intensity_range": True,
+        "data_value": False,
+        "additional_info": None,
+    },
+    {"img": np.array([[0, 1], [1, 2]])},
+    "test data statistics:\nShape: (2, 2)\nIntensity range: (0, 2)",
+]
+
+TEST_CASE_4 = [
+    {
+        "keys": "img",
+        "prefix": "test data",
+        "data_shape": True,
+        "intensity_range": True,
+        "data_value": True,
+        "additional_info": None,
+    },
+    {"img": np.array([[0, 1], [1, 2]])},
+    "test data statistics:\nShape: (2, 2)\nIntensity range: (0, 2)\nValue: [[0 1]\n [1 2]]",
+]
+
+TEST_CASE_5 = [
+    {
+        "keys": "img",
+        "prefix": "test data",
+        "data_shape": True,
+        "intensity_range": True,
+        "data_value": True,
+        "additional_info": lambda x: np.mean(x),
+    },
+    {"img": np.array([[0, 1], [1, 2]])},
+    "test data statistics:\nShape: (2, 2)\nIntensity range: (0, 2)\nValue: [[0 1]\n [1 2]]\nAdditional info: 1.0",
+]
+
+TEST_CASE_6 = [
+    {"img": np.array([[0, 1], [1, 2]])},
+    "test data statistics:\nShape: (2, 2)\nIntensity range: (0, 2)\nValue: [[0 1]\n [1 2]]\nAdditional info: 1.0\n",
+]
+
+
+class TestDataStatsd(unittest.TestCase):
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5])
+    def test_value(self, input_param, input_data, expected_print):
+        transform = DataStatsd(**input_param)
+        _ = transform(input_data)
+        self.assertEqual(transform.printer.output, expected_print)
+
+    @parameterized.expand([TEST_CASE_6])
+    def test_file(self, input_data, expected_print):
+        with tempfile.TemporaryDirectory() as tempdir:
+            filename = os.path.join(tempdir, "test_stats.log")
+            handler = logging.FileHandler(filename, mode="w")
+            input_param = {
+                "keys": "img",
+                "prefix": "test data",
+                "data_shape": True,
+                "intensity_range": True,
+                "data_value": True,
+                "additional_info": lambda x: np.mean(x),
+                "logger_handler": handler,
+            }
+            transform = DataStatsd(**input_param)
+            _ = transform(input_data)
+            handler.stream.close()
+            transform.printer._logger.removeHandler(handler)
+            with open(filename, "r") as f:
+                content = f.read()
+                self.assertEqual(content, expected_print)
+            os.remove(filename)
+
+
+if __name__ == "__main__":
+    unittest.main()