Repository: Project-MONAI/MONAI

Issue:
Add docstring for `not_nans` in metrics
Now `get_not_nans` is added for metrics, but `not_nans` does not have the corresponding descriptions.


Discussion:
In addition, the docsting of `get_not_nans` in class `ConfusionMatrixMetric` is wrong according to its `aggregate` function. Now, if there are two metrics and `get_not_nans` is True, it will return a list like:
```
[metric1_result, metric1_not_nans, metric2_result, metric2_not_nans]
```
Let me also change it into:
```
[(metric1_result, metric1_not_nans), (metric2_result, metric2_not_nans)]
```
and change the docstring as well.
Hi @yiheng-wang-nv ,

Thanks for raising this.
Sure, please submit a PR for it.

Thanks.


Patch:
diff --git a/monai/metrics/confusion_matrix.py b/monai/metrics/confusion_matrix.py
--- a/monai/metrics/confusion_matrix.py
+++ b/monai/metrics/confusion_matrix.py
@@ -49,7 +49,11 @@ class ConfusionMatrixMetric(CumulativeIterationMetric):
             if ``False``, compute reduction on the confusion matrices first, defaults to ``False``.
         reduction: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``,
             ``"mean_channel"``, ``"sum_channel"``}
-        get_not_nans: whether to return the `not_nans` count, if True, aggregate() returns (metric, not_nans).
+        get_not_nans: whether to return the `not_nans` count, if True, aggregate() returns [(metric, not_nans), ...]. If False,
+            aggregate() returns [metric, ...].
+            Here `not_nans` count the number of not nans for True Positive, False Positive, True Negative and False Negative.
+            Its shape depends on the shape of the metric, and it has one more dimension with size 4. For example, if the shape
+            of the metric is [3, 3], `not_nans` has the shape [3, 3, 4].
 
     """
 
@@ -118,9 +122,10 @@ def aggregate(self):  # type: ignore
             else:
                 f, not_nans = do_metric_reduction(data, self.reduction)
                 f = compute_confusion_matrix_metric(metric_name, f)
-            results.append(f)
             if self.get_not_nans:
-                results.append(not_nans)
+                results.append((f, not_nans))
+            else:
+                results.append(f)
         return results
 
 
diff --git a/monai/metrics/hausdorff_distance.py b/monai/metrics/hausdorff_distance.py
--- a/monai/metrics/hausdorff_distance.py
+++ b/monai/metrics/hausdorff_distance.py
@@ -46,6 +46,7 @@ class HausdorffDistanceMetric(CumulativeIterationMetric):
             ``"mean_channel"``, ``"sum_channel"``}
             Define the mode to reduce computation result. Defaults to ``"mean"``.
         get_not_nans: whether to return the `not_nans` count, if True, aggregate() returns (metric, not_nans).
+            Here `not_nans` count the number of not nans for the metric, thus its shape equals to the shape of the metric.
 
     """
 
diff --git a/monai/metrics/meandice.py b/monai/metrics/meandice.py
--- a/monai/metrics/meandice.py
+++ b/monai/metrics/meandice.py
@@ -39,6 +39,7 @@ class DiceMetric(CumulativeIterationMetric):
             ``"mean_channel"``, ``"sum_channel"``}
             Define the mode to reduce computation result. Defaults to ``"mean"``.
         get_not_nans: whether to return the `not_nans` count, if True, aggregate() returns (metric, not_nans).
+            Here `not_nans` count the number of not nans for the metric, thus its shape equals to the shape of the metric.
 
     """
 
diff --git a/monai/metrics/regression.py b/monai/metrics/regression.py
--- a/monai/metrics/regression.py
+++ b/monai/metrics/regression.py
@@ -34,6 +34,7 @@ class RegressionMetric(CumulativeIterationMetric):
             ``"mean_channel"``, ``"sum_channel"``}
             Define the mode to reduce computation result. Defaults to ``"mean"``.
         get_not_nans: whether to return the `not_nans` count, if True, aggregate() returns (metric, not_nans).
+            Here `not_nans` count the number of not nans for the metric, thus its shape equals to the shape of the metric.
 
     """
 
diff --git a/monai/metrics/surface_distance.py b/monai/metrics/surface_distance.py
--- a/monai/metrics/surface_distance.py
+++ b/monai/metrics/surface_distance.py
@@ -41,6 +41,7 @@ class SurfaceDistanceMetric(CumulativeIterationMetric):
             ``"mean_channel"``, ``"sum_channel"``}
             Define the mode to reduce computation result. Defaults to ``"mean"``.
         get_not_nans: whether to return the `not_nans` count, if True, aggregate() returns (metric, not_nans).
+            Here `not_nans` count the number of not nans for the metric, thus its shape equals to the shape of the metric.
 
     """
 
diff --git a/monai/metrics/utils.py b/monai/metrics/utils.py
--- a/monai/metrics/utils.py
+++ b/monai/metrics/utils.py
@@ -50,6 +50,8 @@ def do_metric_reduction(
 ):
     """
     This function is to do the metric reduction for calculated metrics of each example's each class.
+    The function also returns `not_nans`, which counts the number of not nans for the metric.
+
     Args:
         f: a tensor that contains the calculated metric scores per batch and
             per class. The first two dims should be batch and class.


Test patch:
diff --git a/tests/test_compute_confusion_matrix.py b/tests/test_compute_confusion_matrix.py
--- a/tests/test_compute_confusion_matrix.py
+++ b/tests/test_compute_confusion_matrix.py
@@ -236,7 +236,7 @@ def test_compute_sample(self, input_data, expected_value):
         vals["y"] = params.pop("y")
         metric = ConfusionMatrixMetric(**params)
         metric(**vals)
-        result, _ = metric.aggregate()
+        result, _ = metric.aggregate()[0]
         np.testing.assert_allclose(result, expected_value, atol=1e-4, rtol=1e-4)
 
     @parameterized.expand(TEST_CASES_COMPUTE_SAMPLE_MULTI_METRICS)
@@ -248,9 +248,9 @@ def test_compute_sample_multiple_metrics(self, input_data, expected_values):
         metric = ConfusionMatrixMetric(**params)
         metric(**vals)
         results = metric.aggregate()
-        for idx in range(0, len(results), 2):
-            result = results[idx]
-            expected_value = expected_values[int(idx / 2)]
+        for idx in range(len(results)):
+            result = results[idx][0]
+            expected_value = expected_values[idx]
             np.testing.assert_allclose(result, expected_value, atol=1e-4, rtol=1e-4)
 
     @parameterized.expand(TEST_CASES_COMPUTE_SAMPLE_NAN)
@@ -261,7 +261,7 @@ def test_compute_sample_with_nan(self, input_data, expected_value, expected_not_
         vals["y"] = params.pop("y")
         metric = ConfusionMatrixMetric(**params)
         metric(**vals)
-        result, not_nans = metric.aggregate()
+        result, not_nans = metric.aggregate()[0]
         np.testing.assert_allclose(result, expected_value, atol=1e-4, rtol=1e-4)
         np.testing.assert_allclose(not_nans, expected_not_nans, atol=1e-4, rtol=1e-4)
 
@@ -274,7 +274,7 @@ def test_clf_with_nan(self, input_data, expected_value):
         metric = ConfusionMatrixMetric(**params)
         result = metric(**vals)
         np.testing.assert_allclose(result, expected_value, atol=1e-4, rtol=1e-4)
-        result, _ = metric.aggregate()
+        result, _ = metric.aggregate()[0]
         expected_value, _ = do_metric_reduction(expected_value, "mean_channel")
         expected_value = compute_confusion_matrix_metric("tpr", expected_value)
         np.testing.assert_allclose(result, expected_value, atol=1e-4, rtol=1e-4)