Repository: Project-MONAI/MONAI

Issue:
Add sample count in metric_summary report
**Is your feature request related to a problem? Please describe.**
Need to add the sample count for every class in the `metric_summary` report, and enhance the doc-string of average computation progress.
Feature request from internal research team.



Discussion:

Patch:
diff --git a/monai/handlers/metrics_saver.py b/monai/handlers/metrics_saver.py
--- a/monai/handlers/metrics_saver.py
+++ b/monai/handlers/metrics_saver.py
@@ -47,13 +47,22 @@ class MetricsSaver:
             if not None, every metric_details array will save a separate `{metric name}_raw.csv` file.
         batch_transform: callable function to extract the meta_dict from input batch data if saving metric details.
             used to extract filenames from input dict data.
-        summary_ops: expected computation operations to generate the summary report based on specified metric_details.
-            it can be: None, "*" or list of strings.
-            None - don't generate summary report for every specified metric_details
+        summary_ops: expected computation operations to generate the summary report.
+            it can be: None, "*" or list of strings, default to None.
+            None - don't generate summary report for every expected metric_details.
             "*" - generate summary report for every metric_details with all the supported operations.
             list of strings - generate summary report for every metric_details with specified operations, they
-            should be within this list: [`mean`, `median`, `max`, `min`, `90percent`, `std`].
-            default to None.
+            should be within list: ["mean", "median", "max", "min", "<int>percentile", "std", "notnans"].
+            the number in "<int>percentile" should be [0, 100], like: "15percentile". default: "90percentile".
+            for more details, please check: https://numpy.org/doc/stable/reference/generated/numpy.nanpercentile.html.
+            note that: for the overall summary, it computes `nanmean` of all classes for each image first,
+            then compute summary. example of the generated summary report::
+
+                class    mean    median    max    5percentile 95percentile  notnans
+                class0  6.0000   6.0000   7.0000   5.1000      6.9000       2.0000
+                class1  6.0000   6.0000   6.0000   6.0000      6.0000       1.0000
+                mean    6.2500   6.2500   7.0000   5.5750      6.9250       2.0000
+
         save_rank: only the handler on specified rank will save to files in multi-gpus validation, default to 0.
         delimiter: the delimiter character in CSV file, default to "\t".
         output_type: expected output file type, supported types: ["csv"], default to "csv".
diff --git a/monai/handlers/utils.py b/monai/handlers/utils.py
--- a/monai/handlers/utils.py
+++ b/monai/handlers/utils.py
@@ -140,15 +140,24 @@ def write_metrics_reports(
         images: name or path of every input image corresponding to the metric_details data.
             if None, will use index number as the filename of every input image.
         metrics: a dictionary of (metric name, metric value) pairs.
-        metric_details: a dictionary of (metric name, metric raw values) pairs, usually, it comes from metrics computation,
-            for example, the raw value can be the mean_dice of every channel of every input image.
+        metric_details: a dictionary of (metric name, metric raw values) pairs, usually, it comes from metrics
+            computation, for example, the raw value can be the mean_dice of every channel of every input image.
         summary_ops: expected computation operations to generate the summary report.
-            it can be: None, "*" or list of strings.
-            None - don't generate summary report for every expected metric_details
+            it can be: None, "*" or list of strings, default to None.
+            None - don't generate summary report for every expected metric_details.
             "*" - generate summary report for every metric_details with all the supported operations.
             list of strings - generate summary report for every metric_details with specified operations, they
-            should be within this list: [`mean`, `median`, `max`, `min`, `90percent`, `std`].
-            default to None.
+            should be within list: ["mean", "median", "max", "min", "<int>percentile", "std", "notnans"].
+            the number in "<int>percentile" should be [0, 100], like: "15percentile". default: "90percentile".
+            for more details, please check: https://numpy.org/doc/stable/reference/generated/numpy.nanpercentile.html.
+            note that: for the overall summary, it computes `nanmean` of all classes for each image first,
+            then compute summary. example of the generated summary report::
+
+                class    mean    median    max    5percentile 95percentile  notnans
+                class0  6.0000   6.0000   7.0000   5.1000      6.9000       2.0000
+                class1  6.0000   6.0000   6.0000   6.0000      6.0000       1.0000
+                mean    6.2500   6.2500   7.0000   5.5750      6.9250       2.0000
+
         deli: the delimiter character in the file, default to "\t".
         output_type: expected output file type, supported types: ["csv"], default to "csv".
 
@@ -190,15 +199,23 @@ def write_metrics_reports(
                         "median": np.nanmedian,
                         "max": np.nanmax,
                         "min": np.nanmin,
-                        "90percent": lambda x: np.nanpercentile(x, 10),
+                        "90percentile": lambda x: np.nanpercentile(x[0], x[1]),
                         "std": np.nanstd,
+                        "notnans": lambda x: (~np.isnan(x)).sum(),
                     }
                 )
                 ops = ensure_tuple(summary_ops)
                 if "*" in ops:
                     ops = tuple(supported_ops.keys())
 
+                def _compute_op(op: str, d: np.ndarray):
+                    if op.endswith("percentile"):
+                        threshold = int(op.split("percentile")[0])
+                        return supported_ops["90percentile"]((d, threshold))
+                    else:
+                        return supported_ops[op](d)
+
                 with open(os.path.join(save_dir, f"{k}_summary.csv"), "w") as f:
                     f.write(f"class{deli}{deli.join(ops)}\n")
                     for i, c in enumerate(np.transpose(v)):
-                        f.write(f"{class_labels[i]}{deli}{deli.join([f'{supported_ops[k](c):.4f}' for k in ops])}\n")
+                        f.write(f"{class_labels[i]}{deli}{deli.join([f'{_compute_op(k, c):.4f}' for k in ops])}\n")
diff --git a/monai/transforms/spatial/array.py b/monai/transforms/spatial/array.py
--- a/monai/transforms/spatial/array.py
+++ b/monai/transforms/spatial/array.py
@@ -1437,7 +1437,8 @@ def __call__(
                 See also: https://pytorch.org/docs/stable/nn.functional.html#grid-sample
         """
         self.randomize()
-        # if not doing transform and spatial size not defined, nothing to do except convert to float and convert numpy/torch
+        # if not doing transform and spatial size not defined, nothing to do
+        # except convert to float and convert numpy/torch
         if not self._do_transform and not spatial_size and not self.spatial_size:
             img = img.float() if isinstance(img, torch.Tensor) else img.astype("float32")
             return torch.Tensor(img) if self.resampler.as_tensor_output else np.array(img)


Test patch:
diff --git a/tests/test_handler_metrics_saver.py b/tests/test_handler_metrics_saver.py
--- a/tests/test_handler_metrics_saver.py
+++ b/tests/test_handler_metrics_saver.py
@@ -28,7 +28,7 @@ def test_content(self):
                 metrics=["metric1", "metric2"],
                 metric_details=["metric3", "metric4"],
                 batch_transform=lambda x: x["image_meta_dict"],
-                summary_ops=["mean", "median", "max", "90percent"],
+                summary_ops=["mean", "median", "max", "5percentile", "95percentile", "notnans"],
             )
             # set up engine
             data = [
@@ -46,7 +46,7 @@ def _save_metrics(engine):
                 engine.state.metrics = {"metric1": 1, "metric2": 2}
                 engine.state.metric_details = {
                     "metric3": torch.tensor([[1, 2], [2, 3]]),
-                    "metric4": torch.tensor([[5, 6], [7, 8]]),
+                    "metric4": torch.tensor([[5, 6], [7, torch.tensor(float("nan"))]]),
                 }
 
             metrics_saver.attach(engine)
@@ -67,17 +67,17 @@ def _save_metrics(engine):
                         self.assertEqual(row, [f"filepath{i}\t{float(i)}\t{float(i + 1)}\t{i + 0.5}"])
             self.assertTrue(os.path.exists(os.path.join(tempdir, "metric3_summary.csv")))
             # check the metric_summary.csv and content
-            with open(os.path.join(tempdir, "metric3_summary.csv")) as f:
+            with open(os.path.join(tempdir, "metric4_summary.csv")) as f:
                 f_csv = csv.reader(f)
                 for i, row in enumerate(f_csv):
                     if i == 1:
-                        self.assertEqual(row, ["class0\t1.5000\t1.5000\t2.0000\t1.1000"])
+                        self.assertEqual(row, ["class0\t6.0000\t6.0000\t7.0000\t5.1000\t6.9000\t2.0000"])
                     elif i == 2:
-                        self.assertEqual(row, ["class1\t2.5000\t2.5000\t3.0000\t2.1000"])
+                        self.assertEqual(row, ["class1\t6.0000\t6.0000\t6.0000\t6.0000\t6.0000\t1.0000"])
                     elif i == 3:
-                        self.assertEqual(row, ["mean\t2.0000\t2.0000\t2.5000\t1.6000"])
+                        self.assertEqual(row, ["mean\t6.2500\t6.2500\t7.0000\t5.5750\t6.9250\t2.0000"])
             self.assertTrue(os.path.exists(os.path.join(tempdir, "metric4_raw.csv")))
-            self.assertTrue(os.path.exists(os.path.join(tempdir, "metric4_summary.csv")))
+            self.assertTrue(os.path.exists(os.path.join(tempdir, "metric3_summary.csv")))
 
 
 if __name__ == "__main__":
diff --git a/tests/test_handler_metrics_saver_dist.py b/tests/test_handler_metrics_saver_dist.py
--- a/tests/test_handler_metrics_saver_dist.py
+++ b/tests/test_handler_metrics_saver_dist.py
@@ -20,6 +20,7 @@
 from ignite.engine import Engine, Events
 
 from monai.handlers import MetricsSaver
+from monai.handlers.utils import evenly_divisible_all_gather
 from tests.utils import DistCall, DistTestCase, SkipIfBeforePyTorchVersion
 
 
@@ -72,6 +73,13 @@ def _save_metrics1(engine):
                     "metric4": torch.tensor([[6, 7], [7, 8]]),
                 }
 
+        @engine.on(Events.EPOCH_COMPLETED)
+        def _all_gather(engine):
+            scores = engine.state.metric_details["metric3"]
+            engine.state.metric_details["metric3"] = evenly_divisible_all_gather(data=scores)
+            scores = engine.state.metric_details["metric4"]
+            engine.state.metric_details["metric4"] = evenly_divisible_all_gather(data=scores)
+
         metrics_saver.attach(engine)
         engine.run(data, max_epochs=1)
 
@@ -96,11 +104,11 @@ def _save_metrics1(engine):
                 f_csv = csv.reader(f)
                 for i, row in enumerate(f_csv):
                     if i == 1:
-                        self.assertEqual(row, ["class0\t1.0000\t1.0000\t1.0000\t1.0000\t1.0000\t0.0000"])
+                        self.assertEqual(row, ["class0\t2.0000\t2.0000\t3.0000\t1.0000\t2.8000\t0.8165\t3.0000"])
                     elif i == 2:
-                        self.assertEqual(row, ["class1\t2.0000\t2.0000\t2.0000\t2.0000\t2.0000\t0.0000"])
+                        self.assertEqual(row, ["class1\t3.0000\t3.0000\t4.0000\t2.0000\t3.8000\t0.8165\t3.0000"])
                     elif i == 3:
-                        self.assertEqual(row, ["mean\t1.5000\t1.5000\t1.5000\t1.5000\t1.5000\t0.0000"])
+                        self.assertEqual(row, ["mean\t2.5000\t2.5000\t3.5000\t1.5000\t3.3000\t0.8165\t3.0000"])
             self.assertTrue(os.path.exists(os.path.join(tempdir, "metric4_raw.csv")))
             self.assertTrue(os.path.exists(os.path.join(tempdir, "metric4_summary.csv")))
 
diff --git a/tests/test_write_metrics_reports.py b/tests/test_write_metrics_reports.py
--- a/tests/test_write_metrics_reports.py
+++ b/tests/test_write_metrics_reports.py
@@ -27,7 +27,7 @@ def test_content(self):
                 images=["filepath1", "filepath2"],
                 metrics={"metric1": 1, "metric2": 2},
                 metric_details={"metric3": torch.tensor([[1, 2], [2, 3]]), "metric4": torch.tensor([[5, 6], [7, 8]])},
-                summary_ops=["mean", "median", "max", "90percent"],
+                summary_ops=["mean", "median", "max", "90percentile"],
                 deli="\t",
                 output_type="csv",
             )
@@ -51,11 +51,11 @@ def test_content(self):
                 f_csv = csv.reader(f)
                 for i, row in enumerate(f_csv):
                     if i == 1:
-                        self.assertEqual(row, ["class0\t1.5000\t1.5000\t2.0000\t1.1000"])
+                        self.assertEqual(row, ["class0\t1.5000\t1.5000\t2.0000\t1.9000"])
                     elif i == 2:
-                        self.assertEqual(row, ["class1\t2.5000\t2.5000\t3.0000\t2.1000"])
+                        self.assertEqual(row, ["class1\t2.5000\t2.5000\t3.0000\t2.9000"])
                     elif i == 3:
-                        self.assertEqual(row, ["mean\t2.0000\t2.0000\t2.5000\t1.6000"])
+                        self.assertEqual(row, ["mean\t2.0000\t2.0000\t2.5000\t2.4000"])
             self.assertTrue(os.path.exists(os.path.join(tempdir, "metric4_raw.csv")))
             self.assertTrue(os.path.exists(os.path.join(tempdir, "metric4_summary.csv")))