Repository: Project-MONAI/MONAI

Issue:
Add distributed support for ROC_AUC metric
**Is your feature request related to a problem? Please describe.**
As ignite doesn't support distributed training for `epoch-gather` metrics, we need to enhance `ROCAUC` referring to:
https://pytorch.org/docs/stable/distributed.html#torch.distributed.gather



Discussion:

Resolving patch:
diff --git a/monai/handlers/roc_auc.py b/monai/handlers/roc_auc.py
--- a/monai/handlers/roc_auc.py
+++ b/monai/handlers/roc_auc.py
@@ -12,11 +12,13 @@
 from typing import Callable, List, Optional, Sequence, Union
 
 import torch
+import torch.distributed as dist
 
 from monai.metrics import compute_roc_auc
 from monai.utils import Average, exact_version, optional_import
 
 Metric, _ = optional_import("ignite.metrics", "0.3.0", exact_version, "Metric")
+reinit__is_reduced, _ = optional_import("ignite.metrics.metric", "0.3.0", exact_version, "reinit__is_reduced")
 
 
 class ROCAUC(Metric):  # type: ignore[valid-type, misc]  # due to optional_import
@@ -44,7 +46,6 @@ class ROCAUC(Metric):  # type: ignore[valid-type, misc]  # due to optional_impor
             :class:`~ignite.engine.Engine` `process_function` output into the
             form expected by the metric. This can be useful if, for example, you have a multi-output model and
             you want to compute the metric with respect to one of the outputs.
-        device: device specification in case of distributed computation usage.
 
     Note:
         ROCAUC expects y to be comprised of 0's and 1's.
@@ -59,18 +60,19 @@ def __init__(
         other_act: Optional[Callable] = None,
         average: Union[Average, str] = Average.MACRO,
         output_transform: Callable = lambda x: x,
-        device: Optional[Union[str, torch.device]] = None,
     ) -> None:
-        super().__init__(output_transform, device=device)
+        super().__init__(output_transform)
         self.to_onehot_y = to_onehot_y
         self.softmax = softmax
         self.other_act = other_act
         self.average: Average = Average(average)
 
+    @reinit__is_reduced
     def reset(self) -> None:
         self._predictions: List[torch.Tensor] = []
         self._targets: List[torch.Tensor] = []
 
+    @reinit__is_reduced
     def update(self, output: Sequence[torch.Tensor]) -> None:
         """
         Args:
@@ -96,6 +98,17 @@ def update(self, output: Sequence[torch.Tensor]) -> None:
     def compute(self):
         _prediction_tensor = torch.cat(self._predictions, dim=0)
         _target_tensor = torch.cat(self._targets, dim=0)
+
+        if dist.is_available() and dist.is_initialized() and not self._is_reduced:
+            # create placeholder to collect the data from all processes:
+            output = [torch.zeros_like(_prediction_tensor) for _ in range(dist.get_world_size())]
+            dist.all_gather(output, _prediction_tensor)
+            _prediction_tensor = torch.cat(output, dim=0)
+            output = [torch.zeros_like(_target_tensor) for _ in range(dist.get_world_size())]
+            dist.all_gather(output, _target_tensor)
+            _target_tensor = torch.cat(output, dim=0)
+            self._is_reduced = True
+
         return compute_roc_auc(
             y_pred=_prediction_tensor,
             y=_target_tensor,


Test patch:
diff --git a/tests/test_handler_rocauc_dist.py b/tests/test_handler_rocauc_dist.py
new file mode 100644
--- /dev/null
+++ b/tests/test_handler_rocauc_dist.py
@@ -0,0 +1,47 @@
+# Copyright 2020 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import numpy as np
+import torch
+import torch.distributed as dist
+
+from monai.handlers import ROCAUC
+
+
+def main():
+    dist.init_process_group(backend="nccl", init_method="env://")
+
+    auc_metric = ROCAUC(to_onehot_y=True, softmax=True)
+
+    if dist.get_rank() == 0:
+        y_pred = torch.tensor([[0.1, 0.9], [0.3, 1.4]], device=torch.device("cuda:0"))
+        y = torch.tensor([[0], [1]], device=torch.device("cuda:0"))
+        auc_metric.update([y_pred, y])
+
+    if dist.get_rank() == 1:
+        y_pred = torch.tensor([[0.2, 0.1], [0.1, 0.5]], device=torch.device("cuda:1"))
+        y = torch.tensor([[0], [1]], device=torch.device("cuda:1"))
+        auc_metric.update([y_pred, y])
+
+    result = auc_metric.compute()
+    np.testing.assert_allclose(0.75, result)
+
+    dist.destroy_process_group()
+
+
+# python -m torch.distributed.launch --nproc_per_node=NUM_GPUS_PER_NODE
+#        --nnodes=NUM_NODES --node_rank=INDEX_CURRENT_NODE
+#        --master_addr="192.168.1.1" --master_port=1234
+#        test_handler_rocauc_dist.py
+
+if __name__ == "__main__":
+    main()