Repository: Project-MONAI/MONAI

Issue:
Change exception to warning in AUC metric
**Is your feature request related to a problem? Please describe.**
Currently, when the input data is all 0 or all 1, AUC will raise an error:
https://github.com/Project-MONAI/MONAI/blob/dev/monai/metrics/rocauc.py#L69

@SachidanandAlle , @madil90 , and our internal QAs all encountered this error in classification tasks before, especially when **verifying the pipeline with very few samples**. For example, it's easy to trigger the exception in chest X-ray classification task as the task has 15 classes, easy to get all 0 or all 1 samples for a class.

I would like to change the exception to warning and skip the AUC computation for this case, or add a flag to control whether a warning or exception?
@ericspod @wyli @rijobro Do you guys have any other concerns?

Thanks in advance.



Discussion:

Patch:
diff --git a/monai/metrics/rocauc.py b/monai/metrics/rocauc.py
--- a/monai/metrics/rocauc.py
+++ b/monai/metrics/rocauc.py
@@ -9,6 +9,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import warnings
 from typing import Union, cast
 
 import numpy as np
@@ -66,7 +67,8 @@ def _calculate(y_pred: torch.Tensor, y: torch.Tensor) -> float:
     if not (y.ndimension() == y_pred.ndimension() == 1 and len(y) == len(y_pred)):
         raise AssertionError("y and y_pred must be 1 dimension data with same length.")
     if not y.unique().equal(torch.tensor([0, 1], dtype=y.dtype, device=y.device)):
-        raise AssertionError("y values must be 0 or 1, can not be all 0 or all 1.")
+        warnings.warn("y values must be 0 or 1, can not be all 0 or all 1, skip AUC computation and return `Nan`.")
+        return float("nan")
     n = len(y)
     indices = y_pred.argsort()
     y = y[indices].cpu().numpy()


Test patch:
diff --git a/tests/test_compute_roc_auc.py b/tests/test_compute_roc_auc.py
--- a/tests/test_compute_roc_auc.py
+++ b/tests/test_compute_roc_auc.py
@@ -68,9 +68,20 @@
     0.62,
 ]
 
+TEST_CASE_8 = [
+    torch.tensor([[0.1, 0.9], [0.3, 1.4], [0.2, 0.1], [0.1, 0.5]]),
+    torch.tensor([[0], [0], [0], [0]]),
+    True,
+    2,
+    "macro",
+    float("nan"),
+]
+
 
 class TestComputeROCAUC(unittest.TestCase):
-    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7])
+    @parameterized.expand(
+        [TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7, TEST_CASE_8]
+    )
     def test_value(self, y_pred, y, softmax, to_onehot, average, expected_value):
         y_pred_trans = Compose([ToTensor(), Activations(softmax=softmax)])
         y_trans = Compose([ToTensor(), AsDiscrete(to_onehot=to_onehot)])
@@ -79,7 +90,9 @@ def test_value(self, y_pred, y, softmax, to_onehot, average, expected_value):
         result = compute_roc_auc(y_pred=y_pred, y=y, average=average)
         np.testing.assert_allclose(expected_value, result, rtol=1e-5)
 
-    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7])
+    @parameterized.expand(
+        [TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7, TEST_CASE_8]
+    )
     def test_class_value(self, y_pred, y, softmax, to_onehot, average, expected_value):
         y_pred_trans = Compose([ToTensor(), Activations(softmax=softmax)])
         y_trans = Compose([ToTensor(), AsDiscrete(to_onehot=to_onehot)])