Repository: Lightning-AI/torchmetrics

Issue:
BinaryMatthewsCorrCoef outputs 0 if target and prediction contains only one case either positive or negative case
### Discussed in https://github.com/Lightning-AI/torchmetrics/discussions/1810

<div type='discussions-op-text'>

<sup>Originally posted by **lokesh1199** May 25, 2023</sup>
Hi,

When the given target and pred contains only one case either positive or negative (case  - 1), The output of BinaryMatthewsCorrCoef is always 0. But we can clearly say that predictions are same as the targets. And for the case - 2, If the preds and targets are completely different, the MCC is also zero. Is this expected behaviour with MCC or some other issue.

```python
mcc = BinaryMatthewsCorrCoef()

# preds and targets are equal (case - 1)
print(mcc(torch.ones(10), torch.ones(10)))
print(mcc(torch.zeros(10), torch.zeros(10)))

# preds and targets are completely different (case - 2)
print(mcc(torch.ones(10), torch.zeros(10)))
```

```python
tensor(0)
tensor(0)
tensor(0)
```
In this [blog](https://bmcgenomics.biomedcentral.com/articles/10.1186/s12864-019-6413-7), They are handling these cases slightly different. Below are the MCC values using their [implementation](https://gist.github.com/lokesh1199/3d899b03e37107565cd91d6ad6d8db90). These values represent model performance more accurately than the torchmetrics version.

case - 1 -> MCC = 1.0
case - 2 -> MCC = -1.0

Kindly tell if this should be handled differently?
</div>


Discussion:
Hi! thanks for your contribution!, great first issue!
Hi @lokesh1199, thanks for raising this issue.
To be fair we have just converted the implementation from sklearn, so I have never though too long about the actual code.
Looking through their issues I can see someone has raised a similar point to what you are doing here:
https://github.com/scikit-learn/scikit-learn/issues/25258
That said, after looking at the paper I can see two arguments:
* If we just look at the formula, the score is undefined (division by 0) for the cases where everything matches and it is therefore a completely arbitrary what the metric should return in this case
* Based on the analysis from the paper if we look at the limit, it does make sense that it should return -1/+1 in the corner cases

I therefore think it makes sense to change the implementation, just for the sake of result making better sense.

Resolving patch:
diff --git a/src/torchmetrics/functional/classification/matthews_corrcoef.py b/src/torchmetrics/functional/classification/matthews_corrcoef.py
--- a/src/torchmetrics/functional/classification/matthews_corrcoef.py
+++ b/src/torchmetrics/functional/classification/matthews_corrcoef.py
@@ -35,10 +35,21 @@
 
 
 def _matthews_corrcoef_reduce(confmat: Tensor) -> Tensor:
-    """Reduce an un-normalized confusion matrix of shape (n_classes, n_classes) into the matthews corrcoef score."""
+    """Reduce an un-normalized confusion matrix of shape (n_classes, n_classes) into the matthews corrcoef score.
+
+    See: https://bmcgenomics.biomedcentral.com/articles/10.1186/s12864-019-6413-7 for more info.
+    """
     # convert multilabel into binary
     confmat = confmat.sum(0) if confmat.ndim == 3 else confmat
 
+    if confmat.numel() == 4:  # binary case
+        tn, fp, fn, tp = confmat.reshape(-1)
+        if tp != 0 and tn == 0 and fp == 0 and fn == 0:
+            return torch.tensor(1.0, dtype=confmat.dtype, device=confmat.device)
+
+        if tp == 0 and tn != 0 and fp == 0 and fn == 0:
+            return torch.tensor(-1.0, dtype=confmat.dtype, device=confmat.device)
+
     tk = confmat.sum(dim=-1).float()
     pk = confmat.sum(dim=-2).float()
     c = torch.trace(confmat).float()
@@ -48,10 +59,22 @@ def _matthews_corrcoef_reduce(confmat: Tensor) -> Tensor:
     cov_ypyp = s**2 - sum(pk * pk)
     cov_ytyt = s**2 - sum(tk * tk)
 
+    numerator = cov_ytyp
     denom = cov_ypyp * cov_ytyt
-    if denom == 0:
+
+    if denom == 0 and confmat.numel() == 4:
+        if tp == 0 or tn == 0:
+            a = tp + tn
+
+        if fp == 0 or fn == 0:
+            b = fp + fn
+
+        eps = torch.tensor(torch.finfo(torch.float32).eps, dtype=torch.float32, device=confmat.device)
+        numerator = torch.sqrt(eps) * (a - b)
+        denom = torch.sqrt(2 * (a + b) * (a + eps) * (b + eps))
+    elif denom == 0:
         return torch.tensor(0, dtype=confmat.dtype, device=confmat.device)
-    return cov_ytyp / torch.sqrt(denom)
+    return numerator / torch.sqrt(denom)
 
 
 def binary_matthews_corrcoef(


Test patch:
diff --git a/tests/unittests/classification/test_matthews_corrcoef.py b/tests/unittests/classification/test_matthews_corrcoef.py
--- a/tests/unittests/classification/test_matthews_corrcoef.py
+++ b/tests/unittests/classification/test_matthews_corrcoef.py
@@ -52,7 +52,7 @@ def _sklearn_matthews_corrcoef_binary(preds, target, ignore_index=None):
 class TestBinaryMatthewsCorrCoef(MetricTester):
     """Test class for `BinaryMatthewsCorrCoef` metric."""
 
-    @pytest.mark.parametrize("ignore_index", [None, -1, 0])
+    @pytest.mark.parametrize("ignore_index", [None, -1])
     @pytest.mark.parametrize("ddp", [True, False])
     def test_binary_matthews_corrcoef(self, inputs, ddp, ignore_index):
         """Test class implementation of metric."""
@@ -71,7 +71,7 @@ def test_binary_matthews_corrcoef(self, inputs, ddp, ignore_index):
             },
         )
 
-    @pytest.mark.parametrize("ignore_index", [None, -1, 0])
+    @pytest.mark.parametrize("ignore_index", [None, -1])
     def test_binary_matthews_corrcoef_functional(self, inputs, ignore_index):
         """Test functional implementation of metric."""
         preds, target = inputs
@@ -234,7 +234,7 @@ def _sklearn_matthews_corrcoef_multilabel(preds, target, ignore_index=None):
 class TestMultilabelMatthewsCorrCoef(MetricTester):
     """Test class for `MultilabelMatthewsCorrCoef` metric."""
 
-    @pytest.mark.parametrize("ignore_index", [None, -1, 0])
+    @pytest.mark.parametrize("ignore_index", [None, -1])
     @pytest.mark.parametrize("ddp", [True, False])
     def test_multilabel_matthews_corrcoef(self, inputs, ddp, ignore_index):
         """Test class implementation of metric."""
@@ -253,7 +253,7 @@ def test_multilabel_matthews_corrcoef(self, inputs, ddp, ignore_index):
             },
         )
 
-    @pytest.mark.parametrize("ignore_index", [None, -1, 0])
+    @pytest.mark.parametrize("ignore_index", [None, -1])
     def test_multilabel_matthews_corrcoef_functional(self, inputs, ignore_index):
         """Test functional implementation of metric."""
         preds, target = inputs
@@ -316,3 +316,34 @@ def test_zero_case_in_multiclass():
     # Example where neither 1 or 2 is present in the target tensor
     out = multiclass_matthews_corrcoef(torch.tensor([0, 1, 2]), torch.tensor([0, 0, 0]), 3)
     assert out == 0.0
+
+
+@pytest.mark.parametrize(
+    ("metric_fn", "preds", "target", "expected"),
+    [
+        (binary_matthews_corrcoef, torch.zeros(10), torch.zeros(10), -1.0),
+        (binary_matthews_corrcoef, torch.ones(10), torch.ones(10), 1.0),
+        (
+            binary_matthews_corrcoef,
+            torch.tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
+            torch.tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]),
+            0.0,
+        ),
+        (
+            partial(multilabel_matthews_corrcoef, num_labels=NUM_CLASSES),
+            torch.zeros(10, NUM_CLASSES).long(),
+            torch.zeros(10, NUM_CLASSES).long(),
+            -1.0,
+        ),
+        (
+            partial(multilabel_matthews_corrcoef, num_labels=NUM_CLASSES),
+            torch.ones(10, NUM_CLASSES).long(),
+            torch.ones(10, NUM_CLASSES).long(),
+            1.0,
+        ),
+    ],
+)
+def test_corner_cases(metric_fn, preds, target, expected):
+    """Test the corner cases of perfect classifiers or completely random classifiers that they work as expected."""
+    out = metric_fn(preds, target)
+    assert out == expected