Repository: pytorch/ignite

Issue:
y and y_pred shape error in _BaseClassification._check_shape()
Hello,

In a single-label, multiple-class classification task, suppose my model gives the output logits `y_pred` of shape (batch, num_classes, 1) and the correct results are `y` of shape (batch, 1), which are the integer labels, then the _check_shape() gives an error
`y must have shape of (batch_size, ...) and y_pred must have "
                             "shape of (batch_size, num_categories, ...) or (batch_size, ...), "
                             "but given {} vs {}.`

which happens because the code squeezes the second dimension of `y`. See https://github.com/pytorch/ignite/blob/c92838c950699839c37f57bb50f79f84ecb71fc3/ignite/metrics/accuracy.py#L24

It seems like the shapes are legal according to the message. Am I doing something wrong here, or what is the intended usage?


Discussion:
@szhang104 thanks for the feedback! Yes, your input shapes seems to be not correctly covered by `Accuracy`.
A solution can be to provide `output_transform` function to get rid of these extra dimensions of size 1:
```python
def output_transform(output):
    y_pred, y = output
    return (y_pred[:, :, 0], y[:, 0])

acc = Accuracy(output_transform=output_transform)
```

HTH
Thanks for the comment.
Is there a plan to change the squeezing logic to accommodate user input shapes like reported?
@szhang104 yes, I think we need to review this logic

Resolving patch:
diff --git a/ignite/metrics/accuracy.py b/ignite/metrics/accuracy.py
--- a/ignite/metrics/accuracy.py
+++ b/ignite/metrics/accuracy.py
@@ -21,14 +21,6 @@ def reset(self):
     def _check_shape(self, output):
         y_pred, y = output
 
-        if y.ndimension() > 1 and y.shape[1] == 1:
-            # (N, 1, ...) -> (N, ...)
-            y = y.squeeze(dim=1)
-
-        if y_pred.ndimension() > 1 and y_pred.shape[1] == 1:
-            # (N, 1, ...) -> (N, ...)
-            y_pred = y_pred.squeeze(dim=1)
-
         if not (y.ndimension() == y_pred.ndimension() or y.ndimension() + 1 == y_pred.ndimension()):
             raise ValueError("y must have shape of (batch_size, ...) and y_pred must have "
                              "shape of (batch_size, num_categories, ...) or (batch_size, ...), "
@@ -46,20 +38,27 @@ def _check_shape(self, output):
         if self._is_multilabel and not (y.shape == y_pred.shape and y.ndimension() > 1 and y.shape[1] != 1):
             raise ValueError("y and y_pred must have same shape of (batch_size, num_categories, ...).")
 
-        return y_pred, y
+    def _check_binary_multilabel_cases(self, output):
+        y_pred, y = output
+
+        if not torch.equal(y, y ** 2):
+            raise ValueError("For binary cases, y must be comprised of 0's and 1's.")
+
+        if not torch.equal(y_pred, y_pred ** 2):
+            raise ValueError("For binary cases, y_pred must be comprised of 0's and 1's.")
 
     def _check_type(self, output):
         y_pred, y = output
 
         if y.ndimension() + 1 == y_pred.ndimension():
-            update_type = "multiclass"
             num_classes = y_pred.shape[1]
+            if num_classes == 1:
+                update_type = "binary"
+                self._check_binary_multilabel_cases((y_pred, y))
+            else:
+                update_type = "multiclass"
         elif y.ndimension() == y_pred.ndimension():
-            if not torch.equal(y, y ** 2):
-                raise ValueError("For binary cases, y must be comprised of 0's and 1's.")
-
-            if not torch.equal(y_pred, y_pred ** 2):
-                raise ValueError("For binary cases, y_pred must be comprised of 0's and 1's.")
+            self._check_binary_multilabel_cases((y_pred, y))
 
             if self._is_multilabel:
                 update_type = "multilabel"
@@ -122,12 +121,12 @@ def reset(self):
         super(Accuracy, self).reset()
 
     def update(self, output):
-
-        y_pred, y = self._check_shape(output)
+        y_pred, y = output
+        self._check_shape((y_pred, y))
         self._check_type((y_pred, y))
 
         if self._type == "binary":
-            correct = torch.eq(y_pred.type(y.type()), y).view(-1)
+            correct = torch.eq(y_pred.view(-1).to(y), y.view(-1))
         elif self._type == "multiclass":
             indices = torch.argmax(y_pred, dim=1)
             correct = torch.eq(indices, y).view(-1)
diff --git a/ignite/metrics/precision.py b/ignite/metrics/precision.py
--- a/ignite/metrics/precision.py
+++ b/ignite/metrics/precision.py
@@ -86,7 +86,8 @@ def __init__(self, output_transform=lambda x: x, average=False, is_multilabel=Fa
                                         average=average, is_multilabel=is_multilabel)
 
     def update(self, output):
-        y_pred, y = self._check_shape(output)
+        y_pred, y = output
+        self._check_shape(output)
         self._check_type((y_pred, y))
 
         if self._type == "binary":
diff --git a/ignite/metrics/recall.py b/ignite/metrics/recall.py
--- a/ignite/metrics/recall.py
+++ b/ignite/metrics/recall.py
@@ -58,7 +58,8 @@ def __init__(self, output_transform=lambda x: x, average=False, is_multilabel=Fa
                                      average=average, is_multilabel=is_multilabel)
 
     def update(self, output):
-        y_pred, y = self._check_shape(output)
+        y_pred, y = output
+        self._check_shape(output)
         self._check_type((y_pred, y))
 
         if self._type == "binary":


Test patch:
diff --git a/tests/ignite/metrics/test_accuracy.py b/tests/ignite/metrics/test_accuracy.py
--- a/tests/ignite/metrics/test_accuracy.py
+++ b/tests/ignite/metrics/test_accuracy.py
@@ -17,16 +17,17 @@ def test_no_update():
 def test__check_shape():
     acc = Accuracy()
 
-    # Check squeezed dimensions
-    y_pred, y = acc._check_shape((torch.randint(0, 2, size=(10, 1, 5, 6)).type(torch.LongTensor),
-                                  torch.randint(0, 2, size=(10, 5, 6)).type(torch.LongTensor)))
-    assert y_pred.shape == (10, 5, 6)
-    assert y.shape == (10, 5, 6)
+    with pytest.raises(ValueError):
+        acc._check_shape((torch.randint(0, 2, size=(10, 1, 5, 12)).type(torch.LongTensor),
+                          torch.randint(0, 2, size=(10, 5, 6)).type(torch.LongTensor)))
 
-    y_pred, y = acc._check_shape((torch.randint(0, 2, size=(10, 5, 6)).type(torch.LongTensor),
-                                  torch.randint(0, 2, size=(10, 1, 5, 6)).type(torch.LongTensor)))
-    assert y_pred.shape == (10, 5, 6)
-    assert y.shape == (10, 5, 6)
+    with pytest.raises(ValueError):
+        acc._check_shape((torch.randint(0, 2, size=(10, 1, 6)).type(torch.LongTensor),
+                          torch.randint(0, 2, size=(10, 5, 6)).type(torch.LongTensor)))
+
+    with pytest.raises(ValueError):
+        acc._check_shape((torch.randint(0, 2, size=(10, 1)).type(torch.LongTensor),
+                          torch.randint(0, 2, size=(10, 5)).type(torch.LongTensor)))
 
 
 def test_binary_wrong_inputs():
@@ -39,7 +40,7 @@ def test_binary_wrong_inputs():
 
     with pytest.raises(ValueError):
         # y_pred values are not thresholded to 0, 1 values
-        acc.update((torch.rand(10, 1),
+        acc.update((torch.rand(10,),
                     torch.randint(0, 2, size=(10,)).type(torch.LongTensor)))
 
     with pytest.raises(ValueError):
@@ -63,17 +64,7 @@ def test_binary_input_N():
     def _test():
         acc = Accuracy()
 
-        y_pred = torch.randint(0, 2, size=(10, 1)).type(torch.LongTensor)
-        y = torch.randint(0, 2, size=(10,)).type(torch.LongTensor)
-        acc.update((y_pred, y))
-        np_y = y.numpy().ravel()
-        np_y_pred = y_pred.numpy().ravel()
-        assert acc._type == 'binary'
-        assert isinstance(acc.compute(), float)
-        assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
-
-        acc.reset()
-        y_pred = torch.randint(0, 2, size=(10, )).type(torch.LongTensor)
+        y_pred = torch.randint(0, 2, size=(10,)).type(torch.LongTensor)
         y = torch.randint(0, 2, size=(10,)).type(torch.LongTensor)
         acc.update((y_pred, y))
         np_y = y.numpy().ravel()
@@ -176,6 +167,53 @@ def _test():
         assert isinstance(acc.compute(), float)
         assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
 
+        # Batched Updates
+        acc.reset()
+        y_pred = torch.randint(0, 2, size=(100, 8, 8)).type(torch.LongTensor)
+        y = torch.randint(0, 2, size=(100, 8, 8)).type(torch.LongTensor)
+
+        batch_size = 16
+        n_iters = y.shape[0] // batch_size + 1
+
+        for i in range(n_iters):
+            idx = i * batch_size
+            acc.update((y_pred[idx: idx + batch_size], y[idx: idx + batch_size]))
+
+        np_y = y.numpy().ravel()
+        np_y_pred = y_pred.numpy().ravel()
+        assert acc._type == 'binary'
+        assert isinstance(acc.compute(), float)
+        assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
+
+    # check multiple random inputs as random exact occurencies are rare
+    for _ in range(10):
+        _test()
+
+
+def test_binary_as_multiclass_input():
+    # Binary accuracy on input of shape (N, 1, ...)
+    def _test():
+        acc = Accuracy()
+
+        y_pred = torch.randint(0, 2, size=(4, 1)).type(torch.LongTensor)
+        y = torch.randint(0, 2, size=(4,)).type(torch.LongTensor)
+        acc.update((y_pred, y))
+        np_y = y.numpy().ravel()
+        np_y_pred = y_pred.numpy().ravel()
+        assert acc._type == 'binary'
+        assert isinstance(acc.compute(), float)
+        assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
+
+        acc.reset()
+        y_pred = torch.randint(0, 2, size=(4, 1, 12)).type(torch.LongTensor)
+        y = torch.randint(0, 2, size=(4, 12)).type(torch.LongTensor)
+        acc.update((y_pred, y))
+        np_y = y.numpy().ravel()
+        np_y_pred = y_pred.numpy().ravel()
+        assert acc._type == 'binary'
+        assert isinstance(acc.compute(), float)
+        assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
+
         # Batched Updates
         acc.reset()
         y_pred = torch.randint(0, 2, size=(100, 1, 8, 8)).type(torch.LongTensor)
@@ -232,9 +270,29 @@ def _test():
         assert isinstance(acc.compute(), float)
         assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
 
+        acc.reset()
+        y_pred = torch.rand(10, 10, 1)
+        y = torch.randint(0, 18, size=(10, 1)).type(torch.LongTensor)
+        acc.update((y_pred, y))
+        np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
+        np_y = y.numpy().ravel()
+        assert acc._type == 'multiclass'
+        assert isinstance(acc.compute(), float)
+        assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
+
+        acc.reset()
+        y_pred = torch.rand(10, 18)
+        y = torch.randint(0, 18, size=(10,)).type(torch.LongTensor)
+        acc.update((y_pred, y))
+        np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
+        np_y = y.numpy().ravel()
+        assert acc._type == 'multiclass'
+        assert isinstance(acc.compute(), float)
+        assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute())
+
         acc.reset()
         y_pred = torch.rand(4, 10)
-        y = torch.randint(0, 10, size=(4, 1)).type(torch.LongTensor)
+        y = torch.randint(0, 10, size=(4,)).type(torch.LongTensor)
         acc.update((y_pred, y))
         np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
         np_y = y.numpy().ravel()
@@ -245,7 +303,7 @@ def _test():
         # 2-classes
         acc.reset()
         y_pred = torch.rand(4, 2)
-        y = torch.randint(0, 2, size=(4, 1)).type(torch.LongTensor)
+        y = torch.randint(0, 2, size=(4,)).type(torch.LongTensor)
         acc.update((y_pred, y))
         np_y_pred = y_pred.numpy().argmax(axis=1).ravel()
         np_y = y.numpy().ravel()
diff --git a/tests/ignite/metrics/test_precision.py b/tests/ignite/metrics/test_precision.py
--- a/tests/ignite/metrics/test_precision.py
+++ b/tests/ignite/metrics/test_precision.py
@@ -32,7 +32,7 @@ def test_binary_wrong_inputs():
 
     with pytest.raises(ValueError):
         # y_pred values are not thresholded to 0, 1 values
-        pr.update((torch.rand(10, 1),
+        pr.update((torch.rand(10,),
                    torch.randint(0, 2, size=(10,)).type(torch.LongTensor)))
 
     with pytest.raises(ValueError):
@@ -56,7 +56,7 @@ def test_binary_input_N():
 
     def _test(average):
         pr = Precision(average=average)
-        y_pred = torch.randint(0, 2, size=(10, 1))
+        y_pred = torch.randint(0, 2, size=(10,))
         y = torch.randint(0, 2, size=(10,)).type(torch.LongTensor)
         pr.update((y_pred, y))
         np_y = y.numpy().ravel()
@@ -67,7 +67,7 @@ def _test(average):
         assert precision_score(np_y, np_y_pred, average='binary') == pytest.approx(pr_compute)
 
         pr.reset()
-        y_pred = torch.randint(0, 2, size=(10, ))
+        y_pred = torch.randint(0, 2, size=(10,))
         y = torch.randint(0, 2, size=(10,)).type(torch.LongTensor)
         pr.update((y_pred, y))
         np_y = y.numpy().ravel()
@@ -142,7 +142,7 @@ def _test(average):
 
         # Batched Updates
         pr.reset()
-        y_pred = torch.randint(0, 2, size=(100, 5))
+        y_pred = torch.randint(0, 2, size=(100, 1, 5))
         y = torch.randint(0, 2, size=(100, 1, 5)).type(torch.LongTensor)
 
         batch_size = 16
@@ -191,11 +191,10 @@ def _test(average):
         pr_compute = pr.compute() if average else pr.compute().numpy()
         assert precision_score(np_y, np_y_pred, average='binary') == pytest.approx(pr_compute)
 
-        pr = Precision(average=average)
         # Batched Updates
         pr.reset()
         y_pred = torch.randint(0, 2, size=(100, 12, 10))
-        y = torch.randint(0, 2, size=(100, 1, 12, 10)).type(torch.LongTensor)
+        y = torch.randint(0, 2, size=(100, 12, 10)).type(torch.LongTensor)
 
         batch_size = 16
         n_iters = y.shape[0] // batch_size + 1
@@ -278,7 +277,7 @@ def _test(average):
 
         pr.reset()
         y_pred = torch.rand(10, 4)
-        y = torch.randint(0, 4, size=(10, 1)).type(torch.LongTensor)
+        y = torch.randint(0, 4, size=(10,)).type(torch.LongTensor)
         pr.update((y_pred, y))
         num_classes = y_pred.shape[1]
         np_y_pred = y_pred.argmax(dim=1).numpy().ravel()
@@ -295,7 +294,7 @@ def _test(average):
         # 2-classes
         pr.reset()
         y_pred = torch.rand(10, 2)
-        y = torch.randint(0, 2, size=(10, 1)).type(torch.LongTensor)
+        y = torch.randint(0, 2, size=(10,)).type(torch.LongTensor)
         pr.update((y_pred, y))
         num_classes = y_pred.shape[1]
         np_y_pred = y_pred.argmax(dim=1).numpy().ravel()
diff --git a/tests/ignite/metrics/test_recall.py b/tests/ignite/metrics/test_recall.py
--- a/tests/ignite/metrics/test_recall.py
+++ b/tests/ignite/metrics/test_recall.py
@@ -56,7 +56,7 @@ def test_binary_input_N():
 
     def _test(average):
         re = Recall(average=average)
-        y_pred = torch.randint(0, 2, size=(10, 1))
+        y_pred = torch.randint(0, 2, size=(10,))
         y = torch.randint(0, 2, size=(10,)).type(torch.LongTensor)
         re.update((y_pred, y))
         np_y = y.numpy().ravel()
@@ -144,7 +144,7 @@ def _test(average):
         # Batched Updates
         re.reset()
         y_pred = torch.randint(0, 2, size=(100, 5))
-        y = torch.randint(0, 2, size=(100, 1, 5)).type(torch.LongTensor)
+        y = torch.randint(0, 2, size=(100, 5)).type(torch.LongTensor)
 
         batch_size = 16
         n_iters = y.shape[0] // batch_size + 1
@@ -192,11 +192,10 @@ def _test(average):
         re_compute = re.compute() if average else re.compute().numpy()
         assert recall_score(np_y, np_y_pred, average='binary') == pytest.approx(re_compute)
 
-        re = Recall(average=average)
         # Batched Updates
         re.reset()
         y_pred = torch.randint(0, 2, size=(100, 12, 10))
-        y = torch.randint(0, 2, size=(100, 1, 12, 10)).type(torch.LongTensor)
+        y = torch.randint(0, 2, size=(100, 12, 10)).type(torch.LongTensor)
 
         batch_size = 16
         n_iters = y.shape[0] // batch_size + 1
@@ -279,7 +278,7 @@ def _test(average):
 
         re.reset()
         y_pred = torch.rand(10, 4)
-        y = torch.randint(0, 4, size=(10, 1)).type(torch.LongTensor)
+        y = torch.randint(0, 4, size=(10,)).type(torch.LongTensor)
         re.update((y_pred, y))
         num_classes = y_pred.shape[1]
         np_y_pred = y_pred.argmax(dim=1).numpy().ravel()
@@ -296,7 +295,7 @@ def _test(average):
         # 2-classes
         re.reset()
         y_pred = torch.rand(10, 2)
-        y = torch.randint(0, 2, size=(10, 1)).type(torch.LongTensor)
+        y = torch.randint(0, 2, size=(10,)).type(torch.LongTensor)
         re.update((y_pred, y))
         num_classes = y_pred.shape[1]
         np_y_pred = y_pred.argmax(dim=1).numpy().ravel()
@@ -692,7 +691,7 @@ def _test(average):
         y = torch.ones(4).type(torch.LongTensor)
         re.update((y_pred, y))
 
-        y_pred = torch.zeros(4, 1)
+        y_pred = torch.zeros(4,)
         y = torch.ones(4).type(torch.LongTensor)
 
         with pytest.raises(RuntimeError):