metrics.classification.ConfusionMatrix accumulates counts
🐛 Bug
I recently switsched form pl version 1.1.8 to 1.2.1 and now metrics.classification.ConfusionMatrix accumulates counts over epochs instead of updating the confusion matrix.
These are the confusion matrices printed in each validation_epoch_end over 5 epochs using the code below.

[[92. 10.]
[ 0.  0.]]
[[184.  18.]
[  0.   0.]]
[[276.  26.]
[  0.   0.]]
[[368.  34.]
[  0.   0.]]
[[460.  42.]
[  0.   0.]]

Please reproduce using the BoringModel
To Reproduce
import os
from numpy.lib.npyio import save

import torch
from torch.random import seed
from torch.utils.data import Dataset

from pl_examples import cli_lightning_logo
from pytorch_lightning import LightningModule, Trainer, seed_everything
from pytorch_lightning.loggers import CSVLogger

from pytorch_lightning import metrics
from pytorch_lightning.metrics.utils import to_categorical


class RandomDataset(Dataset):
    """
    >>> RandomDataset(size=10, length=20)  # doctest: +ELLIPSIS
    <...bug_report_model.RandomDataset object at ...>
    """

    def __init__(self, size, length):
        self.len = length
        self.data = torch.randn(length, size)

    def __getitem__(self, index):
        return self.data[index]

    def __len__(self):
        return self.len


class BoringModel(LightningModule):
    """
    >>> BoringModel()  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
    BoringModel(
      (layer): Linear(...)
    )
    """

    def __init__(self):
        """
        Testing PL Module

        Use as follows:
        - subclass
        - modify the behavior for what you want

        class TestModel(BaseTestModel):
            def training_step(...):
                # do your own thing

        or:

        model = BaseTestModel()
        model.training_epoch_end = None

        """
        super().__init__()
        self.layer = torch.nn.Linear(32, 2)

    def forward(self, x):
        return self.layer(x)

    def loss(self, batch, prediction):
        # An arbitrary loss to have a loss that updates the model weights during `Trainer.fit` calls
        return torch.nn.functional.mse_loss(prediction, torch.ones_like(prediction))

    def step(self, x):
        x = self.layer(x)
        out = torch.nn.functional.mse_loss(x, torch.ones_like(x))
        return out

    def training_step(self, batch, batch_idx):
        output = self.layer(batch)
        loss = self.loss(batch, output)
        return {"loss": loss}

    def training_step_end(self, training_step_outputs):
        return training_step_outputs

    def training_epoch_end(self, outputs) -> None:
        torch.stack([x["loss"] for x in outputs]).mean()

    def validation_step(self, batch, batch_idx):
        output = self.layer(batch)
        loss = self.loss(batch, output)
        return {"x": loss}

    def validation_epoch_end(self, outputs) -> None:
        torch.stack([x['x'] for x in outputs]).mean()

    def test_step(self, batch, batch_idx):
        output = self.layer(batch)
        loss = self.loss(batch, output)
        return {"y": loss}

    def test_epoch_end(self, outputs) -> None:
        torch.stack([x["y"] for x in outputs]).mean()

    def configure_optimizers(self):
        optimizer = torch.optim.SGD(self.layer.parameters(), lr=0.1)
        lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1)
        return [optimizer], [lr_scheduler]


#  NOTE: If you are using a cmd line to run your script,
#  provide the cmd line as below.
#  opt = "--max_epochs 1 --limit_train_batches 1".split(" ")
#  parser = ArgumentParser()
#  args = parser.parse_args(opt)

def test_run():

    class TestModel(BoringModel):

        def __init__(self):
            super().__init__()

            self.valid_confmat = metrics.classification.ConfusionMatrix(
                num_classes=2,
                compute_on_step=False
            )

        @staticmethod
        def _y_to_cat(logits, targets):
            y_pred = to_categorical(
                tensor=torch.sigmoid(logits),
                argmax_dim=1
            )
            y_true = to_categorical(
                tensor=targets,
                argmax_dim=1
            )
            return y_pred, y_true

        def validation_step(self, batch, batch_idx):
            output = self.layer(batch)
            loss = self.loss(batch, output)

            # update metrics
            _preds, _targets = self._y_to_cat(
                logits=output,
                targets=torch.ones_like(output)
            )

            self.valid_confmat(preds=_preds, target=_targets)

            return {"loss": loss, "batch_size": batch.shape[0]}

        def validation_epoch_end(self, outputs):
            cm = self.valid_confmat.compute()
            print(cm.cpu().detach().numpy())

    # fake data
    train_data = torch.utils.data.DataLoader(RandomDataset(32, 100))
    val_data = torch.utils.data.DataLoader(RandomDataset(32, 100))
    test_data = torch.utils.data.DataLoader(RandomDataset(32, 100))

    # logger
    csvlogger = CSVLogger(
        save_dir=os.getcwd()
    )
    # model
    seed_everything(0)
    model = TestModel()
    trainer = Trainer(
        default_root_dir=os.getcwd(),
        max_epochs=5,
        weights_summary=None,
        logger=csvlogger,
        deterministic=True
    )
    trainer.fit(model, train_data, val_data)
    trainer.test(test_dataloaders=test_data)


if __name__ == '__main__':
    cli_lightning_logo()
    test_run()
Expected behavior
Confusion Matrix should be updated for each epoch, counts should not accumulate.
Environment
/home/user/miniconda/lib/python3.8/site-packages/torch/cuda/init.py:52: UserWarning: CUDA initialization: Found no NVIDIA driver on your system. Please check that you have an NVIDIA GPU and installed a driver from http://www.nvidia.com/Download/index.aspx (Triggered internally at  /pytorch/c10/cuda/CUDAFunctions.cpp:100.)
return torch._C._cuda_getDeviceCount() > 0

CUDA:
- GPU:
- available:         False
- version:           10.2
Packages:
- numpy:             1.19.2
- pyTorch_debug:     False
- pyTorch_version:   1.7.1
- pytorch-lightning: 1.2.1
- tqdm:              4.51.0
System:
- OS:                Linux
- architecture:
- 64bit
-
- processor:         x86_64
- python:            3.8.5
- version:           #1 SMP Tue Jun 23 12:58:10 UTC 2020