Repository: Lightning-AI/torchmetrics

Issue:
MinMaxMetric for wrapping other metrics
## 🚀 Feature

### Motivation

- `MinMaxMetric` is a metric that simply wraps another metric (e.g.`val_acc`) and creates a new metric that tracks the `min`, `max`  or both values of `val_acc`. 

### Pitch

- I personally use it to quickly see the `max_val_acc` of a complete experiment in TensorBoard (instead of going through the graph manually to find the max value) but I can see other usecases as well. 
- It was discussed in the PL Slack [here](https://pytorch-lightning.slack.com/archives/CU5PF2E2J/p1614602988020800) and clearly resonated with more users

### Additional context
- Happy to submit a PR for this feature, as I do have already a (incomplete) `MaxMetric` code implemented [here](https://gist.github.com/janhenriklambrechts/67a0faf5dc05d4e8d4d16973d1c03929)


Discussion:
Hi! thanks for your contribution!, great first issue!
Hi @janhenriklambrechts are you still interested in sending a PR for this feature?
Absolutely, you can assign it to me. 
Think I will be able to start working on it in a few days :)
@janhenriklambrechts Nice :]
Feel free to ping if you have any question (here or on slack).
If you need inspiration, then I have a open PR for another wrapper metric (https://github.com/PyTorchLightning/metrics/pull/101) for creating bootstrapped versions of metrics.
Thank you so much for the inspiration! Very useful!
This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.

Will only be able to work on it from beginning of July onward @SkafteNicki - is that OK?
> Will only be able to work on it from beginning of July onward @SkafteNicki - is that OK?

that is great! looking forward :rabbit: 
@janhenriklambrechts any updates on this? :]
Hi @SkafteNicki, completely lost track of this - I am starting this weekend! Thanks for pinging me :)
@janhenriklambrechts no problem. Let me know if you need any help :]

Resolving patch:
diff --git a/torchmetrics/__init__.py b/torchmetrics/__init__.py
--- a/torchmetrics/__init__.py
+++ b/torchmetrics/__init__.py
@@ -75,7 +75,7 @@
     ROUGEScore,
     SacreBLEUScore,
 )
-from torchmetrics.wrappers import BootStrapper, MetricTracker, MultioutputWrapper  # noqa: E402
+from torchmetrics.wrappers import BootStrapper, MetricTracker, MinMaxMetric, MultioutputWrapper  # noqa: E402
 
 __all__ = [
     "functional",
@@ -117,6 +117,7 @@
     "Metric",
     "MetricCollection",
     "MetricTracker",
+    "MinMaxMetric",
     "MinMetric",
     "MultioutputWrapper",
     "PearsonCorrcoef",
diff --git a/torchmetrics/wrappers/__init__.py b/torchmetrics/wrappers/__init__.py
--- a/torchmetrics/wrappers/__init__.py
+++ b/torchmetrics/wrappers/__init__.py
@@ -12,5 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from torchmetrics.wrappers.bootstrapping import BootStrapper  # noqa: F401
+from torchmetrics.wrappers.minmax import MinMaxMetric  # noqa: F401
 from torchmetrics.wrappers.multioutput import MultioutputWrapper  # noqa: F401
 from torchmetrics.wrappers.tracker import MetricTracker  # noqa: F401
diff --git a/torchmetrics/wrappers/minmax.py b/torchmetrics/wrappers/minmax.py
new file mode 100644
--- /dev/null
+++ b/torchmetrics/wrappers/minmax.py
@@ -0,0 +1,119 @@
+# Copyright The PyTorch Lightning team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Callable, Dict, Optional, Union
+
+import torch
+from torch import Tensor
+
+from torchmetrics.metric import Metric
+
+
+class MinMaxMetric(Metric):
+    """Wrapper Metric that tracks both the minimum and maximum of a scalar/tensor across an experiment. The min/max
+    value will be updated each time `.compute` is called.
+
+    Args:
+        base_metric:
+            The metric of which you want to keep track of its maximum and minimum values.
+        compute_on_step:
+            Forward only calls ``update()`` and return None if this is set to False. default: True
+        dist_sync_on_step:
+            Synchronize metric state across processes at each ``forward()``
+            before returning the value at the step. default: False
+        process_group:
+            Specify the process group on which synchronization is called. default: None (which selects the entire world)
+        dist_sync_fn:
+            Callback that performs the allgather operation on the metric state. When ``None``, DDP
+            will be used to perform the allgather
+
+    Raises:
+        ValueError
+            If ``base_metric` argument is not an subclasses instance of ``torchmetrics.Metric``
+
+    Example::
+        >>> import torch
+        >>> from torchmetrics import Accuracy
+        >>> from pprint import pprint
+        >>> base_metric = Accuracy()
+        >>> minmax_metric = MinMaxMetric(base_metric)
+        >>> preds_1 = torch.Tensor([[0.1, 0.9], [0.2, 0.8]])
+        >>> preds_2 = torch.Tensor([[0.9, 0.1], [0.2, 0.8]])
+        >>> labels = torch.Tensor([[0, 1], [0, 1]]).long()
+        >>> pprint(minmax_metric(preds_1, labels))
+        {'max': tensor(1.), 'min': tensor(1.), 'raw': tensor(1.)}
+        >>> pprint(minmax_metric.compute())
+        {'max': tensor(1.), 'min': tensor(1.), 'raw': tensor(1.)}
+        >>> minmax_metric.update(preds_2, labels)
+        >>> pprint(minmax_metric.compute())
+        {'max': tensor(1.), 'min': tensor(0.7500), 'raw': tensor(0.7500)}
+    """
+
+    min_val: Tensor
+    max_val: Tensor
+
+    def __init__(
+        self,
+        base_metric: Metric,
+        compute_on_step: bool = True,
+        dist_sync_on_step: bool = False,
+        process_group: Optional[Any] = None,
+        dist_sync_fn: Callable = None,
+    ) -> None:
+        super().__init__(
+            compute_on_step=compute_on_step,
+            dist_sync_on_step=dist_sync_on_step,
+            process_group=process_group,
+            dist_sync_fn=dist_sync_fn,
+        )
+        if not isinstance(base_metric, Metric):
+            raise ValueError(
+                f"Expected base metric to be an instance of `torchmetrics.Metric` but received {base_metric}"
+            )
+        self._base_metric = base_metric
+        self.register_buffer("min_val", torch.tensor(float("inf")))
+        self.register_buffer("max_val", torch.tensor(float("-inf")))
+
+    def update(self, *args: Any, **kwargs: Any) -> None:  # type: ignore
+        """Updates the underlying metric."""
+        self._base_metric.update(*args, **kwargs)
+
+    def compute(self) -> Dict[str, Tensor]:  # type: ignore
+        """Computes the underlying metric as well as max and min values for this metric.
+
+        Returns a dictionary that consists of the computed value (``raw``), as well as the minimum (``min``) and maximum
+        (``max``) values.
+        """
+        val = self._base_metric.compute()
+        if not self._is_suitable_val(val):
+            raise RuntimeError(
+                f"Returned value from base metric should be a scalar (int, float or tensor of size 1, but got {val}"
+            )
+        self.max_val = val if self.max_val < val else self.max_val
+        self.min_val = val if self.min_val > val else self.min_val
+        return {"raw": val, "max": self.max_val, "min": self.min_val}
+
+    def reset(self) -> None:
+        """Sets ``max_val`` and ``min_val`` to the initialization bounds and resets the base metric."""
+        super().reset()
+        self._base_metric.reset()
+
+    @staticmethod
+    def _is_suitable_val(val: Union[int, float, Tensor]) -> bool:
+        """Utility function that checks whether min/max value."""
+        if isinstance(val, (int, float)):
+            return True
+        if isinstance(val, Tensor):
+            return val.numel() == 1
+        return False


Test patch:
diff --git a/tests/wrappers/test_minmax.py b/tests/wrappers/test_minmax.py
new file mode 100644
--- /dev/null
+++ b/tests/wrappers/test_minmax.py
@@ -0,0 +1,119 @@
+from copy import deepcopy
+from functools import partial
+
+import pytest
+import torch
+
+from tests.helpers import seed_all
+from tests.helpers.testers import BATCH_SIZE, NUM_BATCHES, NUM_CLASSES, MetricTester
+from torchmetrics import Accuracy, ConfusionMatrix, MeanSquaredError
+from torchmetrics.wrappers import MinMaxMetric
+
+seed_all(42)
+
+
+class TestingMinMaxMetric(MinMaxMetric):
+    """wrap metric to fit testing framework."""
+
+    def compute(self):
+        """instead of returning dict, return as list."""
+        output_dict = super().compute()
+        return [output_dict["raw"], output_dict["min"], output_dict["max"]]
+
+    def forward(self, *args, **kwargs):
+        self.update(*args, **kwargs)
+        return self.compute()
+
+
+def compare_fn(preds, target, base_fn):
+    """comparing function for minmax wrapper."""
+    min, max = 1e6, -1e6  # pick some very large numbers for comparing
+    for i in range(NUM_BATCHES):
+        val = base_fn(preds[: (i + 1) * BATCH_SIZE], target[: (i + 1) * BATCH_SIZE]).cpu().numpy()
+        min = min if min < val else val
+        max = max if max > val else val
+    raw = base_fn(preds, target)
+    return [raw.cpu().numpy(), min, max]
+
+
+def compare_fn_ddp(preds, target, base_fn):
+    min, max = 1e6, -1e6  # pick some very large numbers for comparing
+    for i, j in zip(range(0, NUM_BATCHES, 2), range(1, NUM_BATCHES, 2)):
+        p = torch.cat([preds[i * BATCH_SIZE : (i + 1) * BATCH_SIZE], preds[j * BATCH_SIZE : (j + 1) * BATCH_SIZE]])
+        t = torch.cat([target[i * BATCH_SIZE : (i + 1) * BATCH_SIZE], target[j * BATCH_SIZE : (j + 1) * BATCH_SIZE]])
+        base_fn.update(p, t)
+        val = base_fn.compute().cpu().numpy()
+        min = min if min < val else val
+        max = max if max > val else val
+    raw = base_fn(preds, target)
+    return [raw.cpu().numpy(), min, max]
+
+
+@pytest.mark.parametrize(
+    "preds, target, base_metric",
+    [
+        (
+            torch.rand(NUM_BATCHES, BATCH_SIZE, NUM_CLASSES).softmax(dim=-1),
+            torch.randint(NUM_CLASSES, (NUM_BATCHES, BATCH_SIZE)),
+            Accuracy(num_classes=NUM_CLASSES),
+        ),
+        (torch.randn(NUM_BATCHES, BATCH_SIZE), torch.randn(NUM_BATCHES, BATCH_SIZE), MeanSquaredError()),
+    ],
+)
+class TestMultioutputWrapper(MetricTester):
+    """Test the MinMaxMetric wrapper works as expected."""
+
+    @pytest.mark.parametrize("ddp", [True, False])
+    def test_multioutput_wrapper(self, preds, target, base_metric, ddp):
+        self.run_class_metric_test(
+            ddp,
+            preds,
+            target,
+            TestingMinMaxMetric,
+            partial(compare_fn_ddp if ddp else compare_fn, base_fn=deepcopy(base_metric)),
+            dist_sync_on_step=False,
+            metric_args=dict(base_metric=base_metric),
+            check_batch=False,
+            check_scriptable=False,
+        )
+
+
+@pytest.mark.parametrize(
+    "preds, labels, raws, maxs, mins",
+    [
+        (
+            ([[0.9, 0.1], [0.2, 0.8]], [[0.1, 0.9], [0.2, 0.8]], [[0.1, 0.9], [0.8, 0.2]]),
+            [[0, 1], [0, 1]],
+            (0.5, 1.0, 0.5),
+            (0.5, 1.0, 1.0),
+            (0.5, 0.5, 0.5),
+        )
+    ],
+)
+def test_basic_example(preds, labels, raws, maxs, mins) -> None:
+    """tests that both min and max versions of MinMaxMetric operate correctly after calling compute."""
+    acc = Accuracy()
+    min_max_acc = MinMaxMetric(acc)
+    labels = torch.Tensor(labels).long()
+
+    for i in range(3):
+        preds_ = torch.Tensor(preds[i])
+        min_max_acc(preds_, labels)
+        acc = min_max_acc.compute()
+        assert acc["raw"] == raws[i]
+        assert acc["max"] == maxs[i]
+        assert acc["min"] == mins[i]
+
+
+def test_no_base_metric() -> None:
+    """tests that ValueError is raised when no base_metric is passed."""
+    with pytest.raises(ValueError, match=r"Expected base metric to be an instance .*"):
+        MinMaxMetric([])
+
+
+def test_no_scalar_compute() -> None:
+    """tests that an assertion error is thrown if the wrapped basemetric gives a non-scalar on compute."""
+    min_max_nsm = MinMaxMetric(ConfusionMatrix(num_classes=2))
+
+    with pytest.raises(RuntimeError, match=r"Returned value from base metric should be a scalar .*"):
+        min_max_nsm.compute()