Repository: Lightning-AI/torchmetrics

Issue:
Metric Collection Tracker Comptability
## 🐛 Bug
Hi , 
In the last version , you added the option to combine MetricTracker with MetricCollection.
When I used it , I tried to use the "best_metric" method for MetricTracker. Unfortunately , When I have metrics which doesn't have a best metric option (like confusion matrix) in the MetricCollection object , "best_metric" method return an error.
 
I think that as this function should be general , you should have support for this case as well 

#### Code sample
```py
from torchmetrics import MetricTracker, MetricCollection,Accuracy,Recall,ConfusionMatrix
_ = torch.manual_seed(42)
tracker = MetricTracker(MetricCollection([Accuracy(),Recall(),ConfusionMatrix(num_classes=2)]))
for epoch in range(5):
    tracker.increment()
    for batch_idx in range(5):
        preds, target = torch.randint(0,2,(100,)), torch.randint(0, 2, (100,))
        tracker.update(preds, target)
    print(f"current stats={tracker.compute()}")
from pprint import pprint
best_res, which_epoch = tracker.best_metric(return_step=True)
pprint(best_res)
```


That's Result in this Error:
```
Traceback (most recent call last):
  File "<input>", line 16, in <module>
  File "/Users/*****/.pyenv/versions/*****/lib/python3.7/site-packages/torchmetrics/wrappers/tracker.py", line 177, in best_metric
    idx[k], best[k] = out[0].item(), out[1].item()
ValueError: only one element tensors can be converted to Python scalars
```

### Expected behavior
The solution is to wrap the iteration over the metrics in MetricCollection with a try and except and just ignore the non-computable metrics 
from the original code of MetricTracker.best_metric (line 174)
```py
            for i, (k, v) in enumerate(res.items()):
                try:
                    fn = torch.max if maximize[i] else torch.min
                    out = fn(v, 0)
                    idx[k], best[k] = out[0].item(), out[1].item()
                except:
                    some predefined message for that case 
# after running over al metrics if there no metric that has a "score"              
            if len(idx)==0:
                   some predefined error 
                   return []
```




### Environment

- OS (Linux):
- Python & PyTorch Version (3.7.5, 1.10)
- TorchMetrics version 0.8.0.dev0




Discussion:

Resolving patch:
diff --git a/torchmetrics/wrappers/tracker.py b/torchmetrics/wrappers/tracker.py
--- a/torchmetrics/wrappers/tracker.py
+++ b/torchmetrics/wrappers/tracker.py
@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import warnings
 from copy import deepcopy
 from typing import Any, Dict, List, Tuple, Union
 
@@ -152,7 +153,14 @@ def reset_all(self) -> None:
 
     def best_metric(
         self, return_step: bool = False
-    ) -> Union[float, Tuple[int, float], Dict[str, float], Tuple[Dict[str, int], Dict[str, float]]]:
+    ) -> Union[
+        None,
+        float,
+        Tuple[int, float],
+        Tuple[None, None],
+        Dict[str, Union[float, None]],
+        Tuple[Dict[str, Union[int, None]], Dict[str, Union[float, None]]],
+    ]:
         """Returns the highest metric out of all tracked.
 
         Args:
@@ -163,18 +171,39 @@ def best_metric(
         """
         if isinstance(self._base_metric, Metric):
             fn = torch.max if self.maximize else torch.min
-            idx, best = fn(self.compute_all(), 0)
-            if return_step:
-                return idx.item(), best.item()
-            return best.item()
-        else:
+            try:
+                idx, best = fn(self.compute_all(), 0)
+                if return_step:
+                    return idx.item(), best.item()
+                return best.item()
+            except ValueError as error:
+                warnings.warn(
+                    f"Encountered the following error when trying to get the best metric: {error}"
+                    "this is probably due to the 'best' not being defined for this metric."
+                    "Returning `None` instead.",
+                    UserWarning,
+                )
+                if return_step:
+                    return None, None
+                return None
+
+        else:  # this is a metric collection
             res = self.compute_all()
             maximize = self.maximize if isinstance(self.maximize, list) else len(res) * [self.maximize]
             idx, best = {}, {}
             for i, (k, v) in enumerate(res.items()):
-                fn = torch.max if maximize[i] else torch.min
-                out = fn(v, 0)
-                idx[k], best[k] = out[0].item(), out[1].item()
+                try:
+                    fn = torch.max if maximize[i] else torch.min
+                    out = fn(v, 0)
+                    idx[k], best[k] = out[0].item(), out[1].item()
+                except ValueError as error:
+                    warnings.warn(
+                        f"Encountered the following error when trying to get the best metric for metric {k}:"
+                        f"{error} this is probably due to the 'best' not being defined for this metric."
+                        "Returning `None` instead.",
+                        UserWarning,
+                    )
+                    idx[k], best[k] = None, None
 
             if return_step:
                 return idx, best


Test patch:
diff --git a/tests/wrappers/test_tracker.py b/tests/wrappers/test_tracker.py
--- a/tests/wrappers/test_tracker.py
+++ b/tests/wrappers/test_tracker.py
@@ -16,7 +16,15 @@
 import torch
 
 from tests.helpers import seed_all
-from torchmetrics import Accuracy, MeanAbsoluteError, MeanSquaredError, MetricCollection, Precision, Recall
+from torchmetrics import (
+    Accuracy,
+    ConfusionMatrix,
+    MeanAbsoluteError,
+    MeanSquaredError,
+    MetricCollection,
+    Precision,
+    Recall,
+)
 from torchmetrics.wrappers import MetricTracker
 
 seed_all(42)
@@ -117,3 +125,40 @@ def test_tracker(base_metric, metric_input, maximize):
     else:
         assert val != 0.0
         assert idx in list(range(5))
+
+
+@pytest.mark.parametrize(
+    "base_metric",
+    [
+        ConfusionMatrix(3),
+        MetricCollection([ConfusionMatrix(3), Accuracy(3)]),
+    ],
+)
+def test_best_metric_for_not_well_defined_metric_collection(base_metric):
+    """Test that if user tries to compute the best metric for a metric that does not have a well defined best, we
+    throw an warning and return None."""
+    tracker = MetricTracker(base_metric)
+    for _ in range(3):
+        tracker.increment()
+        for _ in range(5):
+            tracker.update(torch.randint(3, (10,)), torch.randint(3, (10,)))
+
+    with pytest.warns(UserWarning, match="Encountered the following error when trying to get the best metric.*"):
+        best = tracker.best_metric()
+        if isinstance(best, dict):
+            assert best["Accuracy"] is not None
+            assert best["ConfusionMatrix"] is None
+        else:
+            assert best is None
+
+    with pytest.warns(UserWarning, match="Encountered the following error when trying to get the best metric.*"):
+        idx, best = tracker.best_metric(return_step=True)
+
+        if isinstance(best, dict):
+            assert best["Accuracy"] is not None
+            assert best["ConfusionMatrix"] is None
+            assert idx["Accuracy"] is not None
+            assert idx["ConfusionMatrix"] is None
+        else:
+            assert best is None
+            assert idx is None