Repository: Lightning-AI/torchmetrics

Issue:
Calibration error does not support negative logits
Hi there,

It appears that `CalibrationError` currently does not support some of the input data types mentioned in [this table](https://torchmetrics.readthedocs.io/en/stable/pages/classification.html#input-types) from the docs. In particular, it seems to break when fed with 

- `preds=logits` where `logits` is a `(N, C)` float32 tensor with potentially negative values.
- `preds=predictions` where `predictions` is a `(N,)` int tensor with the predicted labels.

It still works with softmax-ed logits (`preds=logits.softmax(-1)`) or, generally, with  and `(N,)`-dimensional float32 tensors (i.e. the binary input in the input data table mentioned above).

To reproduce:

```python
N, C = 10, 3
targets = torch.randint(C, (N,))

# (N, C) non-negative: works
preds = torch.rand((N, C))  # non-negative
CalibrationError()(preds=preds, target=targets)

# (N, C) potentially negative: fails
preds = torch.randn((N, C))  # potetially negative
CalibrationError()(preds=preds, target=targets)

# (N,) int type: fails
CalibrationError()(preds=targets, target=targets)

# (N,) float type non-negative: works
preds = torch.rand((N,))  # binary non-negative
CalibrationError()(preds=preds, target=targets)

# (N,) float type potentially negative: fails
preds = torch.randn((N,))  # binary potentially negative
CalibrationError()(preds=preds, target=targets)
```

Torchmetrics: v0.8
Pytorch: v1.11


Discussion:
Hi! thanks for your contribution!, great first issue!

Resolving patch:
diff --git a/torchmetrics/functional/classification/calibration_error.py b/torchmetrics/functional/classification/calibration_error.py
--- a/torchmetrics/functional/classification/calibration_error.py
+++ b/torchmetrics/functional/classification/calibration_error.py
@@ -143,8 +143,12 @@ def _ce_update(preds: Tensor, target: Tensor) -> Tuple[Tensor, Tensor]:
     _, _, mode = _input_format_classification(preds, target)
 
     if mode == DataType.BINARY:
+        if not ((0 <= preds) * (preds <= 1)).all():
+            preds = preds.sigmoid()
         confidences, accuracies = preds, target
     elif mode == DataType.MULTICLASS:
+        if not ((0 <= preds) * (preds <= 1)).all():
+            preds = preds.softmax(dim=1)
         confidences, predictions = preds.max(dim=1)
         accuracies = predictions.eq(target)
     elif mode == DataType.MULTIDIM_MULTICLASS:


Test patch:
diff --git a/tests/classification/test_calibration_error.py b/tests/classification/test_calibration_error.py
--- a/tests/classification/test_calibration_error.py
+++ b/tests/classification/test_calibration_error.py
@@ -3,8 +3,10 @@
 
 import numpy as np
 import pytest
+from scipy.special import softmax as _softmax
 
-from tests.classification.inputs import _input_binary_prob
+from tests.classification.inputs import _input_binary_logits, _input_binary_prob
+from tests.classification.inputs import _input_multiclass_logits as _input_mcls_logits
 from tests.classification.inputs import _input_multiclass_prob as _input_mcls_prob
 from tests.classification.inputs import _input_multidim_multiclass_prob as _input_mdmc_prob
 from tests.classification.inputs import _input_multilabel_prob as _input_mlb_prob
@@ -24,8 +26,12 @@
 def _sk_calibration(preds, target, n_bins, norm, debias=False):
     _, _, mode = _input_format_classification(preds, target, threshold=THRESHOLD)
     sk_preds, sk_target = preds.numpy(), target.numpy()
-
+    if mode == DataType.BINARY:
+        if not np.logical_and(0 <= sk_preds, sk_preds <= 1).all():
+            sk_preds = 1.0 / (1 + np.exp(-sk_preds))  # sigmoid transform
     if mode == DataType.MULTICLASS:
+        if not np.logical_and(0 <= sk_preds, sk_preds <= 1).all():
+            sk_preds = _softmax(sk_preds, axis=1)
         # binary label is whether or not the predicted class is correct
         sk_target = np.equal(np.argmax(sk_preds, axis=1), sk_target)
         sk_preds = np.max(sk_preds, axis=1)
@@ -46,7 +52,9 @@ def _sk_calibration(preds, target, n_bins, norm, debias=False):
     "preds, target",
     [
         (_input_binary_prob.preds, _input_binary_prob.target),
+        (_input_binary_logits.preds, _input_binary_logits.target),
         (_input_mcls_prob.preds, _input_mcls_prob.target),
+        (_input_mcls_logits.preds, _input_mcls_logits.target),
         (_input_mdmc_prob.preds, _input_mdmc_prob.target),
     ],
 )