Repository: pytorch/ignite

Issue:
Remove `activation` argument from AveragePrecision and ROC_AUC
I propose to remove `activation` argument from `AveragePrecision` and `ROC_AUC`:
https://github.com/pytorch/ignite/blob/862ab0073e461a32bf5b53fa015f88c143ae9079/ignite/contrib/metrics/roc_auc.py#L23-L28
to be coherent with `Accuracy` et friends and the usage of `output_transform` to apply the binarization/activation :
https://github.com/pytorch/ignite/blob/862ab0073e461a32bf5b53fa015f88c143ae9079/ignite/metrics/accuracy.py#L81-L93




Discussion:
@vfdev-5 I agree, this would be better aligned with the other metrics. Is it ok if I work on this issue?
@anmolsjoshi yes go ahead, it is not yet assigned.

Resolving patch:
diff --git a/ignite/contrib/metrics/average_precision.py b/ignite/contrib/metrics/average_precision.py
--- a/ignite/contrib/metrics/average_precision.py
+++ b/ignite/contrib/metrics/average_precision.py
@@ -1,16 +1,13 @@
-from functools import partial
 from ignite.metrics import EpochMetric
 
 
-def average_precision_compute_fn(y_preds, y_targets, activation=None):
+def average_precision_compute_fn(y_preds, y_targets):
     try:
         from sklearn.metrics import average_precision_score
     except ImportError:
         raise RuntimeError("This contrib module requires sklearn to be installed.")
 
     y_true = y_targets.numpy()
-    if activation is not None:
-        y_preds = activation(y_preds)
     y_pred = y_preds.numpy()
     return average_precision_score(y_true, y_pred)
 
@@ -21,14 +18,23 @@ class AveragePrecision(EpochMetric):
     sklearn.metrics.average_precision_score.html#sklearn.metrics.average_precision_score>`_ .
 
     Args:
-        activation (callable, optional): optional function to apply on prediction tensors,
-            e.g. `activation=torch.sigmoid` to transform logits.
         output_transform (callable, optional): a callable that is used to transform the
             :class:`~ignite.engine.Engine`'s `process_function`'s output into the
             form expected by the metric. This can be useful if, for example, you have a multi-output model and
             you want to compute the metric with respect to one of the outputs.
 
+    AveragePrecision expects y to be comprised of 0's and 1's. y_pred must either be probability estimates or
+    confidence values. To apply an activation to y_pred, use output_transform as shown below:
+
+    .. code-block:: python
+
+        def activated_output_transform(output):
+            y_pred, y = output
+            y_pred = torch.softmax(y_pred)
+            return y_pred, y
+
+        avg_precision = AveragePrecision(activated_output_transform)
+
     """
     def __init__(self, activation=None, output_transform=lambda x: x):
-        super(AveragePrecision, self).__init__(partial(average_precision_compute_fn, activation=activation),
-                                               output_transform=output_transform)
+        super(AveragePrecision, self).__init__(average_precision_compute_fn, output_transform=output_transform)
diff --git a/ignite/contrib/metrics/roc_auc.py b/ignite/contrib/metrics/roc_auc.py
--- a/ignite/contrib/metrics/roc_auc.py
+++ b/ignite/contrib/metrics/roc_auc.py
@@ -1,16 +1,13 @@
-from functools import partial
 from ignite.metrics import EpochMetric
 
 
-def roc_auc_compute_fn(y_preds, y_targets, activation=None):
+def roc_auc_compute_fn(y_preds, y_targets):
     try:
         from sklearn.metrics import roc_auc_score
     except ImportError:
         raise RuntimeError("This contrib module requires sklearn to be installed.")
 
     y_true = y_targets.numpy()
-    if activation is not None:
-        y_preds = activation(y_preds)
     y_pred = y_preds.numpy()
     return roc_auc_score(y_true, y_pred)
 
@@ -22,14 +19,23 @@ class ROC_AUC(EpochMetric):
     sklearn.metrics.roc_auc_score.html#sklearn.metrics.roc_auc_score>`_ .
 
     Args:
-        activation (callable, optional): optional function to apply on prediction tensors,
-            e.g. `activation=torch.sigmoid` to transform logits.
         output_transform (callable, optional): a callable that is used to transform the
             :class:`~ignite.engine.Engine`'s `process_function`'s output into the
             form expected by the metric. This can be useful if, for example, you have a multi-output model and
             you want to compute the metric with respect to one of the outputs.
 
+    ROC_AUC expects y to be comprised of 0's and 1's. y_pred must either be probability estimates or confidence
+    values. To apply an activation to y_pred, use output_transform as shown below:
+
+    .. code-block:: python
+
+        def activated_output_transform(output):
+            y_pred, y = output
+            y_pred = torch.sigmoid(y_pred)
+            return y_pred, y
+
+        roc_auc = ROC_AUC(activated_output_transform)
+
     """
-    def __init__(self, activation=None, output_transform=lambda x: x):
-        super(ROC_AUC, self).__init__(partial(roc_auc_compute_fn, activation=activation),
-                                      output_transform=output_transform)
+    def __init__(self, output_transform=lambda x: x):
+        super(ROC_AUC, self).__init__(roc_auc_compute_fn, output_transform=output_transform)


Test patch:
diff --git a/tests/ignite/contrib/metrics/test_average_precision.py b/tests/ignite/contrib/metrics/test_average_precision.py
--- a/tests/ignite/contrib/metrics/test_average_precision.py
+++ b/tests/ignite/contrib/metrics/test_average_precision.py
@@ -51,35 +51,47 @@ def test_ap_score_2():
     assert ap == np_ap
 
 
-def test_ap_score_with_activation():
+def test_integration_ap_score_with_output_transform():
 
+    np.random.seed(1)
     size = 100
-    np_y_pred = np.random.rand(size, 5)
-    np_y_pred_softmax = torch.softmax(torch.from_numpy(np_y_pred), dim=1).numpy()
-    np_y = np.random.randint(0, 2, size=(size, 5), dtype=np.long)
-    np_ap = average_precision_score(np_y, np_y_pred_softmax)
+    np_y_pred = np.random.rand(size, 1)
+    np_y = np.zeros((size,), dtype=np.long)
+    np_y[size // 2:] = 1
+    np.random.shuffle(np_y)
 
-    ap_metric = AveragePrecision(activation=torch.nn.Softmax(dim=1))
-    y_pred = torch.from_numpy(np_y_pred)
-    y = torch.from_numpy(np_y)
+    np_ap = average_precision_score(np_y, np_y_pred)
 
-    ap_metric.reset()
-    ap_metric.update((y_pred, y))
-    ap = ap_metric.compute()
+    batch_size = 10
+
+    def update_fn(engine, batch):
+        idx = (engine.state.iteration - 1) * batch_size
+        y_true_batch = np_y[idx:idx + batch_size]
+        y_pred_batch = np_y_pred[idx:idx + batch_size]
+        return idx, torch.from_numpy(y_pred_batch), torch.from_numpy(y_true_batch)
+
+    engine = Engine(update_fn)
+
+    ap_metric = AveragePrecision(output_transform=lambda x: (x[1], x[2]))
+    ap_metric.attach(engine, 'ap')
+
+    data = list(range(size // batch_size))
+    ap = engine.run(data, max_epochs=1).metrics['ap']
 
     assert ap == np_ap
 
 
-def test_integration_ap_score_with_output_transform():
+def test_integration_ap_score_with_activated_output_transform():
 
     np.random.seed(1)
     size = 100
     np_y_pred = np.random.rand(size, 1)
+    np_y_pred_softmax = torch.softmax(torch.from_numpy(np_y_pred), dim=1).numpy()
     np_y = np.zeros((size,), dtype=np.long)
     np_y[size // 2:] = 1
     np.random.shuffle(np_y)
 
-    np_ap = average_precision_score(np_y, np_y_pred)
+    np_ap = average_precision_score(np_y, np_y_pred_softmax)
 
     batch_size = 10
 
@@ -91,8 +103,8 @@ def update_fn(engine, batch):
 
     engine = Engine(update_fn)
 
-    roc_auc_metric = AveragePrecision(output_transform=lambda x: (x[1], x[2]))
-    roc_auc_metric.attach(engine, 'ap')
+    ap_metric = AveragePrecision(output_transform=lambda x: (torch.softmax(x[1], dim=1), x[2]))
+    ap_metric.attach(engine, 'ap')
 
     data = list(range(size // batch_size))
     ap = engine.run(data, max_epochs=1).metrics['ap']
diff --git a/tests/ignite/contrib/metrics/test_roc_auc.py b/tests/ignite/contrib/metrics/test_roc_auc.py
--- a/tests/ignite/contrib/metrics/test_roc_auc.py
+++ b/tests/ignite/contrib/metrics/test_roc_auc.py
@@ -52,36 +52,47 @@ def test_roc_auc_score_2():
     assert roc_auc == np_roc_auc
 
 
-def test_roc_auc_score_with_activation():
+def test_integration_roc_auc_score_with_output_transform():
 
+    np.random.seed(1)
     size = 100
     np_y_pred = np.random.rand(size, 1)
-    np_y_pred_sigmoid = torch.sigmoid(torch.from_numpy(np_y_pred)).numpy()
     np_y = np.zeros((size,), dtype=np.long)
     np_y[size // 2:] = 1
-    np_roc_auc = roc_auc_score(np_y, np_y_pred_sigmoid)
+    np.random.shuffle(np_y)
 
-    roc_auc_metric = ROC_AUC(activation=torch.sigmoid)
-    y_pred = torch.from_numpy(np_y_pred)
-    y = torch.from_numpy(np_y)
+    np_roc_auc = roc_auc_score(np_y, np_y_pred)
 
-    roc_auc_metric.reset()
-    roc_auc_metric.update((y_pred, y))
-    roc_auc = roc_auc_metric.compute()
+    batch_size = 10
+
+    def update_fn(engine, batch):
+        idx = (engine.state.iteration - 1) * batch_size
+        y_true_batch = np_y[idx:idx + batch_size]
+        y_pred_batch = np_y_pred[idx:idx + batch_size]
+        return idx, torch.from_numpy(y_pred_batch), torch.from_numpy(y_true_batch)
+
+    engine = Engine(update_fn)
+
+    roc_auc_metric = ROC_AUC(output_transform=lambda x: (x[1], x[2]))
+    roc_auc_metric.attach(engine, 'roc_auc')
+
+    data = list(range(size // batch_size))
+    roc_auc = engine.run(data, max_epochs=1).metrics['roc_auc']
 
     assert roc_auc == np_roc_auc
 
 
-def test_integration_roc_auc_score_with_output_transform():
+def test_integration_roc_auc_score_with_activated_output_transform():
 
     np.random.seed(1)
     size = 100
     np_y_pred = np.random.rand(size, 1)
+    np_y_pred_sigmoid = torch.sigmoid(torch.from_numpy(np_y_pred)).numpy()
     np_y = np.zeros((size,), dtype=np.long)
     np_y[size // 2:] = 1
     np.random.shuffle(np_y)
 
-    np_roc_auc = roc_auc_score(np_y, np_y_pred)
+    np_roc_auc = roc_auc_score(np_y, np_y_pred_sigmoid)
 
     batch_size = 10
 
@@ -93,7 +104,7 @@ def update_fn(engine, batch):
 
     engine = Engine(update_fn)
 
-    roc_auc_metric = ROC_AUC(output_transform=lambda x: (x[1], x[2]))
+    roc_auc_metric = ROC_AUC(output_transform=lambda x: (torch.sigmoid(x[1]), x[2]))
     roc_auc_metric.attach(engine, 'roc_auc')
 
     data = list(range(size // batch_size))