Repository: pytorch/ignite

Issue:
Average value metric
Discussed on the slack:
> Joost van Amersfoort [7:14 PM]
I have a very simple question, how do I compute an average over a validation set?I have an evaluator engine and I am considering the EpochMetric, and pass `None` as the label and make the compute_fn only average over the first column, or the `RunningAverage` metric with `alpha=1`. However both feel a bit gross for this very simple problem.

> Interesting, looking at the `Loss` source then it indeed seems that it’s very close to what I want and more scalable than EpochMetric. But the problem is that it’s setup for supervised learning and forces len(output) >= 2


Idea is to provide some metric class similar to `Loss` but with no restrictions, just to compute average on provided iteration values.




Discussion:
Thanks for making this issue @vfdev-5 

Would it be reasonable to just remove the restrictions from Loss? Perhaps make arguments for supervised learning the default arguments.
> Perhaps make arguments for supervised learning the default arguments.

@y0ast should we still compute a `loss_fn` on a single entry ? 
https://github.com/pytorch/ignite/blob/fc85e25dc4f938d780b4c425acb2d40f6cac6f24/ignite/metrics/loss.py#L44

If yes, can you provide an example, please?
In my case, `y_pred` is batch size by 1. So my loss function is just `torch.mean` to average over the batch.

It's a bit ugly to call that a loss function, so perhaps a different metric is a good idea after all.

Resolving patch:
diff --git a/ignite/metrics/__init__.py b/ignite/metrics/__init__.py
--- a/ignite/metrics/__init__.py
+++ b/ignite/metrics/__init__.py
@@ -12,3 +12,4 @@
 from ignite.metrics.running_average import RunningAverage
 from ignite.metrics.metrics_lambda import MetricsLambda
 from ignite.metrics.confusion_matrix import ConfusionMatrix, IoU, mIoU
+from ignite.metrics.accumulation import VariableAccumulation, Average, GeometricAverage
diff --git a/ignite/metrics/accumulation.py b/ignite/metrics/accumulation.py
new file mode 100644
--- /dev/null
+++ b/ignite/metrics/accumulation.py
@@ -0,0 +1,146 @@
+import numbers
+import torch
+
+from ignite.metrics import Metric
+from ignite.exceptions import NotComputableError
+
+
+class VariableAccumulation(Metric):
+    """Single variable accumulator helper to compute (arithmetic, geometric, harmonic) average of a single variable.
+
+    - `update` must receive output of the form `x`.
+    - `x` can be a number or `torch.Tensor`.
+
+    Note:
+
+        The class stores input into two public variables: `accumulator` and `num_examples`.
+        Number of samples is updated following the rule:
+
+        - `+1` if input is a number
+        - `+1` if input is a 1D `torch.Tensor`
+        - `+batch_size` if input is a ND `torch.Tensor`. Batch size is the first dimension (`shape[0]`).
+
+    Args:
+        op (callable): a callable to update accumulator. Method's signature is `(accumulator, output)`.
+            For example, to compute arithmetic mean value, `op = lambda a, x: a + x`.
+        output_transform (callable, optional): a callable that is used to transform the
+            :class:`~ignite.engine.Engine`'s `process_function`'s output into the
+            form expected by the metric. This can be useful if, for example, you have a multi-output model and
+            you want to compute the metric with respect to one of the outputs.
+
+    """
+
+    def __init__(self, op, output_transform=lambda x: x):
+        if not callable(op):
+            raise TypeError("Argument op should be a callable, but given {}".format(type(op)))
+        self.accumulator = None
+        self.num_examples = None
+        self._op = op
+        super(VariableAccumulation, self).__init__(output_transform=output_transform)
+
+    def reset(self):
+        self.accumulator = torch.tensor(0.0, dtype=torch.float64)
+        self.num_examples = torch.tensor(0.0, dtype=torch.float64)
+        super(VariableAccumulation, self).reset()
+
+    def _check_output_type(self, output):
+        if not (isinstance(output, numbers.Number) or isinstance(output, torch.Tensor)):
+            raise TypeError("Output should be a number or torch.Tensor, but given {}".format(type(output)))
+
+    def update(self, output):
+        self._check_output_type(output)
+
+        self.accumulator = self._op(self.accumulator, output)
+        if hasattr(output, 'shape'):
+            self.num_examples += output.shape[0] if len(output.shape) > 1 else 1
+        else:
+            self.num_examples += 1
+
+    def compute(self):
+        return [self.accumulator, self.num_examples]
+
+
+class Average(VariableAccumulation):
+    """Helper class to compute arithmetic average of a single variable.
+
+    - `update` must receive output of the form `x`.
+    - `x` can be a number or `torch.Tensor`.
+
+    Note:
+
+        Number of samples is updated following the rule:
+
+        - `+1` if input is a number
+        - `+1` if input is a 1D `torch.Tensor`
+        - `+batch_size` if input is a ND `torch.Tensor`. Batch size is the first dimension (`shape[0]`).
+
+    Examples:
+
+    .. code-block:: python
+
+        evaluator = ...
+
+        custom_var_mean = Average(output_transform=lambda output: output['custom_var'])
+        custom_var_mean.attach(evaluator, 'mean_custom_var')
+
+        state = evaluator.run(dataset)
+        # state.metrics['mean_custom_var'] -> average of output['custom_var']
+
+    Args:
+        output_transform (callable, optional): a callable that is used to transform the
+            :class:`~ignite.engine.Engine`'s `process_function`'s output into the
+            form expected by the metric. This can be useful if, for example, you have a multi-output model and
+            you want to compute the metric with respect to one of the outputs.
+
+    """
+    def __init__(self, output_transform=lambda x: x):
+
+        def _mean_op(a, x):
+            return a + x
+
+        super(Average, self).__init__(op=_mean_op, output_transform=output_transform)
+
+    def compute(self):
+        if self.num_examples < 1:
+            raise NotComputableError("{} must have at least one example before"
+                                     " it can be computed.".format(self.__class__.__name__))
+
+        return self.accumulator / self.num_examples
+
+
+class GeometricAverage(VariableAccumulation):
+    """Helper class to compute geometric average of a single variable.
+
+    - `update` must receive output of the form `x`.
+    - `x` can be a number or `torch.Tensor`.
+
+    Note:
+
+        Number of samples is updated following the rule:
+
+        - `+1` if input is a number
+        - `+1` if input is a 1D `torch.Tensor`
+        - `+batch_size` if input is a ND `torch.Tensor`. Batch size is the first dimension (`shape[0]`).
+
+    Args:
+        output_transform (callable, optional): a callable that is used to transform the
+            :class:`~ignite.engine.Engine`'s `process_function`'s output into the
+            form expected by the metric. This can be useful if, for example, you have a multi-output model and
+            you want to compute the metric with respect to one of the outputs.
+
+    """
+    def __init__(self, output_transform=lambda x: x):
+
+        def _geom_op(a, x):
+            if not isinstance(x, torch.Tensor):
+                x = torch.tensor(x)
+            return a + torch.log(x)
+
+        super(GeometricAverage, self).__init__(op=_geom_op, output_transform=output_transform)
+
+    def compute(self):
+        if self.num_examples < 1:
+            raise NotComputableError("{} must have at least one example before"
+                                     " it can be computed.".format(self.__class__.__name__))
+
+        return torch.exp(self.accumulator / self.num_examples)


Test patch:
diff --git a/tests/ignite/metrics/test_accumulation.py b/tests/ignite/metrics/test_accumulation.py
new file mode 100644
--- /dev/null
+++ b/tests/ignite/metrics/test_accumulation.py
@@ -0,0 +1,140 @@
+import torch
+import numpy as np
+
+import pytest
+
+from ignite.metrics.accumulation import VariableAccumulation, Average, GeometricAverage
+from ignite.exceptions import NotComputableError
+from ignite.engine import Events, Engine
+
+
+torch.manual_seed(15)
+
+
+def test_variable_accumulation_wrong_inputs():
+
+    with pytest.raises(TypeError, match=r"Argument op should be a callable"):
+        VariableAccumulation(1)
+
+    with pytest.raises(TypeError, match=r"Output should be a number or torch.Tensor,"):
+        mean_acc = VariableAccumulation(lambda a, x: a + x)
+        mean_acc.update((1, 2))
+
+    with pytest.raises(TypeError, match=r"Output should be a number or torch.Tensor,"):
+        mean_acc = VariableAccumulation(lambda a, x: a + x)
+        mean_acc.update("a")
+
+
+def test_variable_accumulation_mean_variable():
+
+    mean_var = VariableAccumulation(lambda a, x: a + x)
+    y_true = torch.rand(100)
+
+    for y in y_true:
+        mean_var.update(y.item())
+
+    a, n = mean_var.compute()
+    assert a.item() == pytest.approx(y_true.sum().item())
+    assert n == len(y_true)
+
+    mean_var = VariableAccumulation(lambda a, x: a + x)
+    y_true = torch.rand(100, 10)
+    for y in y_true:
+        mean_var.update(y)
+
+    a, n = mean_var.compute()
+    assert a.numpy() == pytest.approx(y_true.sum(dim=0).numpy())
+    assert n == len(y_true)
+
+
+def test_average():
+
+    with pytest.raises(NotComputableError):
+        v = Average()
+        v.compute()
+
+    mean_var = Average()
+    y_true = torch.rand(100) + torch.randint(0, 10, size=(100, )).float()
+
+    for y in y_true:
+        mean_var.update(y.item())
+
+    m = mean_var.compute()
+    assert m.item() == pytest.approx(y_true.mean().item())
+
+    mean_var = Average()
+    y_true = torch.rand(100, 10) + torch.randint(0, 10, size=(100, 10)).float()
+    for y in y_true:
+        mean_var.update(y)
+
+    m = mean_var.compute()
+    assert m.numpy() == pytest.approx(y_true.mean(dim=0).numpy())
+
+
+def _geom_mean(t):
+    np_t = t.numpy()
+    return np.exp(np.mean(np.log(np_t), axis=0))
+
+
+def test_geom_average():
+
+    with pytest.raises(NotComputableError):
+        v = GeometricAverage()
+        v.compute()
+
+    mean_var = GeometricAverage()
+    y_true = torch.rand(100) + torch.randint(0, 10, size=(100,)).float()
+
+    for y in y_true:
+        mean_var.update(y.item())
+
+    m = mean_var.compute()
+    assert m.item() == pytest.approx(_geom_mean(y_true))
+
+    mean_var = GeometricAverage()
+    y_true = torch.rand(100, 10) + torch.randint(0, 10, size=(100, 10)).float()
+    for y in y_true:
+        mean_var.update(y)
+
+    m = mean_var.compute()
+    np.testing.assert_almost_equal(m.numpy(), _geom_mean(y_true), decimal=5)
+
+
+def test_integration():
+
+    def _test(metric_cls, true_result_fn):
+
+        size = 100
+        custom_variable = 10.0 + 5.0 * torch.rand(size, 12)
+
+        def update_fn(engine, batch):
+            return 0, custom_variable[engine.state.iteration - 1]
+
+        engine = Engine(update_fn)
+
+        custom_var_mean = metric_cls(output_transform=lambda output: output[1])
+        custom_var_mean.attach(engine, 'agg_custom_var')
+
+        state = engine.run([0] * size)
+        np.testing.assert_almost_equal(state.metrics['agg_custom_var'].numpy(), true_result_fn(custom_variable),
+                                       decimal=5)
+
+        size = 100
+        custom_variable = 10.0 + 5.0 * torch.rand(size)
+
+        def update_fn(engine, batch):
+            return 0, custom_variable[engine.state.iteration - 1].item()
+
+        engine = Engine(update_fn)
+
+        custom_var_mean = metric_cls(output_transform=lambda output: output[1])
+        custom_var_mean.attach(engine, 'agg_custom_var')
+
+        state = engine.run([0] * size)
+        assert state.metrics['agg_custom_var'] == pytest.approx(true_result_fn(custom_variable))
+
+    def _mean(y_true):
+        return y_true.mean(dim=0).numpy()
+
+    _test(Average, _mean)
+    _test(GeometricAverage, _geom_mean)