Repository: pytorch/ignite

Issue:
Running average metric
Following the discussion in #256 on running average values in the TQDM progress bar, IMO it would be better to split running average computation and display logic.

Here is an implementation on how to compute running average value from an instance of Metric:
```python
from ignite.metrics import Metric


class RunningAverage(Metric):
    """Compute running average of a metric
    """
    
    def __init__(self, metric, alpha=0.98):
        assert isinstance(metric, Metric), "Argument metric should be an instance of Metric"
        assert 0.0 < alpha <= 1.0, "Argument alpha should be a float between 0.0 and 1.0"
        self.metric = metric
        self.alpha = alpha
        super(RunningAverage, self).__init__()
        
    def reset(self):
        self._value = None

    def iteration_completed(self, engine):        
        self.metric.iteration_completed(engine)

    def compute(self):
        if self._value is None:
            self._value = self.metric.compute()
        else:
            self._value = self._value * self.alpha + (1.0 - self.alpha) * self.metric.compute()
        return self._value
    
    def attach(self, engine, name):
        # restart average every epoch
        engine.add_event_handler(Events.EPOCH_STARTED, self.started)
        # reset metric at each iteration        
        engine.add_event_handler(Events.ITERATION_COMPLETED, self.metric.started)
        # compute metric
        engine.add_event_handler(Events.ITERATION_COMPLETED, self.iteration_completed)
        # apply running average
        engine.add_event_handler(Events.ITERATION_COMPLETED, self.completed, name)
```

Usage of the proposed API is following:
```python
import numpy as np
import torch

from ignite.engine import Engine, Events, State
from ignite.metrics import Loss, CategoricalAccuracy


n_iters = 100
batch_size = 10
n_classes = 10
data = list(range(n_iters))
y_true_batch_values = iter(np.random.randint(0, n_classes, size=(n_iters, batch_size)))
y_pred_batch_values = iter(np.random.rand(n_iters, batch_size, n_classes))
loss_values = iter(range(n_iters))


def update_fn(engine, batch):
    loss_value = next(loss_values)
    y_true_batch = next(y_true_batch_values)
    y_pred_batch = next(y_pred_batch_values)
    return loss_value, torch.from_numpy(y_pred_batch), torch.from_numpy(y_true_batch)


trainer = Engine(update_fn)

alpha = 0.98
acc_metric = RunningAverage(CategoricalAccuracy(output_transform=lambda x: [x[1], x[2]]), alpha=alpha)
acc_metric.attach(trainer, 'running_avg_accuracy')

loss_metric = RunningAverage(Loss(torch.nn.CrossEntropyLoss(), output_transform=lambda x: [x[1], x[2]]), alpha=alpha)
loss_metric.attach(trainer, 'running_avg_loss')

@trainer.on(Events.ITERATION_COMPLETED)
def log_running_avg_metrics(engine):
    print("running avg accuracy:", engine.state.metrics['running_avg_accuracy'])
    print("running avg loss:", engine.state.metrics['running_avg_loss'])    
    
```

There is however still missing the part of the running average value based on `state.output`.
Maybe we could find a solution together by improving the previous code...

What do you guys think about ?
Thanks 

cc @miguelvr 






Discussion:
not being able to compute it over engine.state.output can be an annoyance. But I guess we can solve it with an `output_transform` function, right?
@miguelvr could you please give an example of API with `output_transform` on engine.state.output ?
I'm not sure to understand your idea
we could make `RunningAverage` a `handler` instead of a `Metric`. I would be something that processes `metrics` rather than being one on itself. 

Plus, if you make it with an `output_tranform` argument, you are able to access both `engine.state.metrics` and `engine.state.output`. 

What do you think?


I see what you mean. I thought about such implementation, maybe it worth to try to implement
Here is naive implementation of `RunningAverage` with joint `Metric` and output:
```python
from ignite.metrics import Metric


class RunningAverage(object):
    """Compute running average of a metric or output
    """
    
    def __init__(self, src, alpha=0.98, output_transform=None):
        assert isinstance(src, Metric) or src == 'output', "Argument src should be a Metric or string 'output'"
        assert 0.0 < alpha <= 1.0, "Argument alpha should be a float between 0.0 and 1.0"

        self.src = src
        if isinstance(src, Metric):
            assert output_transform is None, "Argument output_transform should be None if src is 'output'"
            self._get_src_value = self._get_metric_value
            self.iteration_completed = self._metric_iteration_completed
        else:
            self._get_src_value = self._get_output_value
            self.iteration_completed = self._output_iteration_completed

        self.alpha = alpha
        self._output_transform = output_transform
        super(RunningAverage, self).__init__()
    
    def reset(self, engine):
        self._value = None
        
    def _metric_iteration_completed(self, engine):
        self.src.started(engine)
        self.src.iteration_completed(engine)

    def _output_iteration_completed(self, engine):
        self.src = self._output_transform(engine.state.output)
        
    def completed(self, engine, name):
        if self._value is None:
            self._value = self._get_src_value()
        else:
            self._value = self._value * self.alpha + (1.0 - self.alpha) * self._get_src_value()    
        engine.state.metrics[name] = self._value

    def attach(self, engine, name):
        # restart average every epoch
        engine.add_event_handler(Events.EPOCH_STARTED, self.reset)
        # compute metric
        engine.add_event_handler(Events.ITERATION_COMPLETED, self.iteration_completed)
        # apply running average
        engine.add_event_handler(Events.ITERATION_COMPLETED, self.completed, name)

    def _get_metric_value(self):
        return self.src.compute()
        
    def _get_output_value(self):
        return self.src
```
with API usage is similar to the previous one
```python
acc_metric = RunningAverage(CategoricalAccuracy(output_transform=lambda x: [x[1], x[2]]), alpha=alpha)
acc_metric.attach(trainer, 'running_avg_accuracy')

loss_metric = RunningAverage(Loss(torch.nn.CrossEntropyLoss(), output_transform=lambda x: [x[1], x[2]]), alpha=alpha)
loss_metric.attach(trainer, 'running_avg_loss')

avg_output = RunningAverage('output', output_transform=lambda x: x[0], alpha=alpha)
avg_output.attach(trainer, 'running_avg_output')

@trainer.on(Events.ITERATION_COMPLETED)
def log_running_avg_metrics(engine):
    print("running avg accuracy:", engine.state.metrics['running_avg_accuracy'])
    print("running avg loss:", engine.state.metrics['running_avg_loss'])
    print("running avg output:", engine.state.metrics['running_avg_output'])

```

I think we can leave `RunningAverage` as being a `Metric` and simplify a little bit the code... 

Any thoughts/comments ?
> I think we can leave RunningAverage as being a Metric and simplify a little bit the code...

Actually no real gain of such inheritance. Will leave as a handler.

Resolving patch:
diff --git a/ignite/metrics/__init__.py b/ignite/metrics/__init__.py
--- a/ignite/metrics/__init__.py
+++ b/ignite/metrics/__init__.py
@@ -10,3 +10,4 @@
 from ignite.metrics.recall import Recall
 from ignite.metrics.root_mean_squared_error import RootMeanSquaredError
 from ignite.metrics.top_k_categorical_accuracy import TopKCategoricalAccuracy
+from ignite.metrics.running_average import RunningAverage
diff --git a/ignite/metrics/running_average.py b/ignite/metrics/running_average.py
new file mode 100644
--- /dev/null
+++ b/ignite/metrics/running_average.py
@@ -0,0 +1,72 @@
+from ignite.metrics import Metric
+from ignite.engine import Events
+
+
+class RunningAverage(Metric):
+    """Compute running average of a metric or the output of process function.
+
+    Args:
+        src (Metric or None): input source: an instance of :class:`ignite.metrics.Metric` or None. The latter
+            corresponds to `engine.state.output` which holds the output of process function.
+        alpha (float, optional): running average decay factor, default 0.98
+        output_transform (Callable, optional): a function to use to transform the output if `src` is None and
+            corresponds the output of process function. Otherwise it should be None.
+
+    """
+
+    def __init__(self, src=None, alpha=0.98, output_transform=None):
+        if not (isinstance(src, Metric) or src is None):
+            raise TypeError("Argument src should be a Metric or None")
+        if not (0.0 < alpha <= 1.0):
+            raise ValueError("Argument alpha should be a float between 0.0 and 1.0")
+
+        if isinstance(src, Metric):
+            if output_transform is not None:
+                raise ValueError("Argument output_transform should be None if src is a Metric")
+            self.src = src
+            self._get_src_value = self._get_metric_value
+            self.iteration_completed = self._metric_iteration_completed
+        else:
+            if output_transform is None:
+                raise ValueError("Argument output_transform should not be None if src corresponds "
+                                 "to the output of process function.")
+            self._get_src_value = self._get_output_value
+            self.update = self._output_update
+
+        self.alpha = alpha
+        super(RunningAverage, self).__init__(output_transform=output_transform)
+
+    def reset(self):
+        self._value = None
+
+    def update(self, output):
+        # Implement abstract method
+        pass
+
+    def compute(self):
+        if self._value is None:
+            self._value = self._get_src_value()
+        else:
+            self._value = self._value * self.alpha + (1.0 - self.alpha) * self._get_src_value()
+        return self._value
+
+    def attach(self, engine, name):
+        # restart average every epoch
+        engine.add_event_handler(Events.EPOCH_STARTED, self.started)
+        # compute metric
+        engine.add_event_handler(Events.ITERATION_COMPLETED, self.iteration_completed)
+        # apply running average
+        engine.add_event_handler(Events.ITERATION_COMPLETED, self.completed, name)
+
+    def _get_metric_value(self):
+        return self.src.compute()
+
+    def _get_output_value(self):
+        return self.src
+
+    def _metric_iteration_completed(self, engine):
+        self.src.started(engine)
+        self.src.iteration_completed(engine)
+
+    def _output_update(self, output):
+        self.src = output


Test patch:
diff --git a/tests/ignite/metrics/test_running_average.py b/tests/ignite/metrics/test_running_average.py
new file mode 100644
--- /dev/null
+++ b/tests/ignite/metrics/test_running_average.py
@@ -0,0 +1,106 @@
+
+import numpy as np
+import torch
+
+from ignite.engine import Engine, Events
+from ignite.metrics import CategoricalAccuracy, RunningAverage
+
+import pytest
+
+
+def test_wrong_input_args():
+    with pytest.raises(TypeError):
+        _ = RunningAverage(src=[12, 34])
+
+    with pytest.raises(ValueError):
+        _ = RunningAverage(alpha=-1.0)
+
+    with pytest.raises(ValueError):
+        _ = RunningAverage(CategoricalAccuracy(), output_transform=lambda x: x[0])
+
+    with pytest.raises(ValueError):
+        _ = RunningAverage()
+
+
+def test_integration():
+
+    n_iters = 100
+    batch_size = 10
+    n_classes = 10
+    y_true_batch_values = iter(np.random.randint(0, n_classes, size=(n_iters, batch_size)))
+    y_pred_batch_values = iter(np.random.rand(n_iters, batch_size, n_classes))
+    loss_values = iter(range(n_iters))
+
+    def update_fn(engine, batch):
+        loss_value = next(loss_values)
+        y_true_batch = next(y_true_batch_values)
+        y_pred_batch = next(y_pred_batch_values)
+        return loss_value, torch.from_numpy(y_pred_batch), torch.from_numpy(y_true_batch)
+
+    trainer = Engine(update_fn)
+    alpha = 0.98
+
+    acc_metric = RunningAverage(CategoricalAccuracy(output_transform=lambda x: [x[1], x[2]]), alpha=alpha)
+    acc_metric.attach(trainer, 'running_avg_accuracy')
+
+    avg_output = RunningAverage(output_transform=lambda x: x[0], alpha=alpha)
+    avg_output.attach(trainer, 'running_avg_output')
+
+    running_avg_acc = [None]
+
+    @trainer.on(Events.ITERATION_COMPLETED, running_avg_acc)
+    def manual_running_avg_acc(engine, running_avg_acc):
+        _, y_pred, y = engine.state.output
+        indices = torch.max(y_pred, 1)[1]
+        correct = torch.eq(indices, y).view(-1)
+        num_correct = torch.sum(correct).item()
+        num_examples = correct.shape[0]
+        batch_acc = num_correct * 1.0 / num_examples
+        if running_avg_acc[0] is None:
+            running_avg_acc[0] = batch_acc
+        else:
+            running_avg_acc[0] = running_avg_acc[0] * alpha + (1.0 - alpha) * batch_acc
+        engine.state.running_avg_acc = running_avg_acc[0]
+
+    @trainer.on(Events.EPOCH_STARTED)
+    def running_avg_output_init(engine):
+        engine.state.running_avg_output = None
+
+    @trainer.on(Events.ITERATION_COMPLETED)
+    def running_avg_output_update(engine):
+        if engine.state.running_avg_output is None:
+            engine.state.running_avg_output = engine.state.output[0]
+        else:
+            engine.state.running_avg_output = engine.state.running_avg_output * alpha + \
+                (1.0 - alpha) * engine.state.output[0]
+
+    @trainer.on(Events.ITERATION_COMPLETED)
+    def assert_equal_running_avg_acc_values(engine):
+        assert engine.state.running_avg_acc == engine.state.metrics['running_avg_accuracy'], \
+            "{} vs {}".format(engine.state.running_avg_acc, engine.state.metrics['running_avg_accuracy'])
+
+    @trainer.on(Events.ITERATION_COMPLETED)
+    def assert_equal_running_avg_output_values(engine):
+        assert engine.state.running_avg_output == engine.state.metrics['running_avg_output'], \
+            "{} vs {}".format(engine.state.running_avg_output, engine.state.metrics['running_avg_output'])
+
+    np.random.seed(10)
+    running_avg_acc[0] = None
+    n_iters = 10
+    batch_size = 10
+    n_classes = 10
+    data = list(range(n_iters))
+    loss_values = iter(range(n_iters))
+    y_true_batch_values = iter(np.random.randint(0, n_classes, size=(n_iters, batch_size)))
+    y_pred_batch_values = iter(np.random.rand(n_iters, batch_size, n_classes))
+    trainer.run(data, max_epochs=1)
+
+    running_avg_acc[0] = None
+    n_iters = 10
+    batch_size = 10
+    n_classes = 10
+    data = list(range(n_iters))
+    loss_values = iter(range(n_iters))
+    y_true_batch_values = iter(np.random.randint(0, n_classes, size=(n_iters, batch_size)))
+    y_pred_batch_values = iter(np.random.rand(n_iters, batch_size, n_classes))
+    trainer.run(data, max_epochs=1)