Repository: pytorch/ignite (python)

Issue:
Running average metric
Following the discussion in #256 on running average values in the TQDM progress bar, IMO it would be better to split running average computation and display logic.

Here is an implementation on how to compute running average value from an instance of Metric:
```python
from ignite.metrics import Metric


class RunningAverage(Metric):
    """Compute running average of a metric
    """
    
    def __init__(self, metric, alpha=0.98):
        assert isinstance(metric, Metric), "Argument metric should be an instance of Metric"
        assert 0.0 < alpha <= 1.0, "Argument alpha should be a float between 0.0 and 1.0"
        self.metric = metric
        self.alpha = alpha
        super(RunningAverage, self).__init__()
        
    def reset(self):
        self._value = None

    def iteration_completed(self, engine):        
        self.metric.iteration_completed(engine)

    def compute(self):
        if self._value is None:
            self._value = self.metric.compute()
        else:
            self._value = self._value * self.alpha + (1.0 - self.alpha) * self.metric.compute()
        return self._value
    
    def attach(self, engine, name):
        # restart average every epoch
        engine.add_event_handler(Events.EPOCH_STARTED, self.started)
        # reset metric at each iteration        
        engine.add_event_handler(Events.ITERATION_COMPLETED, self.metric.started)
        # compute metric
        engine.add_event_handler(Events.ITERATION_COMPLETED, self.iteration_completed)
        # apply running average
        engine.add_event_handler(Events.ITERATION_COMPLETED, self.completed, name)
```

Usage of the proposed API is following:
```python
import numpy as np
import torch

from ignite.engine import Engine, Events, State
from ignite.metrics import Loss, CategoricalAccuracy


n_iters = 100
batch_size = 10
n_classes = 10
data = list(range(n_iters))
y_true_batch_values = iter(np.random.randint(0, n_classes, size=(n_iters, batch_size)))
y_pred_batch_values = iter(np.random.rand(n_iters, batch_size, n_classes))
loss_values = iter(range(n_iters))


def update_fn(engine, batch):
    loss_value = next(loss_values)
    y_true_batch = next(y_true_batch_values)
    y_pred_batch = next(y_pred_batch_values)
    return loss_value, torch.from_numpy(y_pred_batch), torch.from_numpy(y_true_batch)


trainer = Engine(update_fn)

alpha = 0.98
acc_metric = RunningAverage(CategoricalAccuracy(output_transform=lambda x: [x[1], x[2]]), alpha=alpha)
acc_metric.attach(trainer, 'running_avg_accuracy')

loss_metric = RunningAverage(Loss(torch.nn.CrossEntropyLoss(), output_transform=lambda x: [x[1], x[2]]), alpha=alpha)
loss_metric.attach(trainer, 'running_avg_loss')

@trainer.on(Events.ITERATION_COMPLETED)
def log_running_avg_metrics(engine):
    print("running avg accuracy:", engine.state.metrics['running_avg_accuracy'])
    print("running avg loss:", engine.state.metrics['running_avg_loss'])    
    
```

There is however still missing the part of the running average value based on `state.output`.
Maybe we could find a solution together by improving the previous code...

What do you guys think about ?
Thanks 

cc @miguelvr 





Pull request:
Add RunningAverage handler to compute running average value of a metr…
…ic or engine.state.output

Here is an implementation of running average for a metric or engine.state.output that is frequently used in our examples and in PR with TQDM.

Any feedback on the implementation, please
Thanks

cc @miguelvr


PS.: CI fails on python 2, will fix soon



Patch:
diff --git a/ignite/metrics/__init__.py b/ignite/metrics/__init__.py
index 59989e18..98c35eaa 100644
--- a/ignite/metrics/__init__.py
+++ b/ignite/metrics/__init__.py
@@ -10,3 +10,4 @@ from ignite.metrics.precision import Precision
 from ignite.metrics.recall import Recall
 from ignite.metrics.root_mean_squared_error import RootMeanSquaredError
 from ignite.metrics.top_k_categorical_accuracy import TopKCategoricalAccuracy
+from ignite.metrics.running_average import RunningAverage
diff --git a/ignite/metrics/running_average.py b/ignite/metrics/running_average.py
new file mode 100644
index 00000000..aa8beae6
--- /dev/null
+++ b/ignite/metrics/running_average.py
@@ -0,0 +1,72 @@
+from ignite.metrics import Metric
+from ignite.engine import Events
+
+
+class RunningAverage(Metric):
+    """Compute running average of a metric or the output of process function.
+
+    Args:
+        src (Metric or None): input source: an instance of :class:`ignite.metrics.Metric` or None. The latter
+            corresponds to `engine.state.output` which holds the output of process function.
+        alpha (float, optional): running average decay factor, default 0.98
+        output_transform (Callable, optional): a function to use to transform the output if `src` is None and
+            corresponds the output of process function. Otherwise it should be None.
+
+    """
+
+    def __init__(self, src=None, alpha=0.98, output_transform=None):
+        if not (isinstance(src, Metric) or src is None):
+            raise TypeError("Argument src should be a Metric or None")
+        if not (0.0 < alpha <= 1.0):
+            raise ValueError("Argument alpha should be a float between 0.0 and 1.0")
+
+        if isinstance(src, Metric):
+            if output_transform is not None:
+                raise ValueError("Argument output_transform should be None if src is a Metric")
+            self.src = src
+            self._get_src_value = self._get_metric_value
+            self.iteration_completed = self._metric_iteration_completed
+        else:
+            if output_transform is None:
+                raise ValueError("Argument output_transform should not be None if src corresponds "
+                                 "to the output of process function.")
+            self._get_src_value = self._get_output_value
+            self.update = self._output_update
+
+        self.alpha = alpha
+        super(RunningAverage, self).__init__(output_transform=output_transform)
+
+    def reset(self):
+        self._value = None
+
+    def update(self, output):
+        # Implement abstract method
+        pass
+
+    def compute(self):
+        if self._value is None:
+            self._value = self._get_src_value()
+        else:
+            self._value = self._value * self.alpha + (1.0 - self.alpha) * self._get_src_value()
+        return self._value
+
+    def attach(self, engine, name):
+        # restart average every epoch
+        engine.add_event_handler(Events.EPOCH_STARTED, self.started)
+        # compute metric
+        engine.add_event_handler(Events.ITERATION_COMPLETED, self.iteration_completed)
+        # apply running average
+        engine.add_event_handler(Events.ITERATION_COMPLETED, self.completed, name)
+
+    def _get_metric_value(self):
+        return self.src.compute()
+
+    def _get_output_value(self):
+        return self.src
+
+    def _metric_iteration_completed(self, engine):
+        self.src.started(engine)
+        self.src.iteration_completed(engine)
+
+    def _output_update(self, output):
+        self.src = output


Test patch:
diff --git a/tests/ignite/metrics/test_running_average.py b/tests/ignite/metrics/test_running_average.py
new file mode 100644
index 00000000..cd4c0e0d
--- /dev/null
+++ b/tests/ignite/metrics/test_running_average.py
@@ -0,0 +1,106 @@
+
+import numpy as np
+import torch
+
+from ignite.engine import Engine, Events
+from ignite.metrics import CategoricalAccuracy, RunningAverage
+
+import pytest
+
+
+def test_wrong_input_args():
+    with pytest.raises(TypeError):
+        _ = RunningAverage(src=[12, 34])
+
+    with pytest.raises(ValueError):
+        _ = RunningAverage(alpha=-1.0)
+
+    with pytest.raises(ValueError):
+        _ = RunningAverage(CategoricalAccuracy(), output_transform=lambda x: x[0])
+
+    with pytest.raises(ValueError):
+        _ = RunningAverage()
+
+
+def test_integration():
+
+    n_iters = 100
+    batch_size = 10
+    n_classes = 10
+    y_true_batch_values = iter(np.random.randint(0, n_classes, size=(n_iters, batch_size)))
+    y_pred_batch_values = iter(np.random.rand(n_iters, batch_size, n_classes))
+    loss_values = iter(range(n_iters))
+
+    def update_fn(engine, batch):
+        loss_value = next(loss_values)
+        y_true_batch = next(y_true_batch_values)
+        y_pred_batch = next(y_pred_batch_values)
+        return loss_value, torch.from_numpy(y_pred_batch), torch.from_numpy(y_true_batch)
+
+    trainer = Engine(update_fn)
+    alpha = 0.98
+
+    acc_metric = RunningAverage(CategoricalAccuracy(output_transform=lambda x: [x[1], x[2]]), alpha=alpha)
+    acc_metric.attach(trainer, 'running_avg_accuracy')
+
+    avg_output = RunningAverage(output_transform=lambda x: x[0], alpha=alpha)
+    avg_output.attach(trainer, 'running_avg_output')
+
+    running_avg_acc = [None]
+
+    @trainer.on(Events.ITERATION_COMPLETED, running_avg_acc)
+    def manual_running_avg_acc(engine, running_avg_acc):
+        _, y_pred, y = engine.state.output
+        indices = torch.max(y_pred, 1)[1]
+        correct = torch.eq(indices, y).view(-1)
+        num_correct = torch.sum(correct).item()
+        num_examples = correct.shape[0]
+        batch_acc = num_correct * 1.0 / num_examples
+        if running_avg_acc[0] is None:
+            running_avg_acc[0] = batch_acc
+        else:
+            running_avg_acc[0] = running_avg_acc[0] * alpha + (1.0 - alpha) * batch_acc
+        engine.state.running_avg_acc = running_avg_acc[0]
+
+    @trainer.on(Events.EPOCH_STARTED)
+    def running_avg_output_init(engine):
+        engine.state.running_avg_output = None
+
+    @trainer.on(Events.ITERATION_COMPLETED)
+    def running_avg_output_update(engine):
+        if engine.state.running_avg_output is None:
+            engine.state.running_avg_output = engine.state.output[0]
+        else:
+            engine.state.running_avg_output = engine.state.running_avg_output * alpha + \
+                (1.0 - alpha) * engine.state.output[0]
+
+    @trainer.on(Events.ITERATION_COMPLETED)
+    def assert_equal_running_avg_acc_values(engine):
+        assert engine.state.running_avg_acc == engine.state.metrics['running_avg_accuracy'], \
+            "{} vs {}".format(engine.state.running_avg_acc, engine.state.metrics['running_avg_accuracy'])
+
+    @trainer.on(Events.ITERATION_COMPLETED)
+    def assert_equal_running_avg_output_values(engine):
+        assert engine.state.running_avg_output == engine.state.metrics['running_avg_output'], \
+            "{} vs {}".format(engine.state.running_avg_output, engine.state.metrics['running_avg_output'])
+
+    np.random.seed(10)
+    running_avg_acc[0] = None
+    n_iters = 10
+    batch_size = 10
+    n_classes = 10
+    data = list(range(n_iters))
+    loss_values = iter(range(n_iters))
+    y_true_batch_values = iter(np.random.randint(0, n_classes, size=(n_iters, batch_size)))
+    y_pred_batch_values = iter(np.random.rand(n_iters, batch_size, n_classes))
+    trainer.run(data, max_epochs=1)
+
+    running_avg_acc[0] = None
+    n_iters = 10
+    batch_size = 10
+    n_classes = 10
+    data = list(range(n_iters))
+    loss_values = iter(range(n_iters))
+    y_true_batch_values = iter(np.random.randint(0, n_classes, size=(n_iters, batch_size)))
+    y_pred_batch_values = iter(np.random.rand(n_iters, batch_size, n_classes))
+    trainer.run(data, max_epochs=1)