Repository: pytorch/ignite

Issue:
Bug in recursive attach of MetricsLambda
As a result of #385, the recursive attach of MetricsLambda is not working as intended. 

The code below throws this error: `NotComputableError: Accuracy must have at least one example before it can be computed.`

```python
import torch

from ignite.metrics import Accuracy, Precision, Recall, MetricsLambda
from ignite.engine import Engine

accuracy_1 = Accuracy()
accuracy_2 = Accuracy()
mean_accuracy = (accuracy_1 + accuracy_2) / 2


metrics = {
#     "a1": accuracy_1,
#     "a2": accuracy_2,    
    "mean accuracy": mean_accuracy,
}


y_pred = torch.randint(0, 2, size=(15, 10, 4)).float()
y = torch.randint(0, 2, size=(15, 10, 4)).long()

def update_fn(engine, batch):
    y_pred, y = batch
    return y_pred, y

validator = Engine(update_fn)

for name, metric in metrics.items():
    metric.attach(validator, name)

def data(y_pred, y):
    for i in range(y_pred.shape[0]):
        yield (y_pred[i], y[i])

d = data(y_pred, y)
state = validator.run(d, max_epochs=1)

print(state.metrics)
```

cc @vfdev-5 @amitibo 


Discussion:

Resolving patch:
diff --git a/ignite/metrics/metrics_lambda.py b/ignite/metrics/metrics_lambda.py
--- a/ignite/metrics/metrics_lambda.py
+++ b/ignite/metrics/metrics_lambda.py
@@ -53,12 +53,18 @@ def compute(self):
         materialized = [i.compute() if isinstance(i, Metric) else i for i in self.args]
         return self.function(*materialized)
 
-    def attach(self, engine, name):
-        # recursively attach all its dependencies
+    def _internal_attach(self, engine):
         for index, metric in enumerate(self.args):
-            if isinstance(metric, Metric):
+            if isinstance(metric, MetricsLambda):
+                metric._internal_attach(engine)
+            elif isinstance(metric, Metric):
                 if not engine.has_event_handler(metric.started, Events.EPOCH_STARTED):
                     engine.add_event_handler(Events.EPOCH_STARTED, metric.started)
                 if not engine.has_event_handler(metric.iteration_completed, Events.ITERATION_COMPLETED):
                     engine.add_event_handler(Events.ITERATION_COMPLETED, metric.iteration_completed)
-        super(MetricsLambda, self).attach(engine, name)
+
+    def attach(self, engine, name):
+        # recursively attach all its dependencies
+        self._internal_attach(engine)
+        # attach only handler on EPOCH_COMPLETED
+        engine.add_event_handler(Events.EPOCH_COMPLETED, self.completed, name)


Test patch:
diff --git a/tests/ignite/metrics/test_metrics_lambda.py b/tests/ignite/metrics/test_metrics_lambda.py
--- a/tests/ignite/metrics/test_metrics_lambda.py
+++ b/tests/ignite/metrics/test_metrics_lambda.py
@@ -3,7 +3,6 @@
 from pytest import approx
 from sklearn.metrics import precision_score, recall_score, f1_score
 import numpy as np
-from numpy.testing import assert_allclose
 import torch
 
 
@@ -123,6 +122,47 @@ def Fbeta(r, p, beta):
     assert f1_true == approx(state.metrics['f1']), "{} vs {}".format(f1_true, state.metrics['f1'])
 
 
+def test_integration_ingredients_not_attached():
+    np.random.seed(1)
+
+    n_iters = 10
+    batch_size = 10
+    n_classes = 10
+
+    y_true = np.arange(0, n_iters * batch_size) % n_classes
+    y_pred = 0.2 * np.random.rand(n_iters * batch_size, n_classes)
+    for i in range(n_iters * batch_size):
+        if np.random.rand() > 0.4:
+            y_pred[i, y_true[i]] = 1.0
+        else:
+            j = np.random.randint(0, n_classes)
+            y_pred[i, j] = 0.7
+
+    y_true_batch_values = iter(y_true.reshape(n_iters, batch_size))
+    y_pred_batch_values = iter(y_pred.reshape(n_iters, batch_size, n_classes))
+
+    def update_fn(engine, batch):
+        y_true_batch = next(y_true_batch_values)
+        y_pred_batch = next(y_pred_batch_values)
+        return torch.from_numpy(y_pred_batch), torch.from_numpy(y_true_batch)
+
+    evaluator = Engine(update_fn)
+
+    precision = Precision(average=False)
+    recall = Recall(average=False)
+
+    def Fbeta(r, p, beta):
+        return torch.mean((1 + beta ** 2) * p * r / (beta ** 2 * p + r)).item()
+
+    F1 = MetricsLambda(Fbeta, recall, precision, 1)
+    F1.attach(evaluator, "f1")
+
+    data = list(range(n_iters))
+    state = evaluator.run(data, max_epochs=1)
+    f1_true = f1_score(y_true, np.argmax(y_pred, axis=-1), average='macro')
+    assert f1_true == approx(state.metrics['f1']), "{} vs {}".format(f1_true, state.metrics['f1'])
+
+
 def test_state_metrics():
 
     y_pred = torch.randint(0, 2, size=(15, 10, 4)).float()
@@ -151,3 +191,97 @@ def data(y_pred, y):
     state = evaluator.run(d, max_epochs=1)
 
     assert set(state.metrics.keys()) == set(["precision", "recall", "f1"])
+
+
+def test_state_metrics_ingredients_not_attached():
+
+    y_pred = torch.randint(0, 2, size=(15, 10, 4)).float()
+    y = torch.randint(0, 2, size=(15, 10, 4)).long()
+
+    def update_fn(engine, batch):
+        y_pred, y = batch
+        return y_pred, y
+
+    evaluator = Engine(update_fn)
+
+    precision = Precision(average=False)
+    recall = Recall(average=False)
+    F1 = precision * recall * 2 / (precision + recall + 1e-20)
+    F1 = MetricsLambda(lambda t: torch.mean(t).item(), F1)
+
+    F1.attach(evaluator, "F1")
+
+    def data(y_pred, y):
+        for i in range(y_pred.shape[0]):
+            yield (y_pred[i], y[i])
+
+    d = data(y_pred, y)
+    state = evaluator.run(d, max_epochs=1)
+
+    assert set(state.metrics.keys()) == set(["F1"])
+
+
+def test_recursive_attachment():
+
+    def _test(composed_metric, metric_name, compute_true_value_fn):
+
+        metrics = {
+            metric_name: composed_metric,
+        }
+
+        y_pred = torch.randint(0, 2, size=(15, 10, 4)).float()
+        y = torch.randint(0, 2, size=(15, 10, 4)).long()
+
+        def update_fn(engine, batch):
+            y_pred, y = batch
+            return y_pred, y
+
+        validator = Engine(update_fn)
+
+        for name, metric in metrics.items():
+            metric.attach(validator, name)
+
+        def data(y_pred, y):
+            for i in range(y_pred.shape[0]):
+                yield (y_pred[i], y[i])
+
+        d = data(y_pred, y)
+        state = validator.run(d, max_epochs=1)
+
+        assert set(state.metrics.keys()) == set([metric_name, ])
+        np_y_pred = y_pred.numpy().ravel()
+        np_y = y.numpy().ravel()
+        assert state.metrics[metric_name].item() == approx(compute_true_value_fn(np_y_pred, np_y))
+
+    precision_1 = Precision()
+    precision_2 = Precision()
+    summed_precision = precision_1 + precision_2
+
+    def compute_true_summed_precision(y_pred, y):
+        p1 = precision_score(y, y_pred)
+        p2 = precision_score(y, y_pred)
+        return p1 + p2
+
+    _test(summed_precision, "summed precision", compute_true_value_fn=compute_true_summed_precision)
+
+    precision_1 = Precision()
+    precision_2 = Precision()
+    mean_precision = (precision_1 + precision_2) / 2
+
+    def compute_true_mean_precision(y_pred, y):
+        p1 = precision_score(y, y_pred)
+        p2 = precision_score(y, y_pred)
+        return (p1 + p2) * 0.5
+
+    _test(mean_precision, "mean precision", compute_true_value_fn=compute_true_mean_precision)
+
+    precision_1 = Precision()
+    precision_2 = Precision()
+    some_metric = 2.0 + 0.2 * (precision_1 * precision_2 + precision_1 - precision_2) ** 0.5
+
+    def compute_true_somemetric(y_pred, y):
+        p1 = precision_score(y, y_pred)
+        p2 = precision_score(y, y_pred)
+        return 2.0 + 0.2 * (p1 * p2 + p1 - p2) ** 0.5
+
+    _test(some_metric, "some metric", compute_true_somemetric)