Repository: ContinualAI/avalanche

Issue:
bug for stream-level metrics in periodic eval
periodic eval is broken in some subtle cases. Instead of calling `eval` on the entire stream, it calls it multiple times for each experience, breaking stream-level metrics.

Reproduce:
```
from avalanche.benchmarks import SplitMNIST
from avalanche.training import Naive
from avalanche.models import SimpleMLP

if __name__ == '__main__':
    benchmark = SplitMNIST(5)
    model = SimpleMLP()
    strat = Naive(model, None, eval_mb_size=512, eval_every=1)

    stream = benchmark.test_stream
    strat.train(stream[0], eval_streams=list(stream[:2]))
```

outputs:
```
-- >> Start of training phase << --
-- >> Start of eval phase << --
-- Starting eval on experience 0 (Task 0) from test stream --
100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [00:05<00:00,  1.08s/it]
> Eval on experience 0 (Task 0) from test stream ended.
        Loss_Exp/eval_phase/test_stream/Task000/Exp000 = 2.3881
        Top1_Acc_Exp/eval_phase/test_stream/Task000/Exp000 = 0.0033
-- >> End of eval phase << --
        Loss_Stream/eval_phase/test_stream/Task000 = 2.3881
        Top1_Acc_Stream/eval_phase/test_stream/Task000 = 0.0033
-- >> Start of eval phase << --
-- Starting eval on experience 1 (Task 0) from test stream --
100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:01<00:00,  2.12it/s]
> Eval on experience 1 (Task 0) from test stream ended.
        Loss_Exp/eval_phase/test_stream/Task000/Exp001 = 2.3041
        Top1_Acc_Exp/eval_phase/test_stream/Task000/Exp001 = 0.2129
-- >> End of eval phase << --
        Loss_Stream/eval_phase/test_stream/Task000 = 2.3041
        Top1_Acc_Stream/eval_phase/test_stream/Task000 = 0.2129
```

which means it's calling `eval` two times instead of a single one with the entire stream.

Comments:

Patch:
diff --git a/avalanche/benchmarks/utils/data.py b/avalanche/benchmarks/utils/data.py
index 533d79ee..9c11f60f 100644
--- a/avalanche/benchmarks/utils/data.py
+++ b/avalanche/benchmarks/utils/data.py
@@ -101,6 +101,10 @@ class AvalancheDataset(FlatData):
                 DeprecationWarning,
             )
 
+        if issubclass(type(datasets), TorchDataset) or  \
+                issubclass(type(datasets), AvalancheDataset):
+            datasets = [datasets]
+
         # NOTES on implementation:
         # - raw datasets operations are implemented by _FlatData
         # - data attributes are implemented by DataAttribute
diff --git a/avalanche/training/templates/base.py b/avalanche/training/templates/base.py
index c0899806..532e8332 100644
--- a/avalanche/training/templates/base.py
+++ b/avalanche/training/templates/base.py
@@ -1,4 +1,5 @@
 import warnings
+from collections import defaultdict
 from typing import Iterable, Sequence, Optional, Union, List
 
 import torch
@@ -90,6 +91,9 @@ class BaseTemplate:
         :param eval_streams: sequence of streams for evaluation.
             If None: use training experiences for evaluation.
             Use [] if you do not want to evaluate during training.
+            Experiences in `eval_streams` are grouped by stream name
+            when calling `eval`. If you use multiple streams, they must
+            have different names.
         """
         self.is_training = True
         self._stop_training = False
@@ -102,7 +106,7 @@ class BaseTemplate:
             experiences = [experiences]
         if eval_streams is None:
             eval_streams = [experiences]
-        self._eval_streams = eval_streams
+        self._eval_streams = _group_experiences_by_stream(eval_streams)
 
         self._before_training(**kwargs)
 
@@ -244,3 +248,20 @@ class BaseTemplate:
 
     def _after_eval_exp(self, **kwargs):
         trigger_plugins(self, "after_eval_exp", **kwargs)
+
+
+def _group_experiences_by_stream(eval_streams):
+    exps = []
+    # First, we unpack the list of experiences.
+    for exp in eval_streams:
+        if isinstance(exp, Iterable):
+            exps.extend(exp)
+        else:
+            exps.append(exp)
+    # Then, we group them by stream.
+    exps_by_stream = defaultdict(list)
+    for exp in exps:
+        sname = exp.origin_stream.name
+        exps_by_stream[sname].append(exp)
+    # Finally, we return a list of lists.
+    return list(exps_by_stream.values())


Test patch:
diff --git a/tests/test_avalanche_dataset.py b/tests/test_avalanche_dataset.py
index e7cf460b..67ccdecc 100644
--- a/tests/test_avalanche_dataset.py
+++ b/tests/test_avalanche_dataset.py
@@ -56,6 +56,17 @@ class FrozenTransformGroupsCenterCrop:
 
 
 class AvalancheDatasetTests(unittest.TestCase):
+
+    def test_avalanche_dataset_creation_without_list(self):
+        dataset_mnist = load_image_benchmark()
+        dataset = AvalancheDataset(dataset_mnist)
+        self.assertIsInstance(dataset, AvalancheDataset)
+        self.assertEqual(len(dataset_mnist), len(dataset))
+
+        dataset = AvalancheDataset(dataset)
+        self.assertIsInstance(dataset, AvalancheDataset)
+        self.assertEqual(len(dataset_mnist), len(dataset))
+
     def test_disallowed_attribute_name(self):
         d_sz = 3
         xdata = torch.rand(d_sz, 2)
diff --git a/tests/training/test_strategies.py b/tests/training/test_strategies.py
index 84fae146..a49b2d20 100644
--- a/tests/training/test_strategies.py
+++ b/tests/training/test_strategies.py
@@ -48,11 +48,37 @@ from avalanche.training.supervised.icarl import ICaRL
 from avalanche.training.supervised.joint_training import AlreadyTrainedError
 from avalanche.training.supervised.strategy_wrappers import PNNStrategy
 from avalanche.training.templates import SupervisedTemplate
+from avalanche.training.templates.base import _group_experiences_by_stream
 from avalanche.training.utils import get_last_fc_layer
 from tests.unit_tests_utils import get_fast_benchmark, get_device
 
 
 class BaseStrategyTest(unittest.TestCase):
+    def test_eval_streams_normalization(self):
+        benchmark = get_fast_benchmark()
+        train_len = len(benchmark.train_stream)
+        test_len = len(benchmark.test_stream)
+
+        res = _group_experiences_by_stream(benchmark.test_stream)
+        assert len(res) == 1
+        assert len(res[0]) == test_len
+
+        res = _group_experiences_by_stream([benchmark.test_stream])
+        assert len(res) == 1
+        assert len(res[0]) == test_len
+
+        res = _group_experiences_by_stream(
+            [*benchmark.test_stream, *benchmark.train_stream])
+        assert len(res) == 2
+        assert len(res[0]) == test_len
+        assert len(res[1]) == train_len
+
+        res = _group_experiences_by_stream(
+            [benchmark.test_stream, benchmark.train_stream])
+        assert len(res) == 2
+        assert len(res[0]) == test_len
+        assert len(res[1]) == train_len
+
     def test_periodic_eval(self):
         model = SimpleMLP(input_size=6, hidden_size=10)
         model.classifier = IncrementalClassifier(model.classifier.in_features)