Repository: continualai/avalanche
Issue: classes_in_this_experience fails on certain type of experiences

🐛 **Describe the bug**
Accessing the experience attribute classes_in_this_experience fails on certain types of experience (DatasetExperience in my case).

🐜 **To Reproduce**

```{python}
#!/usr/bin/env python3                                                                                      
import unittest                                                                                             
from avalanche.benchmarks.classic import SplitCIFAR10                                                       
from avalanche.benchmarks.generators import benchmark_with_validation_stream                               
                                                                                                            
class TestBug(unittest.TestCase):                                                                           
    def test_load_scenario(self):                                                                           
        scenario = SplitCIFAR10(5)                                               
        scenario = benchmark_with_validation_stream(scenario, validation_size=0.05)                         
        classes = scenario.train_stream[0].classes_in_this_experience  
```

🦋 **Additional context**
I already did a gitbisect on the above test and here is the output:

a1ee5e19362b171bbf295c8e9ba0ec46700ccf1a is the first bad commit                                           
commit a1ee5e19362b171bbf295c8e9ba0ec46700ccf1a                                                            
Author: @lrzpellegrini                                            
Date:   Fri Apr 14 17:14:13 2023 +0200                                                                     
                                                                                                           
    Simplified generics chain. Better naming of ClassificationScenario.


Changed files:
avalanche/evaluation/metrics/cumulative_accuracies.py, tests/test_high_level_generators.py, avalanche/benchmarks/generators/benchmark_generators.py

diff --git a/avalanche/benchmarks/generators/benchmark_generators.py b/avalanche/benchmarks/generators/benchmark_generators.py
index 4466f9ff..a5465d88 100644
--- a/avalanche/benchmarks/generators/benchmark_generators.py
+++ b/avalanche/benchmarks/generators/benchmark_generators.py
@@ -32,13 +32,17 @@ from typing import (
 )
 
 import torch
+from avalanche.benchmarks.scenarios.classification_scenario import \\
+    ClassificationScenario
 
 from avalanche.benchmarks.scenarios.dataset_scenario import (
     DatasetScenario,
     DatasetStream,
     FactoryBasedStream,
     StreamDef,
+    TStreamsUserDict,
 )
+from avalanche.benchmarks.scenarios.detection_scenario import DetectionScenario
 from avalanche.benchmarks.scenarios.generic_benchmark_creation import *
 from avalanche.benchmarks.scenarios import (
     StreamUserDef,
@@ -550,6 +554,32 @@ def _make_plain_experience(
     )
 
 
+def _smart_benchmark_factory(
+    original_benchmark: DatasetScenario,
+    new_streams_definitions: TStreamsUserDict,
+    complete_test_set_only: bool
+) -> DatasetScenario:
+    
+    if isinstance(original_benchmark, ClassificationScenario):
+        return ClassificationScenario(
+            stream_definitions=new_streams_definitions,
+            complete_test_set_only=complete_test_set_only
+        )
+    elif isinstance(original_benchmark, DetectionScenario):
+        return DetectionScenario(
+            stream_definitions=new_streams_definitions,
+            complete_test_set_only=complete_test_set_only
+        )
+    else:
+        # Generic scenario
+        return DatasetScenario(
+            stream_definitions=new_streams_definitions,
+            complete_test_set_only=complete_test_set_only,
+            stream_factory=FactoryBasedStream,
+            experience_factory=_make_plain_experience,
+        )
+
+
 def data_incremental_benchmark(
     benchmark_instance: DatasetScenario[TDatasetStream,
                                         TDatasetExperience,
@@ -562,6 +592,20 @@ def data_incremental_benchmark(
         [DatasetExperience[TCLDataset]],
         Sequence[TCLDataset]
     ]] = None,
+    *,
+    benchmark_factory: Optional[Callable[
+        [
+            DatasetScenario[TDatasetStream,
+                            TDatasetExperience,
+                            TCLDataset],
+            TStreamsUserDict,
+            bool
+        ], DatasetScenario[
+            DatasetStream[DatasetExperience[TCLDataset]],
+            DatasetExperience[TCLDataset],
+            TCLDataset]
+        ]
+    ] = _smart_benchmark_factory,
     experience_factory: Optional[Callable[
         [DatasetStream[DatasetExperience[TCLDataset]], int], 
         DatasetExperience[TCLDataset]
@@ -616,8 +660,14 @@ def data_incremental_benchmark(
         A good starting to understand the mechanism is to look at the
         implementation of the standard splitting function
         :func:`fixed_size_experience_split_strategy`.
-    :param experience_factory: The experience factory.
-        Defaults to :class:`DatasetExperience`.
+    :param benchmark_factory: The scenario factory. Defaults to 
+        `_smart_experience_factory`, which will try to create a benchmark of the
+        same class of the originating one. Can be None, in which case a generic
+        :class:`DatasetScenario` will be used coupled with the factory defined
+        by the `experience_factory` parameter.
+    :param experience_factory: The experience factory. Ignored if
+        `scenario_factory` is not None. Otherwise, defaults to
+        :class:`DatasetExperience`.
     :return: The Data Incremental benchmark instance.
     """
 
@@ -678,8 +728,18 @@ def data_incremental_benchmark(
     complete_test_set_only = (
         benchmark_instance.complete_test_set_only
         and len(stream_definitions["test"].exps_data) == 1
-    ) 
+    )
+
+    if benchmark_factory is not None:
+        # Try to create a benchmark of the same class of the
+        # initial benchmark.
+        return benchmark_factory(
+            benchmark_instance,
+            stream_definitions,
+            complete_test_set_only
+        )
 
+    # Generic benchmark class
     if experience_factory is None:
         experience_factory = _make_plain_experience
 
@@ -852,7 +912,7 @@ def _lazy_train_val_split(
 
 
 def benchmark_with_validation_stream(
-    benchmark_instance: DatasetScenario[TSizedCLStream,
+    benchmark_instance: DatasetScenario[TDatasetStream,
                                         TDatasetExperience,
                                         TCLDataset],
     validation_size: Union[int, float] = 0.5,
@@ -864,6 +924,18 @@ def benchmark_with_validation_stream(
         Tuple[TCLDataset, TCLDataset],
     ]] = None,
     *,
+    benchmark_factory: Optional[Callable[
+        [
+            DatasetScenario[TDatasetStream,
+                            TDatasetExperience,
+                            TCLDataset],
+            TStreamsUserDict,
+            bool
+        ], DatasetScenario[
+                DatasetStream[DatasetExperience[TCLDataset]],
+                DatasetExperience[TCLDataset],
+                TCLDataset]]
+    ] = _smart_benchmark_factory,
     experience_factory: Optional[Callable[
         [DatasetStream[DatasetExperience[TCLDataset]], int],
         DatasetExperience[TCLDataset]
@@ -928,8 +1000,14 @@ def benchmark_with_validation_stream(
         A good starting to understand the mechanism is to look at the
         implementation of the standard splitting function
         :func:`random_validation_split_strategy`.
-    :param experience_factory: The experience factory. Defaults to
-        :class:`GenericExperience`.
+    :param benchmark_factory: The scenario factory. Defaults to 
+        `_smart_experience_factory`, which will try to create a benchmark of the
+        same class of the originating one. Can be None, in which case a generic
+        :class:`DatasetScenario` will be used coupled with the factory defined
+        by the `experience_factory` parameter.
+    :param experience_factory: The experience factory. Ignored if
+        `scenario_factory` is not None. Otherwise, defaults to
+        :class:`DatasetExperience`.
     :param lazy_splitting: If True, the stream will be split in a lazy way.
         If False, the stream will be split immediately. Defaults to None, which
         means that the stream will be split in a lazy or non-lazy way depending
@@ -971,7 +1049,7 @@ def benchmark_with_validation_stream(
             f"benchmark instance"
         )
 
-    stream: TSizedCLStream = streams[input_stream]
+    stream: TDatasetStream = streams[input_stream]
 
     if lazy_splitting is None:
         split_lazily = original_stream_definitions[input_stream].is_lazy
@@ -1026,6 +1104,16 @@ def benchmark_with_validation_stream(
 
     complete_test_set_only = benchmark_instance.complete_test_set_only
 
+    if benchmark_factory is not None:
+        # Try to create a benchmark of the same class of the
+        # initial benchmark.
+        return benchmark_factory(
+            benchmark_instance,
+            stream_definitions,
+            complete_test_set_only
+        )
+
+    # Generic benchmark class
     if experience_factory is None:
         experience_factory = _make_plain_experience
 
diff --git a/avalanche/evaluation/metrics/cumulative_accuracies.py b/avalanche/evaluation/metrics/cumulative_accuracies.py
index db5e46af..4f96a522 100644
--- a/avalanche/evaluation/metrics/cumulative_accuracies.py
+++ b/avalanche/evaluation/metrics/cumulative_accuracies.py
@@ -30,7 +30,7 @@ if TYPE_CHECKING:
     from avalanche.training.templates import SupervisedTemplate
 
 
-class CumulativeAccuracy(Metric[float]):
+class CumulativeAccuracy(Metric[Dict[int, float]]):
     """
     Metric used by the CumulativeAccuracyPluginMetric,
     holds a dictionnary of per-task cumulative accuracies
@@ -81,7 +81,7 @@ class CumulativeAccuracy(Metric[float]):
                 true_positives / total_patterns, total_patterns
             )
 
-    def result(self) -> float:
+    def result(self) -> Dict[int, float]:
         """Retrieves the running accuracy.
 
         Calling this method will not change the internal state of the metric.
@@ -100,7 +100,9 @@ class CumulativeAccuracy(Metric[float]):
             self._mean_accuracy[t].reset()
 
 
-class CumulativeAccuracyPluginMetric(GenericPluginMetric):
+class CumulativeAccuracyPluginMetric(
+    GenericPluginMetric[Dict[int, float], CumulativeAccuracy]
+):
     def __init__(self, reset_at="stream", emit_at="stream", mode="eval"):
         """
         Creates the CumulativeAccuracy plugin metric,
@@ -109,10 +111,10 @@ class CumulativeAccuracyPluginMetric(GenericPluginMetric):
         features for class-incremental learning"
         Soutif et. al, https://arxiv.org/abs/2106.11930
         """
-        self._accuracy = CumulativeAccuracy()
+
         self.classes_seen_so_far = set()
         self.classes_splits = {}
-        super().__init__(self._accuracy, 
+        super().__init__(CumulativeAccuracy(), 
                          reset_at=reset_at, 
                          emit_at=emit_at, 
                          mode=mode)
@@ -143,19 +145,21 @@ class CumulativeAccuracyPluginMetric(GenericPluginMetric):
         self.classes_seen_so_far = self.classes_seen_so_far.union(new_classes)
         self.classes_splits[task_id] = self.classes_seen_so_far
 
-    def reset(self, strategy=None) -> None:
+    def reset(self) -> None:
         self._metric.reset()
 
-    def result(self, strategy=None) -> float:
+    def result(self) -> Dict[int, float]:
         return self._metric.result()
 
     def update(self, strategy):
-        self._accuracy.update(self.classes_splits, 
-                              strategy.mb_output, 
-                              strategy.mb_y)
+        self._metric.update(
+            self.classes_splits,
+            strategy.mb_output,
+            strategy.mb_y)
 
     def _package_result(self, strategy: "SupervisedTemplate") -> "MetricResult":
-        metric_value = self.result(strategy)
+        assert strategy.experience is not None
+        metric_value = self.result()
         plot_x_position = strategy.clock.train_iterations
 
         phase_name, task_label = phase_and_task(strategy)
@@ -182,7 +186,9 @@ class CumulativeAccuracyPluginMetric(GenericPluginMetric):
         return "CumulativeAccuracy"
 
 
-class CumulativeForgettingPluginMetric(GenericPluginMetric):
+class CumulativeForgettingPluginMetric(
+    GenericPluginMetric[Dict[int, float], CumulativeAccuracy]
+):
     """
     The CumulativeForgetting metric, describing the accuracy loss
     detected for a certain experience.
@@ -199,7 +205,6 @@ class CumulativeForgettingPluginMetric(GenericPluginMetric):
         """
         Creates an instance of the CumulativeForgetting metric.
         """
-        self._accuracy = CumulativeAccuracy()
 
         self.classes_splits = {}
         self.classes_seen_so_far = set()
@@ -209,7 +214,7 @@ class CumulativeForgettingPluginMetric(GenericPluginMetric):
 
         self.train_task_id = None
 
-        super().__init__(self._accuracy, 
+        super().__init__(CumulativeAccuracy(), 
                          reset_at=reset_at, 
                          emit_at=emit_at, 
                          mode=mode)
@@ -247,15 +252,16 @@ class CumulativeForgettingPluginMetric(GenericPluginMetric):
         else:
             self.train_task_id = experience.current_experience
 
-    def reset(self, strategy=None):
-        self._accuracy.reset()
+    def reset(self):
+        self._metric.reset()
 
-    def result(self, strategy=None) -> float:
+    def result(self) -> Dict[int, float]:
         forgetting = self._compute_forgetting()
         return forgetting
 
     def _package_result(self, strategy: "SupervisedTemplate") -> "MetricResult":
-        metric_value = self.result(strategy)
+        assert strategy.experience is not None
+        metric_value = self.result()
         plot_x_position = strategy.clock.train_iterations
 
         phase_name, task_label = phase_and_task(strategy)
@@ -279,12 +285,13 @@ class CumulativeForgettingPluginMetric(GenericPluginMetric):
         return metrics
 
     def update(self, strategy):
-        self._accuracy.update(self.classes_splits, 
-                              strategy.mb_output, 
-                              strategy.mb_y)
+        self._metric.update(
+            self.classes_splits, 
+            strategy.mb_output, 
+            strategy.mb_y)
 
     def _compute_forgetting(self):
-        for t, item in self._accuracy.result().items():
+        for t, item in self._metric.result().items():
             if t not in self.initial:
                 self.initial[t] = item
             else:
diff --git a/tests/test_high_level_generators.py b/tests/test_high_level_generators.py
index 43e1d974..c0147490 100644
--- a/tests/test_high_level_generators.py
+++ b/tests/test_high_level_generators.py
@@ -500,6 +500,12 @@ class HighLevelGeneratorTests(unittest.TestCase):
         self.assertTrue(torch.equal(test_x, mb[0]))
         self.assertTrue(torch.equal(test_y, mb[1]))
 
+        # Regression test for #1371
+        self.assertEquals(
+            [0],
+            valid_benchmark.train_stream[0].classes_in_this_experience
+        )
+
     def test_lazy_benchmark_with_validation_stream_fixed_size(self):
         lazy_options = [None, True, False]
         for lazy_option in lazy_options: