Repository: continualai/avalanche
Issue: Function "scenario.classes_in_exp_range" Stopped Working

Executing the following code:
```
from os.path import expanduser
from avalanche.benchmarks import nc_benchmark
from avalanche.benchmarks.datasets import CIFAR100


train_set = CIFAR100(expanduser("~") + "/.avalanche/data/cifar100/", train=True, download=True, )
test_set = CIFAR100(expanduser("~") + "/.avalanche/data/cifar100/", train=False, download=True, )

scenario = nc_benchmark(
    train_dataset=train_set,
    test_dataset=test_set,
    n_experiences=5,
    task_labels=False, seed=1234,
    shuffle=False)

print(scenario.classes_in_exp_range(0, 3))
```
results in the error:
```
Traceback (most recent call last):
  File "/home/rosasco/iCaRL_Avalanche/test.py", line 17, in <module>
    print(scenario.classes_in_exp_range(0, 3))
  File "/home/rosasco/miniconda3/envs/thesis/lib/python3.8/site-packages/avalanche/benchmarks/scenarios/new_classes/nc_scenario.py", line 465, in classes_in_exp_range
    self.classes_in_experience[exp_start:exp_end]
  File "/home/rosasco/miniconda3/envs/thesis/lib/python3.8/site-packages/avalanche/benchmarks/scenarios/generic_cl_scenario.py", line 489, in __getitem__
    return self._default_lcie[stream_name_or_exp_id]
  File "/home/rosasco/miniconda3/envs/thesis/lib/python3.8/site-packages/avalanche/benchmarks/scenarios/generic_cl_scenario.py", line 504, in __getitem__
    return set(self._scenario.stream_definitions[self._stream]
AttributeError: 'list' object has no attribute 'targets'
```

Changed files:
tests/test_nc_sit_scenario.py, avalanche/benchmarks/scenarios/generic_cl_scenario.py, avalanche/benchmarks/scenarios/new_classes/nc_scenario.py

diff --git a/avalanche/benchmarks/scenarios/generic_cl_scenario.py b/avalanche/benchmarks/scenarios/generic_cl_scenario.py
index 74788c18..3fd87e7d 100644
--- a/avalanche/benchmarks/scenarios/generic_cl_scenario.py
+++ b/avalanche/benchmarks/scenarios/generic_cl_scenario.py
@@ -10,6 +10,7 @@ from torch.utils.data.dataset import Dataset
 from avalanche.benchmarks.scenarios.generic_definitions import \\
     TExperience, ScenarioStream, TScenarioStream, Experience, TScenario
 from avalanche.benchmarks.utils import AvalancheDataset
+from avalanche.benchmarks.utils.dataset_utils import manage_advanced_indexing
 
 TGenericCLScenario = TypeVar('TGenericCLScenario', bound='GenericCLScenario')
 TGenericExperience = TypeVar('TGenericExperience', bound='GenericExperience')
@@ -485,7 +486,7 @@ class LazyStreamClassesInExps(Mapping[str, Sequence[Set[int]]]):
         warnings.warn(
             'Using classes_in_experience[exp_id] is deprecated. '
             'Consider using classes_in_experience[stream_name][exp_id]'
-            'instead.')
+            'instead.', stacklevel=2)
         return self._default_lcie[stream_name_or_exp_id]
 
     def __iter__(self):
@@ -501,14 +502,25 @@ class LazyClassesInExps(Sequence[Set[int]]):
         return len(self._scenario.streams[self._stream])
 
     def __getitem__(self, exp_id) -> Set[int]:
-        return set(self._scenario.stream_definitions[self._stream]
-                   .exps_data[exp_id].targets)
+        return manage_advanced_indexing(
+            exp_id, self._get_single_exp_classes,
+            len(self), LazyClassesInExps._slice_collate)
 
     def __str__(self):
         return '[' + \\
                ', '.join([str(self[idx]) for idx in range(len(self))]) + \\
                ']'
 
+    def _get_single_exp_classes(self, exp_id):
+        return set(self._scenario.stream_definitions[self._stream]
+                   .exps_data[exp_id].targets)
+
+    @staticmethod
+    def _slice_collate(*classes_in_exps: Set[int]):
+        return [
+            list(x) for x in classes_in_exps
+        ]
+
 
 def _get_slice_ids(slice_definition: Union[int, slice, Iterable[int]],
                    sliceable_len: int) -> List[int]:
diff --git a/avalanche/benchmarks/scenarios/new_classes/nc_scenario.py b/avalanche/benchmarks/scenarios/new_classes/nc_scenario.py
index 2b85ce04..27fd27b8 100644
--- a/avalanche/benchmarks/scenarios/new_classes/nc_scenario.py
+++ b/avalanche/benchmarks/scenarios/new_classes/nc_scenario.py
@@ -457,12 +457,12 @@ class NCScenario(GenericCLScenario['NCExperience']):
         if exp_end is None:
             return [
                 item for sublist in
-                self.classes_in_experience[exp_start:]
+                self.classes_in_experience['train'][exp_start:]
                 for item in sublist]
 
         return [
             item for sublist in
-            self.classes_in_experience[exp_start:exp_end]
+            self.classes_in_experience['train'][exp_start:exp_end]
             for item in sublist]
 
 
diff --git a/tests/test_nc_sit_scenario.py b/tests/test_nc_sit_scenario.py
index 0f7f7e96..4905d488 100644
--- a/tests/test_nc_sit_scenario.py
+++ b/tests/test_nc_sit_scenario.py
@@ -2,12 +2,13 @@ import unittest
 
 from os.path import expanduser
 
+import random
 from PIL.Image import Image
 from torch import Tensor
 from torchvision.datasets import MNIST
 from torchvision.transforms import ToTensor
 
-from avalanche.benchmarks.datasets import CIFAR100
+from avalanche.benchmarks.datasets import CIFAR100, get_default_dataset_location
 from avalanche.benchmarks.scenarios.new_classes import NCExperience
 from avalanche.benchmarks.utils import AvalancheSubset, AvalancheDataset
 from avalanche.benchmarks.scenarios.new_classes.nc_utils import \\
@@ -356,6 +357,53 @@ class SITTests(unittest.TestCase):
         ds_test_train = scenario.test_stream[0].dataset.train()
         self.assertIsInstance(ds_test_train[0][0], Tensor)
 
+    def test_nc_benchmark_classes_in_exp_range(self):
+        train_set = CIFAR100(get_default_dataset_location('cifar100'),
+                             train=True, download=True)
+
+        test_set = CIFAR100(get_default_dataset_location('cifar100'),
+                            train=False, download=True)
+
+        benchmark_instance = nc_benchmark(
+            train_dataset=train_set,
+            test_dataset=test_set,
+            n_experiences=5,
+            task_labels=False,
+            seed=1234,
+            shuffle=False)
+
+        cie_data = benchmark_instance.classes_in_exp_range(0, None)
+        self.assertEqual(5, len(cie_data))
+
+        for i in range(5):
+            expected = set(range(i*20, (i+1)*20))
+            self.assertSetEqual(expected, set(cie_data[i]))
+
+        cie_data = benchmark_instance.classes_in_exp_range(1, 4)
+        self.assertEqual(3, len(cie_data))
+
+        for i in range(1, 3):
+            expected = set(range(i * 20, (i + 1) * 20))
+            self.assertSetEqual(expected, set(cie_data[i-1]))
+
+        random_class_order = list(range(100))
+        random.shuffle(random_class_order)
+        benchmark_instance = nc_benchmark(
+            train_dataset=train_set,
+            test_dataset=test_set,
+            n_experiences=5,
+            task_labels=False,
+            seed=1234,
+            fixed_class_order=random_class_order,
+            shuffle=False)
+
+        cie_data = benchmark_instance.classes_in_exp_range(0, None)
+        self.assertEqual(5, len(cie_data))
+
+        for i in range(5):
+            expected = set(random_class_order[i * 20: (i + 1) * 20])
+            self.assertSetEqual(expected, set(cie_data[i]))
+
 
 if __name__ == '__main__':
     unittest.main()