Repository: Project-MONAI/MONAI

Issue:
Potentially insufficient shuffling in SmartCacheDataset
**Describe the bug**
By design, in each training epoch, SmartCacheDataset only presents `cache_num` of data points to the data loader where data shuffling is achieved.

There's a potential of insufficient shuffling at the initial training epoches, when `cache_num`/`cache_rate` is small, and the cache replacements are not fast enough.

**Expected behavior**
would be great to have a global data list shuffling before running the training epochs with the cached subsets.


**Additional context**
https://github.com/Project-MONAI/MONAI/blob/466a0bf629cbf98920935dd6cf4512dc5c908a8f/monai/data/dataset.py#L628



Discussion:

Patch:
diff --git a/monai/data/dataset.py b/monai/data/dataset.py
--- a/monai/data/dataset.py
+++ b/monai/data/dataset.py
@@ -547,7 +547,7 @@ def __getitem__(self, index):
         return data
 
 
-class SmartCacheDataset(CacheDataset):
+class SmartCacheDataset(Randomizable, CacheDataset):
     """
     Re-implementation of the SmartCache mechanism in NVIDIA Clara-train SDK.
     At any time, the cache pool only keeps a subset of the whole dataset. In each epoch, only the items
@@ -594,6 +594,8 @@ def __init__(
         num_init_workers: Optional[int] = None,
         num_replace_workers: Optional[int] = None,
         progress: bool = True,
+        shuffle: bool = True,
+        seed: int = 0,
     ) -> None:
         """
         Args:
@@ -609,8 +611,14 @@ def __init__(
             num_replace_workers: the number of worker threads to prepare the replacement cache for every epoch.
                 If num_replace_workers is None then the number returned by os.cpu_count() is used.
             progress: whether to display a progress bar when caching for the first epoch.
+            shuffle: whether to shuffle the whole data list before preparing the cache content for first epoch.
+            seed: random seed if shuffle is `True`, default to `0`.
 
         """
+        if shuffle:
+            self.set_random_state(seed=seed)
+            self.randomize(data)
+
         super().__init__(data, transform, cache_num, cache_rate, num_init_workers, progress)
         if self._cache is None:
             self._cache = self._fill_cache()
@@ -636,6 +644,12 @@ def __init__(
 
         self._compute_data_idx()
 
+    def randomize(self, data: Sequence) -> None:
+        try:
+            self.R.shuffle(data)
+        except TypeError as e:
+            warnings.warn(f"input data can't be shuffled in SmartCacheDataset with numpy.random.shuffle(): {e}.")
+
     def _compute_data_idx(self):
         """
         Update the replacement data position in the total data.
@@ -789,6 +803,18 @@ def __len__(self):
         """
         return self.cache_num
 
+    def __getitem__(self, index):
+        """
+        Raise exception if didn't call the expected APIs in SmartCacheDataset.
+
+        """
+        if not self.is_started():
+            raise RuntimeError(
+                "if using MONAI workflows, please add `SmartCacheHandler` to the handler list of trainer,"
+                "otherwise, please make sure to call `start()`, `update_cache()`, `shutdown()` during training."
+            )
+        return super().__getitem__(index)
+
 
 class ZipDataset(Dataset):
     """


Test patch:
diff --git a/tests/test_handler_smartcache.py b/tests/test_handler_smartcache.py
--- a/tests/test_handler_smartcache.py
+++ b/tests/test_handler_smartcache.py
@@ -36,7 +36,7 @@ def _train_func(engine, batch):
         engine = Engine(_train_func)
 
         # set up testing handler
-        dataset = SmartCacheDataset(data, transform=None, replace_rate=0.2, cache_num=5)
+        dataset = SmartCacheDataset(data, transform=None, replace_rate=0.2, cache_num=5, shuffle=False)
         data_loader = torch.utils.data.DataLoader(dataset, batch_size=5)
         SmartCacheHandler(dataset).attach(engine)
 
diff --git a/tests/test_smartcachedataset.py b/tests/test_smartcachedataset.py
--- a/tests/test_smartcachedataset.py
+++ b/tests/test_smartcachedataset.py
@@ -63,13 +63,39 @@ def test_shape(self, replace_rate, num_replace_workers, transform):
                 dataset.start()
                 for _ in range(3):
                     dataset.update_cache()
-                    self.assertIsNotNone(dataset._cache[15])
-                    if isinstance(dataset._cache[15]["image"], np.ndarray):
-                        np.testing.assert_allclose(dataset._cache[15]["image"], dataset._cache[15]["label"])
+                    self.assertIsNotNone(dataset[15])
+                    if isinstance(dataset[15]["image"], np.ndarray):
+                        np.testing.assert_allclose(dataset[15]["image"], dataset[15]["label"])
                     else:
-                        self.assertIsInstance(dataset._cache[15]["image"], str)
+                        self.assertIsInstance(dataset[15]["image"], str)
                 dataset.shutdown()
 
+    def test_shuffle(self):
+        test_data = [{"image": f"test_image{i}.nii.gz"} for i in range(20)]
+        dataset = SmartCacheDataset(
+            data=test_data,
+            transform=None,
+            replace_rate=0.1,
+            cache_num=16,
+            num_init_workers=4,
+            num_replace_workers=4,
+            shuffle=True,
+            seed=123,
+        )
+
+        dataset.start()
+        for i in range(3):
+            dataset.update_cache()
+
+            if i == 0:
+                self.assertEqual(dataset[15]["image"], "test_image18.nii.gz")
+            elif i == 1:
+                self.assertEqual(dataset[15]["image"], "test_image13.nii.gz")
+            else:
+                self.assertEqual(dataset[15]["image"], "test_image5.nii.gz")
+
+        dataset.shutdown()
+
 
 if __name__ == "__main__":
     unittest.main()