Repository: Project-MONAI/MONAI

Issue:
Fix random train/val split in Datasets
**Is your feature request related to a problem? Please describe.**
Now we use `if random number < 0.2, XXX` logic to select validation dataset which can't guarantee val image number for a small dataset.



Discussion:

Resolving patch:
diff --git a/monai/apps/datasets.py b/monai/apps/datasets.py
--- a/monai/apps/datasets.py
+++ b/monai/apps/datasets.py
@@ -13,7 +13,9 @@
 import sys
 from typing import Any, Callable, Dict, List, Optional, Sequence, Union
 
-from monai.apps.utils import download_and_extract
+import numpy as np
+
+from monai.apps.utils import download_and_extract, split_dataset
 from monai.data import CacheDataset, load_decathlon_datalist, load_decathlon_properties
 from monai.transforms import LoadNiftid, LoadPNGd, Randomizable
 from monai.utils import ensure_tuple
@@ -151,11 +153,10 @@ class DecathlonDataset(Randomizable, CacheDataset):
             for further usage, use `AddChanneld` or `AsChannelFirstd` to convert the shape to [C, H, W, D].
         download: whether to download and extract the Decathlon from resource link, default is False.
             if expected file already exists, skip downloading even set it to True.
+        val_frac: percentage of of validation fraction in the whole dataset, default is 0.2.
             user can manually copy tar file or dataset folder to the root directory.
-        seed: random seed to randomly split `training`, `validation` and `test` datasets, default is 0.
-        val_frac: percentage of of validation fraction from the `training` section, default is 0.2.
-            Decathlon data only contains `training` section with labels and `test` section without labels,
-            so randomly select fraction from the `training` section as the `validation` section.
+        seed: random seed to randomly shuffle the datalist before splitting into training and validation, default is 0.
+            note to set same seed for `training` and `validation` sections.
         cache_num: number of items to be cached. Default is `sys.maxsize`.
             will take the minimum of (cache_num, data_length x cache_rate, data_length).
         cache_rate: percentage of cached data in total, default is 1.0 (cache all).
@@ -181,11 +182,11 @@ class DecathlonDataset(Randomizable, CacheDataset):
             ]
         )
 
-        data = DecathlonDataset(
-            root_dir="./", task="Task09_Spleen", transform=transform, section="validation", download=True
+        val_data = DecathlonDataset(
+            root_dir="./", task="Task09_Spleen", transform=transform, section="validation", seed=12345, download=True
         )
 
-        print(data[0]["image"], data[0]["label"])
+        print(val_data[0]["image"], val_data[0]["label"])
 
     """
 
@@ -243,6 +244,7 @@ def __init__(
             raise RuntimeError(
                 f"Cannot find dataset directory: {dataset_dir}, please use download=True to download it."
             )
+        self.indices: np.ndarray = np.array([])
         data = self._generate_data_list(dataset_dir)
         # as `release` key has typo in Task04 config file, ignore it.
         property_keys = [
@@ -259,8 +261,15 @@ def __init__(
         self._properties = load_decathlon_properties(os.path.join(dataset_dir, "dataset.json"), property_keys)
         super().__init__(data, transform, cache_num=cache_num, cache_rate=cache_rate, num_workers=num_workers)
 
-    def randomize(self, data: Optional[Any] = None) -> None:
-        self.rann = self.R.random()
+    def get_indices(self) -> np.ndarray:
+        """
+        Get the indices of datalist used in this dataset.
+
+        """
+        return self.indices
+
+    def randomize(self, data: List[int]) -> None:
+        self.R.shuffle(data)
 
     def get_properties(self, keys: Optional[Union[Sequence[str], str]] = None):
         """
@@ -278,17 +287,128 @@ def get_properties(self, keys: Optional[Union[Sequence[str], str]] = None):
     def _generate_data_list(self, dataset_dir: str) -> List[Dict]:
         section = "training" if self.section in ["training", "validation"] else "test"
         datalist = load_decathlon_datalist(os.path.join(dataset_dir, "dataset.json"), True, section)
-        if section == "test":
+        return self._split_datalist(datalist)
+
+    def _split_datalist(self, datalist: List[Dict]) -> List[Dict]:
+        if self.section == "test":
             return datalist
         else:
-            data = list()
-            for i in datalist:
-                self.randomize()
-                if self.section == "training":
-                    if self.rann < self.val_frac:
-                        continue
+            length = len(datalist)
+            indices = np.arange(length)
+            self.randomize(indices)
+
+            val_length = int(length * self.val_frac)
+            if self.section == "training":
+                self.indices = indices[val_length:]
+            else:
+                self.indices = indices[:val_length]
+
+            return [datalist[i] for i in self.indices]
+
+
+class CVDecathlonDataset:
+    """
+    Cross vaidation dataset based on the general `DecathlonDataset`.
+
+    Args:
+        root_dir: user's local directory for caching and loading the MSD datasets.
+        task: which task to download and execute: one of list ("Task01_BrainTumour", "Task02_Heart",
+            "Task03_Liver", "Task04_Hippocampus", "Task05_Prostate", "Task06_Lung", "Task07_Pancreas",
+            "Task08_HepaticVessel", "Task09_Spleen", "Task10_Colon").
+        transform: transforms to execute operations on input data. the default transform is `LoadNiftid`,
+            which can load Nifti format data into numpy array with [H, W, D] or [H, W, D, C] shape.
+            for further usage, use `AddChanneld` or `AsChannelFirstd` to convert the shape to [C, H, W, D].
+        download: whether to download and extract the Decathlon from resource link, default is False.
+            if expected file already exists, skip downloading even set it to True.
+        seed: random seed to randomly shuffle the datalist before splitting into N folds, default is 0.
+            note to set same seed for all the related datasets(`training` and `validation` sections in all N folds).
+        nsplits: number of folds, split the training data into N folds. each fold is then used once as a
+            validation while the N - 1 remaining folds form the training set.
+        cache_num: number of items to be cached. Default is `sys.maxsize`.
+            will take the minimum of (cache_num, data_length x cache_rate, data_length).
+        cache_rate: percentage of cached data in total, default is 1.0 (cache all).
+            will take the minimum of (cache_num, data_length x cache_rate, data_length).
+        num_workers: the number of worker threads to use.
+            if 0 a single thread will be used. Default is 0.
+
+    Example of 5 folds cross validation training::
+
+        # use same random seed for all the datasets
+        cvdataset = CVDecathlonDataset(root_dir="./", task="Task09_Spleen", seed=12345, nsplts=5, download=True)
+        dataset_fold0_train = cvdataset.get_dataset(fold=0, section="training")
+        dataset_fold0_val = cvdataset.get_dataset(fold=0, section="validation")
+        # execute training for fold 0 ...
+
+        dataset_fold1_train = cvdataset.get_dataset(fold=1, section="training")
+        dataset_fold1_val = cvdataset.get_dataset(fold=1, section="validation")
+        # execute training for fold 1 ...
+
+        ...
+
+        dataset_fold4_train = ...
+        # execute training for fold 4 ...
+
+    """
+
+    def __init__(
+        self,
+        root_dir: str,
+        task: str,
+        transform: Union[Sequence[Callable], Callable] = LoadNiftid(["image", "label"]),
+        download: bool = False,
+        seed: int = 0,
+        nsplits: int = 5,
+        cache_num: int = sys.maxsize,
+        cache_rate: float = 1.0,
+        num_workers: int = 0,
+    ) -> None:
+        self.root_dir = root_dir
+        self.task = task
+        self.transform = transform
+        self.download = download
+        self.seed = seed
+        if nsplits < 2:
+            raise ValueError("nplits must be greater than 1 for cross validation.")
+        self.nsplits = nsplits
+        self.cache_num = cache_num
+        self.cache_rate = cache_rate
+        self.num_workers = num_workers
+
+    def get_dataset(self, fold: int, section: str):
+        """
+        Generate dataset for specified fold index and section in the cross validation group.
+
+        Args:
+            fold: index of fold for training and validation.
+            section: expected data section, can be: `training` or `validation`.
+
+        """
+        if fold > self.nsplits - 1:
+            raise ValueError("fold index should be in [0, nsplit - 1].")
+        nsplits = self.nsplits
+
+        class _NsplitsDataset(DecathlonDataset):
+            def _split_datalist(self, datalist: List[Dict]) -> List[Dict]:
+                if section not in ["training", "validation"]:
+                    raise ValueError("section must be training or validation.")
+                length = len(datalist)
+                indices = np.arange(length)
+                self.randomize(indices)
+                fold_indices = split_dataset(nsplits, length)
+                if section == "training":
+                    self.indices = np.delete(indices, np.s_[fold_indices[fold][0] : fold_indices[fold][1]], axis=None)
                 else:
-                    if self.rann >= self.val_frac:
-                        continue
-                data.append(i)
-            return data
+                    self.indices = indices[fold_indices[fold][0] : fold_indices[fold][1]]
+                return [datalist[i] for i in self.indices]
+
+        return _NsplitsDataset(
+            root_dir=self.root_dir,
+            task=self.task,
+            section=section,
+            transform=self.transform,
+            download=self.download,
+            seed=self.seed,
+            cache_num=self.cache_num,
+            cache_rate=self.cache_rate,
+            num_workers=self.num_workers,
+        )
diff --git a/monai/apps/utils.py b/monai/apps/utils.py
--- a/monai/apps/utils.py
+++ b/monai/apps/utils.py
@@ -20,6 +20,8 @@
 from urllib.error import ContentTooShortError, HTTPError, URLError
 from urllib.request import Request, urlopen, urlretrieve
 
+import numpy as np
+
 from monai.utils import min_version, optional_import
 
 gdown, has_gdown = optional_import("gdown", "3.6")
@@ -249,3 +251,22 @@ def download_and_extract(
     """
     download_url(url=url, filepath=filepath, hash_val=hash_val, hash_type=hash_type)
     extractall(filepath=filepath, output_dir=output_dir, hash_val=hash_val, hash_type=hash_type)
+
+
+def split_dataset(nsplits: int, length: int):
+    """
+    Split dataset into N folds, return a list of indices for every fold.
+
+    Args:
+        nsplits: expected fold number.
+        length: dataset size in total.
+
+    """
+    sizes = np.full(nsplits, length // nsplits, dtype=np.int)
+    sizes[: length % nsplits] += 1
+    indices = list()
+    pos = 0
+    for size in sizes:
+        indices.append([pos, pos + size])
+        pos += size
+    return indices


Test patch:
diff --git a/tests/test_cvdecathlondataset.py b/tests/test_cvdecathlondataset.py
new file mode 100644
--- /dev/null
+++ b/tests/test_cvdecathlondataset.py
@@ -0,0 +1,75 @@
+# Copyright 2020 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import unittest
+from urllib.error import ContentTooShortError, HTTPError
+
+from monai.apps import CVDecathlonDataset
+from monai.transforms import AddChanneld, Compose, LoadNiftid, ScaleIntensityd, ToTensord
+from tests.utils import skip_if_quick
+
+
+class TestCVDecathlonDataset(unittest.TestCase):
+    @skip_if_quick
+    def test_values(self):
+        testing_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "testing_data")
+        transform = Compose(
+            [
+                LoadNiftid(keys=["image", "label"]),
+                AddChanneld(keys=["image", "label"]),
+                ScaleIntensityd(keys="image"),
+                ToTensord(keys=["image", "label"]),
+            ]
+        )
+
+        def _test_dataset(dataset):
+            self.assertEqual(len(dataset), 52)
+            self.assertTrue("image" in dataset[0])
+            self.assertTrue("label" in dataset[0])
+            self.assertTrue("image_meta_dict" in dataset[0])
+            self.assertTupleEqual(dataset[0]["image"].shape, (1, 34, 49, 41))
+
+        cvdataset = CVDecathlonDataset(
+            root_dir=testing_dir,
+            task="Task04_Hippocampus",
+            transform=transform,
+            download=True,
+            seed=12345,
+            nsplits=5,
+        )
+
+        try:  # will start downloading if testing_dir doesn't have the Decathlon files
+            data = cvdataset.get_dataset(fold=0, section="validation")
+        except (ContentTooShortError, HTTPError, RuntimeError) as e:
+            print(str(e))
+            if isinstance(e, RuntimeError):
+                # FIXME: skip MD5 check as current downloading method may fail
+                self.assertTrue(str(e).startswith("md5 check"))
+            return  # skipping this test due the network connection errors
+
+        _test_dataset(data)
+
+        # test training data for fold 0 of 5 splits
+        data = cvdataset.get_dataset(fold=0, section="training")
+        self.assertTupleEqual(data[0]["image"].shape, (1, 34, 48, 40))
+        self.assertEqual(len(data), 208)
+        # test train / validation for fold 4 of 5 splits
+        data = cvdataset.get_dataset(fold=4, section="validation")
+        self.assertTupleEqual(data[0]["image"].shape, (1, 33, 55, 29))
+        self.assertEqual(len(data), 52)
+        data = cvdataset.get_dataset(fold=4, section="training")
+        self.assertTupleEqual(data[0]["image"].shape, (1, 34, 49, 41))
+        self.assertEqual(len(data), 208)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_decathlondataset.py b/tests/test_decathlondataset.py
--- a/tests/test_decathlondataset.py
+++ b/tests/test_decathlondataset.py
@@ -37,7 +37,7 @@ def _test_dataset(dataset):
             self.assertTrue("image" in dataset[0])
             self.assertTrue("label" in dataset[0])
             self.assertTrue("image_meta_dict" in dataset[0])
-            self.assertTupleEqual(dataset[0]["image"].shape, (1, 33, 47, 34))
+            self.assertTupleEqual(dataset[0]["image"].shape, (1, 36, 47, 44))
 
         try:  # will start downloading if testing_dir doesn't have the Decathlon files
             data = DecathlonDataset(
@@ -59,9 +59,15 @@ def _test_dataset(dataset):
             root_dir=testing_dir, task="Task04_Hippocampus", transform=transform, section="validation", download=False
         )
         _test_dataset(data)
+        # test validation without transforms
         data = DecathlonDataset(root_dir=testing_dir, task="Task04_Hippocampus", section="validation", download=False)
-        self.assertTupleEqual(data[0]["image"].shape, (33, 47, 34))
+        self.assertTupleEqual(data[0]["image"].shape, (36, 47, 44))
+        self.assertEqual(len(data), 52)
+        data = DecathlonDataset(root_dir=testing_dir, task="Task04_Hippocampus", section="training", download=False)
+        self.assertTupleEqual(data[0]["image"].shape, (34, 56, 31))
+        self.assertEqual(len(data), 208)
 
+        # test dataset properties
         data = DecathlonDataset(
             root_dir=testing_dir,
             task="Task04_Hippocampus",
diff --git a/tests/test_split_dataset.py b/tests/test_split_dataset.py
new file mode 100644
--- /dev/null
+++ b/tests/test_split_dataset.py
@@ -0,0 +1,36 @@
+# Copyright 2020 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from parameterized import parameterized
+
+from monai.apps import split_dataset
+
+TEST_CASE_1 = [{"nsplits": 5, "length": 10}, [[0, 2], [2, 4], [4, 6], [6, 8], [8, 10]]]
+
+TEST_CASE_2 = [{"nsplits": 5, "length": 13}, [[0, 3], [3, 6], [6, 9], [9, 11], [11, 13]]]
+
+TEST_CASE_3 = [{"nsplits": 5, "length": 11}, [[0, 3], [3, 5], [5, 7], [7, 9], [9, 11]]]
+
+TEST_CASE_4 = [{"nsplits": 1, "length": 10}, [[0, 10]]]
+
+
+class TestSplitDataset(unittest.TestCase):
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4])
+    def test_shape(self, input_param, expected_indices):
+        indices = split_dataset(**input_param)
+        for data, expected in zip(indices, expected_indices):
+            self.assertListEqual(data, expected)
+
+
+if __name__ == "__main__":
+    unittest.main()