Repository: Project-MONAI/MONAI

Issue:
error in multi-process accessing of the same non-exist persistent cache
**Is your feature request related to a problem? Please describe.**
the persistent dataset will first check the existence of a cache directory and the create a new one if needed:
https://github.com/Project-MONAI/MONAI/blob/feb3a334b7bbf302b13a6da80e0b022a4cf75a4e/monai/data/dataset.py#L163-L165
these steps may run into a race condition in a multiprocess context.

```py
python -m tests.test_persistentdataset
persistent 1
persistent 0
create /var/folders/6f/fdkl7m0x7sz3nj_t7p3ccgz00000gp/T/tmpu578spse/test
create /var/folders/6f/fdkl7m0x7sz3nj_t7p3ccgz00000gp/T/tmpu578spse/test
Process Process-2:
Traceback (most recent call last):
  File "/usr/local/anaconda3/envs/py37/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
    self.run()
  File "/usr/local/anaconda3/envs/py37/lib/python3.7/multiprocessing/process.py", line 99, in run
    self._target(*self._args, **self._kwargs)
  File "MONAI/tests/utils.py", line 296, in run_process
    raise e
  File "MONAI/tests/utils.py", line 287, in run_process
    func(*args, **kwargs)
  File "MONAI/tests/utils.py", line 471, in _call_original_func
    return f(*args, **kwargs)
  File "MONAI/tests/test_persistentdataset.py", line 166, in test_mp_dataset
    ds = PersistentDataset(items, transform=_InplaceXform(), cache_dir=cache_dir)
  File "MONAI/monai/data/dataset.py", line 172, in __init__
    self.cache_dir.mkdir(parents=True)
  File "/usr/local/anaconda3/envs/py37/lib/python3.7/pathlib.py", line 1273, in mkdir
    self._accessor.mkdir(self, mode)
FileExistsError: [Errno 17] File exists: '/var/folders/6f/fdkl7m0x7sz3nj_t7p3ccgz00000gp/T/tmpu578spse/test'
F
======================================================================
FAIL: test_mp_dataset (__main__.TestDistCreateDataset)
----------------------------------------------------------------------
Traceback (most recent call last):
  File MONAI/tests/utils.py", line 343, in _wrapper
    assert results.get(), "Distributed call failed."
AssertionError: Distributed call failed.

----------------------------------------------------------------------
```
error in multi-process accessing of the same non-exist persistent cache
**Is your feature request related to a problem? Please describe.**
the persistent dataset will first check the existence of a cache directory and the create a new one if needed:
https://github.com/Project-MONAI/MONAI/blob/feb3a334b7bbf302b13a6da80e0b022a4cf75a4e/monai/data/dataset.py#L163-L165
these steps may run into a race condition in a multiprocess context.

```py
python -m tests.test_persistentdataset
persistent 1
persistent 0
create /var/folders/6f/fdkl7m0x7sz3nj_t7p3ccgz00000gp/T/tmpu578spse/test
create /var/folders/6f/fdkl7m0x7sz3nj_t7p3ccgz00000gp/T/tmpu578spse/test
Process Process-2:
Traceback (most recent call last):
  File "/usr/local/anaconda3/envs/py37/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
    self.run()
  File "/usr/local/anaconda3/envs/py37/lib/python3.7/multiprocessing/process.py", line 99, in run
    self._target(*self._args, **self._kwargs)
  File "MONAI/tests/utils.py", line 296, in run_process
    raise e
  File "MONAI/tests/utils.py", line 287, in run_process
    func(*args, **kwargs)
  File "MONAI/tests/utils.py", line 471, in _call_original_func
    return f(*args, **kwargs)
  File "MONAI/tests/test_persistentdataset.py", line 166, in test_mp_dataset
    ds = PersistentDataset(items, transform=_InplaceXform(), cache_dir=cache_dir)
  File "MONAI/monai/data/dataset.py", line 172, in __init__
    self.cache_dir.mkdir(parents=True)
  File "/usr/local/anaconda3/envs/py37/lib/python3.7/pathlib.py", line 1273, in mkdir
    self._accessor.mkdir(self, mode)
FileExistsError: [Errno 17] File exists: '/var/folders/6f/fdkl7m0x7sz3nj_t7p3ccgz00000gp/T/tmpu578spse/test'
F
======================================================================
FAIL: test_mp_dataset (__main__.TestDistCreateDataset)
----------------------------------------------------------------------
Traceback (most recent call last):
  File MONAI/tests/utils.py", line 343, in _wrapper
    assert results.get(), "Distributed call failed."
AssertionError: Distributed call failed.

----------------------------------------------------------------------
```


Discussion:

Resolving patch:
diff --git a/monai/data/dataset.py b/monai/data/dataset.py
--- a/monai/data/dataset.py
+++ b/monai/data/dataset.py
@@ -168,7 +168,7 @@ def __init__(
         self.hash_func = hash_func
         if self.cache_dir is not None:
             if not self.cache_dir.exists():
-                self.cache_dir.mkdir(parents=True)
+                self.cache_dir.mkdir(parents=True, exist_ok=True)
             if not self.cache_dir.is_dir():
                 raise ValueError("cache_dir must be a directory.")
 


Test patch:
diff --git a/tests/test_persistentdataset.py b/tests/test_persistentdataset.py
--- a/tests/test_persistentdataset.py
+++ b/tests/test_persistentdataset.py
@@ -10,18 +10,15 @@
 # limitations under the License.
 
 import os
-import shutil
 import tempfile
 import unittest
 
 import nibabel as nib
 import numpy as np
-import torch.distributed as dist
 from parameterized import parameterized
 
 from monai.data import PersistentDataset, json_hashing
 from monai.transforms import Compose, LoadImaged, SimulateDelayd, Transform
-from tests.utils import DistCall, DistTestCase
 
 TEST_CASE_1 = [
     Compose(
@@ -127,29 +124,5 @@ def test_shape(self, transform, expected_shape):
                     self.assertTupleEqual(d["image"].shape, expected_shape)
 
 
-class TestDistDataset(DistTestCase):
-    def setUp(self):
-        self.tempdir = tempfile.mkdtemp()
-
-    def tearDown(self):
-        shutil.rmtree(self.tempdir)
-
-    @DistCall(nnodes=1, nproc_per_node=2)
-    def test_mp_dataset(self):
-        print("persistent", dist.get_rank())
-        items = [[list(range(i))] for i in range(5)]
-        ds = PersistentDataset(items, transform=_InplaceXform(), cache_dir=self.tempdir)
-        self.assertEqual(items, [[[]], [[0]], [[0, 1]], [[0, 1, 2]], [[0, 1, 2, 3]]])
-        ds1 = PersistentDataset(items, transform=_InplaceXform(), cache_dir=self.tempdir)
-        self.assertEqual(list(ds1), list(ds))
-        self.assertEqual(items, [[[]], [[0]], [[0, 1]], [[0, 1, 2]], [[0, 1, 2, 3]]])
-
-        ds = PersistentDataset(items, transform=_InplaceXform(), cache_dir=self.tempdir, hash_func=json_hashing)
-        self.assertEqual(items, [[[]], [[0]], [[0, 1]], [[0, 1, 2]], [[0, 1, 2, 3]]])
-        ds1 = PersistentDataset(items, transform=_InplaceXform(), cache_dir=self.tempdir, hash_func=json_hashing)
-        self.assertEqual(list(ds1), list(ds))
-        self.assertEqual(items, [[[]], [[0]], [[0, 1]], [[0, 1, 2]], [[0, 1, 2, 3]]])
-
-
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_persistentdataset_dist.py b/tests/test_persistentdataset_dist.py
new file mode 100644
--- /dev/null
+++ b/tests/test_persistentdataset_dist.py
@@ -0,0 +1,78 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import shutil
+import tempfile
+import unittest
+
+import numpy as np
+import torch.distributed as dist
+
+from monai.data import PersistentDataset, json_hashing
+from monai.transforms import Transform
+from tests.utils import DistCall, DistTestCase
+
+
+class _InplaceXform(Transform):
+    def __call__(self, data):
+        if data:
+            data[0] = data[0] + np.pi
+        else:
+            data.append(1)
+        return data
+
+
+class TestDistDataset(DistTestCase):
+    def setUp(self):
+        self.tempdir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.tempdir)
+
+    @DistCall(nnodes=1, nproc_per_node=2)
+    def test_mp_dataset(self):
+        print("persistent", dist.get_rank())
+        items = [[list(range(i))] for i in range(5)]
+        ds = PersistentDataset(items, transform=_InplaceXform(), cache_dir=self.tempdir)
+        self.assertEqual(items, [[[]], [[0]], [[0, 1]], [[0, 1, 2]], [[0, 1, 2, 3]]])
+        ds1 = PersistentDataset(items, transform=_InplaceXform(), cache_dir=self.tempdir)
+        self.assertEqual(list(ds1), list(ds))
+        self.assertEqual(items, [[[]], [[0]], [[0, 1]], [[0, 1, 2]], [[0, 1, 2, 3]]])
+
+        ds = PersistentDataset(items, transform=_InplaceXform(), cache_dir=self.tempdir, hash_func=json_hashing)
+        self.assertEqual(items, [[[]], [[0]], [[0, 1]], [[0, 1, 2]], [[0, 1, 2, 3]]])
+        ds1 = PersistentDataset(items, transform=_InplaceXform(), cache_dir=self.tempdir, hash_func=json_hashing)
+        self.assertEqual(list(ds1), list(ds))
+        self.assertEqual(items, [[[]], [[0]], [[0, 1]], [[0, 1, 2]], [[0, 1, 2, 3]]])
+
+
+class TestDistCreateDataset(DistTestCase):
+    def setUp(self):
+        self.tempdir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.tempdir)
+
+    @DistCall(nnodes=1, nproc_per_node=2)
+    def test_mp_dataset(self):
+        print("persistent", dist.get_rank())
+        items = [[list(range(i))] for i in range(5)]
+        cache_dir = os.path.join(self.tempdir, "test")
+        ds = PersistentDataset(items, transform=_InplaceXform(), cache_dir=cache_dir)
+        self.assertEqual(items, [[[]], [[0]], [[0, 1]], [[0, 1, 2]], [[0, 1, 2, 3]]])
+        ds1 = PersistentDataset(items, transform=_InplaceXform(), cache_dir=cache_dir)
+        self.assertEqual(list(ds1), list(ds))
+        self.assertEqual(items, [[[]], [[0]], [[0, 1]], [[0, 1, 2]], [[0, 1, 2, 3]]])
+
+
+if __name__ == "__main__":
+    unittest.main()