Repository: Project-MONAI/MONAI

Issue:
Auto3dseg integration
```

predictions
[2023-04-22T04:14:14.404Z] 2023-04-22 04:14:13,627 - INFO - Directory /tmp/tmp8m_dx6rq/work_dir/ensemble_output is created to save ensemble predictions
[2023-04-22T04:14:14.404Z] Traceback (most recent call last):
[2023-04-22T04:14:14.404Z]   File "/opt/conda/lib/python3.8/runpy.py", line 194, in _run_module_as_main
[2023-04-22T04:14:14.404Z]     return _run_code(code, main_globals, None,
[2023-04-22T04:14:14.404Z]   File "/opt/conda/lib/python3.8/runpy.py", line 87, in _run_code
[2023-04-22T04:14:14.404Z]     exec(code, run_globals)
[2023-04-22T04:14:14.404Z]   File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/monai/apps/auto3dseg/__main__.py", line 24, in <module>
[2023-04-22T04:14:14.404Z]     fire.Fire(
[2023-04-22T04:14:14.404Z]   File "/opt/conda/lib/python3.8/site-packages/fire/core.py", line 141, in Fire
[2023-04-22T04:14:14.404Z]     component_trace = _Fire(component, args, parsed_flag_args, context, name)
[2023-04-22T04:14:14.404Z]   File "/opt/conda/lib/python3.8/site-packages/fire/core.py", line 475, in _Fire
[2023-04-22T04:14:14.404Z]     component, remaining_args = _CallAndUpdateTrace(
[2023-04-22T04:14:14.404Z]   File "/opt/conda/lib/python3.8/site-packages/fire/core.py", line 691, in _CallAndUpdateTrace
[2023-04-22T04:14:14.404Z]     component = fn(*varargs, **kwargs)
[2023-04-22T04:14:14.404Z]   File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/monai/apps/auto3dseg/ensemble_builder.py", line 478, in ensemble
[2023-04-22T04:14:14.404Z]     self.set_image_save_transform(**self.kwargs)
[2023-04-22T04:14:14.404Z]   File "/home/jenkins/agent/workspace/MONAI-postmerge/monai_github/monai/apps/auto3dseg/ensemble_builder.py", line 438, in set_image_save_transform
[2023-04-22T04:14:14.404Z]     os.makedirs(output_dir)
[2023-04-22T04:14:14.404Z]   File "/opt/conda/lib/python3.8/os.py", line 223, in makedirs
[2023-04-22T04:14:14.404Z]     mkdir(name, mode)
[2023-04-22T04:14:14.404Z] FileExistsError: [Errno 17] File exists: '/tmp/tmp8m_dx6rq/work_dir/ensemble_output'
[2023-04-22T04:14:14.404Z] 2023-04-22 04:14:13,682 - INFO - Auto3Dseg picked the following networks to ensemble:
[2023-04-22T04:14:14.404Z] 2023-04-22 04:14:13,682 - INFO - segresnet2d_0
[2023-04-22T04:14:14.404Z] 2023-04-22 04:14:13,682 - INFO - Auto3Dseg ensemble prediction outputs will be saved in /tmp/tmp8m_dx6rq/work_dir/ensemble_output.
[2023-04-22T04:14:14.404Z] 
Ensembling (rank 0)...:   0%|          | 0/1 [00:00<?, ?it/s]monai.transforms.io.dictionary LoadImaged.__init__:image_only: Current default value of argument `image_only=False` has been deprecated since version 1.1. It will be changed to `image_only=True` in version 1.3.
[2023-04-22T04:14:14.659Z] [info] checkpoint /tmp/tmp8m_dx6rq/work_dir/segresnet2d_0/model_fold0/best_metric_model.pt loaded
[2023-04-22T04:14:15.589Z] 2023-04-22 04:14:15,372 INFO image_writer.py:197 - writing: /tmp/tmp8m_dx6rq/work_dir/segresnet2d_0/prediction_testing/val_001.fake/val_001.fake_seg.nii.gz

```


Discussion:
Does this happen in multi-node/multi-GPU testing?

This bug looks like two processes are competing each other to do `os.makedirs()` .
Indeed it happened to the multigpu tests and it's random... I think that's the issue. Could you please submit a PR to fix it?

Resolving patch:
diff --git a/monai/apps/auto3dseg/auto_runner.py b/monai/apps/auto3dseg/auto_runner.py
--- a/monai/apps/auto3dseg/auto_runner.py
+++ b/monai/apps/auto3dseg/auto_runner.py
@@ -29,7 +29,9 @@
 from monai.apps.utils import get_logger
 from monai.auto3dseg.utils import algo_to_pickle
 from monai.bundle import ConfigParser
+from monai.transforms import SaveImage
 from monai.utils import AlgoKeys, has_option, look_up_option, optional_import
+from monai.utils.misc import check_kwargs_exist_in_class_init
 
 logger = get_logger(module_name=__name__)
 
@@ -561,7 +563,14 @@ def set_image_save_transform(self, **kwargs: Any) -> None:
 
         """
 
-        self.kwargs.update(kwargs)
+        are_all_args_present, missing_args = check_kwargs_exist_in_class_init(SaveImage, kwargs)
+        if are_all_args_present:
+            self.kwargs.update(kwargs)
+        else:
+            raise ValueError(
+                f"{missing_args} are not supported in monai.transforms.SaveImage,"
+                "Check https://docs.monai.io/en/stable/transforms.html#saveimage for more information."
+            )
 
     def set_prediction_params(self, params: dict[str, Any] | None = None) -> None:
         """
diff --git a/monai/apps/auto3dseg/ensemble_builder.py b/monai/apps/auto3dseg/ensemble_builder.py
--- a/monai/apps/auto3dseg/ensemble_builder.py
+++ b/monai/apps/auto3dseg/ensemble_builder.py
@@ -30,9 +30,9 @@
 from monai.auto3dseg.utils import datafold_read
 from monai.bundle import ConfigParser
 from monai.data import partition_dataset
-from monai.transforms import MeanEnsemble, VoteEnsemble
+from monai.transforms import MeanEnsemble, SaveImage, VoteEnsemble
 from monai.utils.enums import AlgoKeys
-from monai.utils.misc import prob2class
+from monai.utils.misc import check_kwargs_exist_in_class_init, prob2class
 from monai.utils.module import look_up_option, optional_import
 
 tqdm, has_tqdm = optional_import("tqdm", name="tqdm")
@@ -395,7 +395,7 @@ def __init__(
         self.num_fold = num_fold
         self.ensemble_method_name = ensemble_method_name
         self.mgpu = mgpu
-        self.kwargs = kwargs
+        self.kwargs = deepcopy(kwargs)
         self.rank = 0
         self.world_size = 1
         self.device_setting: dict[str, int | str] = {
@@ -406,6 +406,9 @@ def __init__(
             "CMD_PREFIX": os.environ.get("CMD_PREFIX"),  # type: ignore
         }
 
+        # self.kwargs needs to pop out args for set_image_save_transform
+        self.save_image: dict[str, Any] = self._pop_kwargs_to_get_image_save_transform(**self.kwargs)
+
     def set_ensemble_method(self, ensemble_method_name: str = "AlgoEnsembleBestByFold", **kwargs: Any) -> None:
         """
         Set the bundle ensemble method
@@ -428,44 +431,60 @@ def set_ensemble_method(self, ensemble_method_name: str = "AlgoEnsembleBestByFol
         else:
             raise NotImplementedError(f"Ensemble method {self.ensemble_method_name} is not implemented.")
 
-    def set_image_save_transform(self, **kwargs):
+    def _pop_kwargs_to_get_image_save_transform(self, **kwargs):
         """
-        Set the ensemble output transform.
+        Pop the kwargs used to define ImageSave class for the ensemble output.
 
         Args:
             kwargs: image writing parameters for the ensemble inference. The kwargs format follows SaveImage
                 transform. For more information, check https://docs.monai.io/en/stable/transforms.html#saveimage .
 
+        Returns:
+            save_image: a dictionary that can be used to instantiate a SaveImage class in ConfigParser.
         """
 
-        if "output_dir" in kwargs:
-            output_dir = kwargs.pop("output_dir")
-        else:
+        output_dir = kwargs.pop("output_dir", None)
+
+        if output_dir is None:
             output_dir = os.path.join(self.work_dir, "ensemble_output")
-            if self.rank == 0:
-                logger.info(f"The output_dir is not specified. {output_dir} will be used to save ensemble predictions")
+            logger.info(f"The output_dir is not specified. {output_dir} will be used to save ensemble predictions.")
 
         if not os.path.isdir(output_dir):
-            os.makedirs(output_dir)
-            if self.rank == 0:
-                logger.info(f"Directory {output_dir} is created to save ensemble predictions")
-
-        self.output_dir = output_dir
-        output_postfix = kwargs.pop("output_postfix", "ensemble")
-        output_dtype = kwargs.pop("output_dtype", "$np.uint8")
-        resample = kwargs.pop("resample", False)
+            os.makedirs(output_dir, exist_ok=True)
+            logger.info(f"Directory {output_dir} is created to save ensemble predictions")
 
-        self.save_image = {
+        save_image = {
             "_target_": "SaveImage",
             "output_dir": output_dir,
-            "output_postfix": output_postfix,
-            "output_dtype": output_dtype,
-            "resample": resample,
+            "output_postfix": kwargs.pop("output_postfix", "ensemble"),
+            "output_dtype": kwargs.pop("output_dtype", "$np.uint8"),
+            "resample": kwargs.pop("resample", False),
             "print_log": False,
             "savepath_in_metadict": True,
         }
-        if kwargs:
-            self.save_image.update(kwargs)
+
+        are_all_args_save_image, extra_args = check_kwargs_exist_in_class_init(SaveImage, kwargs)
+        if are_all_args_save_image:
+            save_image.update(kwargs)
+        else:
+            # kwargs has extra values for other purposes, for example, pred_params
+            for args in list(kwargs):
+                if args not in extra_args:
+                    save_image.update({args: kwargs.pop(args)})
+
+        return save_image
+
+    def set_image_save_transform(self, **kwargs):
+        """
+        Set the ensemble output transform.
+
+        Args:
+            kwargs: image writing parameters for the ensemble inference. The kwargs format follows SaveImage
+                transform. For more information, check https://docs.monai.io/en/stable/transforms.html#saveimage .
+
+        """
+        kwargs_copy = deepcopy(kwargs)
+        self.save_image = self._pop_kwargs_to_get_image_save_transform(**kwargs_copy)
 
     def set_num_fold(self, num_fold: int = 5) -> None:
         """
@@ -487,7 +506,6 @@ def ensemble(self):
             self.rank = dist.get_rank()
         # set params after init_process_group to know the rank
         self.set_num_fold(num_fold=self.num_fold)
-        self.set_image_save_transform(**self.kwargs)
         self.set_ensemble_method(self.ensemble_method_name, **self.kwargs)
 
         history = import_bundle_algo_history(self.work_dir, only_trained=False)
@@ -514,7 +532,6 @@ def ensemble(self):
         )[self.rank]
         # TO DO: Add some function in ensembler for infer_files update?
         self.ensembler.infer_files = infer_files
-        # self.kwargs has poped out args for set_image_save_transform
         # add rank to pred_params
         self.kwargs["rank"] = self.rank
         self.kwargs["image_save_func"] = self.save_image
@@ -522,7 +539,8 @@ def ensemble(self):
             logger.info("Auto3Dseg picked the following networks to ensemble:")
             for algo in self.ensembler.get_algo_ensemble():
                 logger.info(algo[AlgoKeys.ID])
-            logger.info(f"Auto3Dseg ensemble prediction outputs will be saved in {self.output_dir}.")
+            output_dir = self.save_image["output_dir"]
+            logger.info(f"Auto3Dseg ensemble prediction outputs will be saved in {output_dir}.")
         self.ensembler(pred_param=self.kwargs)
 
         if self.mgpu:
diff --git a/monai/utils/misc.py b/monai/utils/misc.py
--- a/monai/utils/misc.py
+++ b/monai/utils/misc.py
@@ -72,6 +72,7 @@
     "check_key_duplicates",
     "CheckKeyDuplicatesYamlLoader",
     "ConvertUnits",
+    "check_kwargs_exist_in_class_init",
 ]
 
 _seed = None
@@ -800,3 +801,23 @@ def _calculate_conversion_factor(self):
 
     def __call__(self, value: int | float) -> Any:
         return float(value) * self.conversion_factor
+
+
+def check_kwargs_exist_in_class_init(cls, kwargs):
+    """
+    Check if the all keys in kwargs exist in the __init__ method of the class.
+
+    Args:
+        cls: the class to check.
+        kwargs: kwargs to examine.
+
+    Returns:
+        a boolean inidicating if all keys exist.
+        a set of extra keys that are not used in the __init__.
+    """
+    init_signature = inspect.signature(cls.__init__)
+    init_params = set(init_signature.parameters) - {"self"}  # Exclude 'self' from the parameter list
+    input_kwargs = set(kwargs)
+    extra_kwargs = input_kwargs - init_params
+
+    return extra_kwargs == set(), extra_kwargs


Test patch:
diff --git a/tests/test_monai_utils_misc.py b/tests/test_monai_utils_misc.py
--- a/tests/test_monai_utils_misc.py
+++ b/tests/test_monai_utils_misc.py
@@ -15,7 +15,7 @@
 
 from parameterized import parameterized
 
-from monai.utils.misc import to_tuple_of_dictionaries
+from monai.utils.misc import check_kwargs_exist_in_class_init, to_tuple_of_dictionaries
 
 TO_TUPLE_OF_DICTIONARIES_TEST_CASES = [
     ({}, tuple(), tuple()),
@@ -38,6 +38,11 @@
 ]
 
 
+class MiscClass:
+    def __init__(self, arg1, arg2, kwargs1=None, kwargs2=None):
+        pass
+
+
 class TestToTupleOfDictionaries(unittest.TestCase):
     @parameterized.expand(TO_TUPLE_OF_DICTIONARIES_TEST_CASES)
     def test_to_tuple_of_dictionaries(self, dictionary, keys, expected):
@@ -52,3 +57,20 @@ def _test_to_tuple_of_dictionaries(self, dictionary, keys, expected):
             actual = to_tuple_of_dictionaries(dictionary, keys)
             print(actual, expected)
             self.assertTupleEqual(actual, expected)
+
+
+class TestMiscKwargs(unittest.TestCase):
+    def test_kwargs(self):
+        present, extra_args = self._custom_user_function(MiscClass, 1, kwargs1="value1", kwargs2="value2")
+        self.assertEqual(present, True)
+        self.assertEqual(extra_args, set())
+        present, extra_args = self._custom_user_function(MiscClass, 1, kwargs1="value1", kwargs3="value3")
+        self.assertEqual(present, False)
+        self.assertEqual(extra_args, {"kwargs3"})
+
+    def _custom_user_function(self, cls, *args, **kwargs):
+        return check_kwargs_exist_in_class_init(cls, kwargs)
+
+
+if __name__ == "__main__":
+    unittest.main()