Repository: Project-MONAI/MONAI

Issue:
Support multiple config files for MonaiAlgo
**Is your feature request related to a problem? Please describe.**
Current MonaiAlgo doesn't support several train configs that might be needed, e.g. for multigpu training.

**Describe the solution you'd like**
Support lists of config files in MonaiAlgo arguments. Note, ConfigParser already supports it.

**Describe alternatives you've considered**
n/a

**Additional context**
n/a



Discussion:

Resolving patch:
diff --git a/monai/fl/client/monai_algo.py b/monai/fl/client/monai_algo.py
--- a/monai/fl/client/monai_algo.py
+++ b/monai/fl/client/monai_algo.py
@@ -12,7 +12,7 @@
 import logging
 import os
 import sys
-from typing import Optional
+from typing import Optional, Union
 
 import torch
 
@@ -96,9 +96,11 @@ class MonaiAlgo(ClientAlgo):
         bundle_root: path of bundle.
         local_epochs: number of local epochs to execute during each round of local training; defaults to 1.
         send_weight_diff: whether to send weight differences rather than full weights; defaults to `True`.
-        config_train_filename: bundle training config path relative to bundle_root; defaults to "configs/train.json".
-        config_evaluate_filename: bundle evaluation config path relative to bundle_root; defaults to "configs/evaluate.json".
-        config_filters_filename: filter configuration file.
+        config_train_filename: bundle training config path relative to bundle_root. Can be a list of files;
+            defaults to "configs/train.json".
+        config_evaluate_filename: bundle evaluation config path relative to bundle_root. Can be a list of files.
+            If "default", config_evaluate_filename = ["configs/train.json", "configs/evaluate.json"] will be used;
+        config_filters_filename: filter configuration file. Can be a list of files; defaults to `None`.
         disable_ckpt_loading: do not use any CheckpointLoader if defined in train/evaluate configs; defaults to `True`.
         best_model_filepath: location of best model checkpoint; defaults "models/model.pt" relative to `bundle_root`.
         final_model_filepath: location of final model checkpoint; defaults "models/model_final.pt" relative to `bundle_root`.
@@ -117,9 +119,9 @@ def __init__(
         bundle_root: str,
         local_epochs: int = 1,
         send_weight_diff: bool = True,
-        config_train_filename: Optional[str] = "configs/train.json",
-        config_evaluate_filename: Optional[str] = "configs/evaluate.json",
-        config_filters_filename: Optional[str] = None,
+        config_train_filename: Optional[Union[str, list]] = "configs/train.json",
+        config_evaluate_filename: Optional[Union[str, list]] = "default",
+        config_filters_filename: Optional[Union[str, list]] = None,
         disable_ckpt_loading: bool = True,
         best_model_filepath: Optional[str] = "models/model.pt",
         final_model_filepath: Optional[str] = "models/model_final.pt",
@@ -128,7 +130,9 @@ def __init__(
         benchmark: bool = True,
     ):
         self.logger = logging.getLogger(self.__class__.__name__)
-
+        if config_evaluate_filename == "default":
+            # by default, evaluator needs both training and evaluate to be instantiated.
+            config_evaluate_filename = ["configs/train.json", "configs/evaluate.json"]
         self.bundle_root = bundle_root
         self.local_epochs = local_epochs
         self.send_weight_diff = send_weight_diff
@@ -180,16 +184,9 @@ def initialize(self, extra=None):
         # Read bundle config files
         self.bundle_root = os.path.join(self.app_root, self.bundle_root)
 
-        config_train_files = []
-        config_eval_files = []
-        config_filter_files = []
-        if self.config_train_filename:
-            config_train_files.append(os.path.join(self.bundle_root, self.config_train_filename))
-            config_eval_files.append(os.path.join(self.bundle_root, self.config_train_filename))
-        if self.config_evaluate_filename:
-            config_eval_files.append(os.path.join(self.bundle_root, self.config_evaluate_filename))
-        if self.config_filters_filename:  # filters are read by train_parser
-            config_filter_files.append(os.path.join(self.bundle_root, self.config_filters_filename))
+        config_train_files = self._add_config_files(self.config_train_filename)
+        config_eval_files = self._add_config_files(self.config_evaluate_filename)
+        config_filter_files = self._add_config_files(self.config_filters_filename)
 
         # Parse
         self.train_parser = ConfigParser()
@@ -433,3 +430,20 @@ def _check_converted(self, global_weights, local_var_dict, n_converted):
             self.logger.info(
                 f"Converted {n_converted} global variables to match {len(local_var_dict)} local variables."
             )
+
+    def _add_config_files(self, config_files):
+        files = []
+        if config_files:
+            if isinstance(config_files, str):
+                files.append(os.path.join(self.bundle_root, config_files))
+            elif isinstance(config_files, list):
+                for file in config_files:
+                    if isinstance(file, str):
+                        files.append(os.path.join(self.bundle_root, file))
+                    else:
+                        raise ValueError(f"Expected config file to be of type str but got {type(file)}: {file}")
+            else:
+                raise ValueError(
+                    f"Expected config files to be of type str or list but got {type(config_files)}: {config_files}"
+                )
+        return files


Test patch:
diff --git a/tests/test_fl_monai_algo.py b/tests/test_fl_monai_algo.py
--- a/tests/test_fl_monai_algo.py
+++ b/tests/test_fl_monai_algo.py
@@ -38,6 +38,20 @@
         "config_filters_filename": None,
     }
 ]
+TEST_TRAIN_3 = [
+    {
+        "bundle_root": os.path.join(os.path.dirname(__file__)),
+        "config_train_filename": [
+            os.path.join("testing_data", "config_fl_train.json"),
+            os.path.join("testing_data", "config_fl_train.json"),
+        ],
+        "config_evaluate_filename": None,
+        "config_filters_filename": [
+            os.path.join("testing_data", "config_fl_filters.json"),
+            os.path.join("testing_data", "config_fl_filters.json"),
+        ],
+    }
+]
 
 TEST_EVALUATE_1 = [
     {
@@ -55,6 +69,20 @@
         "config_filters_filename": None,
     }
 ]
+TEST_EVALUATE_3 = [
+    {
+        "bundle_root": os.path.join(os.path.dirname(__file__)),
+        "config_train_filename": None,
+        "config_evaluate_filename": [
+            os.path.join("testing_data", "config_fl_evaluate.json"),
+            os.path.join("testing_data", "config_fl_evaluate.json"),
+        ],
+        "config_filters_filename": [
+            os.path.join("testing_data", "config_fl_filters.json"),
+            os.path.join("testing_data", "config_fl_filters.json"),
+        ],
+    }
+]
 
 TEST_GET_WEIGHTS_1 = [
     {
@@ -83,19 +111,38 @@
         "config_filters_filename": os.path.join("testing_data", "config_fl_filters.json"),
     }
 ]
+TEST_GET_WEIGHTS_4 = [
+    {
+        "bundle_root": os.path.join(os.path.dirname(__file__)),
+        "config_train_filename": [
+            os.path.join("testing_data", "config_fl_train.json"),
+            os.path.join("testing_data", "config_fl_train.json"),
+        ],
+        "config_evaluate_filename": None,
+        "send_weight_diff": True,
+        "config_filters_filename": [
+            os.path.join("testing_data", "config_fl_filters.json"),
+            os.path.join("testing_data", "config_fl_filters.json"),
+        ],
+    }
+]
 
 
 @SkipIfNoModule("ignite")
 class TestFLMonaiAlgo(unittest.TestCase):
-    @parameterized.expand([TEST_TRAIN_1, TEST_TRAIN_2])
+    @parameterized.expand([TEST_TRAIN_1, TEST_TRAIN_2, TEST_TRAIN_3])
     def test_train(self, input_params):
-        # get testing data dir and update train config
-        with open(os.path.join(input_params["bundle_root"], input_params["config_train_filename"])) as f:
+        # get testing data dir and update train config; using the first to define data dir
+        if isinstance(input_params["config_train_filename"], list):
+            config_train_filename = input_params["config_train_filename"][0]
+        else:
+            config_train_filename = input_params["config_train_filename"]
+        with open(os.path.join(input_params["bundle_root"], config_train_filename)) as f:
             config_train = json.load(f)
 
         config_train["dataset_dir"] = os.path.join(os.path.dirname(__file__), "testing_data")
 
-        with open(os.path.join(input_params["bundle_root"], input_params["config_train_filename"]), "w") as f:
+        with open(os.path.join(input_params["bundle_root"], config_train_filename), "w") as f:
             json.dump(config_train, f, indent=4)
 
         # initialize algo
@@ -104,7 +151,7 @@ def test_train(self, input_params):
 
         # initialize model
         parser = ConfigParser()
-        parser.read_config(os.path.join(input_params["bundle_root"], input_params["config_train_filename"]))
+        parser.read_config(os.path.join(input_params["bundle_root"], config_train_filename))
         parser.parse()
         network = parser.get_parsed_content("network")
 
@@ -113,15 +160,19 @@ def test_train(self, input_params):
         # test train
         algo.train(data=data, extra={})
 
-    @parameterized.expand([TEST_EVALUATE_1, TEST_EVALUATE_2])
+    @parameterized.expand([TEST_EVALUATE_1, TEST_EVALUATE_2, TEST_EVALUATE_3])
     def test_evaluate(self, input_params):
-        # get testing data dir and update train config
-        with open(os.path.join(input_params["bundle_root"], input_params["config_evaluate_filename"])) as f:
+        # get testing data dir and update train config; using the first to define data dir
+        if isinstance(input_params["config_evaluate_filename"], list):
+            config_eval_filename = input_params["config_evaluate_filename"][0]
+        else:
+            config_eval_filename = input_params["config_evaluate_filename"]
+        with open(os.path.join(input_params["bundle_root"], config_eval_filename)) as f:
             config_evaluate = json.load(f)
 
         config_evaluate["dataset_dir"] = os.path.join(os.path.dirname(__file__), "testing_data")
 
-        with open(os.path.join(input_params["bundle_root"], input_params["config_evaluate_filename"]), "w") as f:
+        with open(os.path.join(input_params["bundle_root"], config_eval_filename), "w") as f:
             json.dump(config_evaluate, f, indent=4)
 
         # initialize algo
@@ -130,7 +181,7 @@ def test_evaluate(self, input_params):
 
         # initialize model
         parser = ConfigParser()
-        parser.read_config(os.path.join(input_params["bundle_root"], input_params["config_evaluate_filename"]))
+        parser.read_config(os.path.join(input_params["bundle_root"], config_eval_filename))
         parser.parse()
         network = parser.get_parsed_content("network")
 
@@ -139,16 +190,20 @@ def test_evaluate(self, input_params):
         # test evaluate
         algo.evaluate(data=data, extra={})
 
-    @parameterized.expand([TEST_GET_WEIGHTS_1, TEST_GET_WEIGHTS_2, TEST_GET_WEIGHTS_3])
+    @parameterized.expand([TEST_GET_WEIGHTS_1, TEST_GET_WEIGHTS_2, TEST_GET_WEIGHTS_3, TEST_GET_WEIGHTS_4])
     def test_get_weights(self, input_params):
-        # get testing data dir and update train config
+        # get testing data dir and update train config; using the first to define data dir
         if input_params["config_train_filename"]:
-            with open(os.path.join(input_params["bundle_root"], input_params["config_train_filename"])) as f:
+            if isinstance(input_params["config_train_filename"], list):
+                config_train_filename = input_params["config_train_filename"][0]
+            else:
+                config_train_filename = input_params["config_train_filename"]
+            with open(os.path.join(input_params["bundle_root"], config_train_filename)) as f:
                 config_train = json.load(f)
 
             config_train["dataset_dir"] = os.path.join(os.path.dirname(__file__), "testing_data")
 
-            with open(os.path.join(input_params["bundle_root"], input_params["config_train_filename"]), "w") as f:
+            with open(os.path.join(input_params["bundle_root"], config_train_filename), "w") as f:
                 json.dump(config_train, f, indent=4)
 
         # initialize algo