Repository: Project-MONAI/MONAI

Issue:
flexible tests for auto3dseg algo templates
**Is your feature request related to a problem? Please describe.**
current the bundle gen tests always use the default algo template hard-coded in the bundlegen module

https://github.com/Project-MONAI/MONAI/blob/de2b48c7a772c1171fb9acc7ff72addc469558a8/tests/test_auto3dseg_ensemble.py#L123

https://github.com/Project-MONAI/MONAI/blob/de2b48c7a772c1171fb9acc7ff72addc469558a8/monai/apps/auto3dseg/bundle_gen.py#L280

would be great to allow for test time options configured for different algo template sources.


Discussion:

Resolving patch:
diff --git a/monai/apps/auto3dseg/bundle_gen.py b/monai/apps/auto3dseg/bundle_gen.py
--- a/monai/apps/auto3dseg/bundle_gen.py
+++ b/monai/apps/auto3dseg/bundle_gen.py
@@ -326,7 +326,7 @@ def _download_algos_url(url: str, at_path: str) -> dict[str, dict[str, str]]:
 def _copy_algos_folder(folder, at_path):
     """
     Copies the algorithm templates folder to at_path.
-    Returns a dictionary of of algorithm templates.
+    Returns a dictionary of algorithm templates.
     """
     folder = os.path.abspath(folder)
     at_path = os.path.abspath(at_path)
@@ -342,7 +342,8 @@ def _copy_algos_folder(folder, at_path):
             algos_all[name] = dict(
                 _target_=f"{name}.scripts.algo.{name.capitalize()}Algo", template_path=os.path.join(at_path, name)
             )
-    if len(algos_all) == 0:
+            logger.info(f"{name} -- {algos_all[name]}")
+    if not algos_all:
         raise ValueError(f"Unable to find any algos in {folder}")
 
     return algos_all
@@ -385,9 +386,11 @@ def __init__(
 
             if os.path.isdir(templates_path_or_url):
                 # if a local folder, copy if necessary
+                logger.info(f"BundleGen from directory {templates_path_or_url}")
                 algos_all = _copy_algos_folder(folder=templates_path_or_url, at_path=at_path)
             elif urlparse(templates_path_or_url).scheme in ("http", "https"):
                 # if url, trigger the download and extract process
+                logger.info(f"BundleGen from {templates_path_or_url}")
                 algos_all = _download_algos_url(url=templates_path_or_url, at_path=at_path)
             else:
                 raise ValueError(f"{self.__class__} received invalid templates_path_or_url: {templates_path_or_url}")


Test patch:
diff --git a/tests/test_auto3dseg_ensemble.py b/tests/test_auto3dseg_ensemble.py
--- a/tests/test_auto3dseg_ensemble.py
+++ b/tests/test_auto3dseg_ensemble.py
@@ -24,7 +24,13 @@
 from monai.data import create_test_image_3d
 from monai.utils import optional_import, set_determinism
 from monai.utils.enums import AlgoEnsembleKeys
-from tests.utils import SkipIfBeforePyTorchVersion, skip_if_downloading_fails, skip_if_no_cuda, skip_if_quick
+from tests.utils import (
+    SkipIfBeforePyTorchVersion,
+    get_testing_algo_template_path,
+    skip_if_downloading_fails,
+    skip_if_no_cuda,
+    skip_if_quick,
+)
 
 _, has_tb = optional_import("torch.utils.tensorboard", name="SummaryWriter")
 
@@ -120,7 +126,10 @@ def test_ensemble(self) -> None:
 
         with skip_if_downloading_fails():
             bundle_generator = BundleGen(
-                algo_path=work_dir, data_stats_filename=da_output_yaml, data_src_cfg_name=data_src_cfg
+                algo_path=work_dir,
+                data_stats_filename=da_output_yaml,
+                data_src_cfg_name=data_src_cfg,
+                templates_path_or_url=get_testing_algo_template_path(),
             )
         bundle_generator.generate(work_dir, num_fold=1)
         history = bundle_generator.get_history()
diff --git a/tests/test_auto3dseg_hpo.py b/tests/test_auto3dseg_hpo.py
--- a/tests/test_auto3dseg_hpo.py
+++ b/tests/test_auto3dseg_hpo.py
@@ -24,7 +24,12 @@
 from monai.bundle.config_parser import ConfigParser
 from monai.data import create_test_image_3d
 from monai.utils import optional_import
-from tests.utils import SkipIfBeforePyTorchVersion, skip_if_downloading_fails, skip_if_no_cuda
+from tests.utils import (
+    SkipIfBeforePyTorchVersion,
+    get_testing_algo_template_path,
+    skip_if_downloading_fails,
+    skip_if_no_cuda,
+)
 
 _, has_tb = optional_import("torch.utils.tensorboard", name="SummaryWriter")
 optuna, has_optuna = optional_import("optuna")
@@ -120,7 +125,10 @@ def setUp(self) -> None:
         ConfigParser.export_config_file(data_src, data_src_cfg)
         with skip_if_downloading_fails():
             bundle_generator = BundleGen(
-                algo_path=work_dir, data_stats_filename=da_output_yaml, data_src_cfg_name=data_src_cfg
+                algo_path=work_dir,
+                data_stats_filename=da_output_yaml,
+                data_src_cfg_name=data_src_cfg,
+                templates_path_or_url=get_testing_algo_template_path(),
             )
         bundle_generator.generate(work_dir, num_fold=1)
 
diff --git a/tests/test_integration_autorunner.py b/tests/test_integration_autorunner.py
--- a/tests/test_integration_autorunner.py
+++ b/tests/test_integration_autorunner.py
@@ -23,7 +23,13 @@
 from monai.bundle.config_parser import ConfigParser
 from monai.data import create_test_image_3d
 from monai.utils import optional_import
-from tests.utils import SkipIfBeforePyTorchVersion, skip_if_downloading_fails, skip_if_no_cuda, skip_if_quick
+from tests.utils import (
+    SkipIfBeforePyTorchVersion,
+    get_testing_algo_template_path,
+    skip_if_downloading_fails,
+    skip_if_no_cuda,
+    skip_if_quick,
+)
 
 _, has_tb = optional_import("torch.utils.tensorboard", name="SummaryWriter")
 _, has_nni = optional_import("nni")
@@ -107,7 +113,9 @@ def setUp(self) -> None:
     @skip_if_no_cuda
     def test_autorunner(self) -> None:
         work_dir = os.path.join(self.test_path, "work_dir")
-        runner = AutoRunner(work_dir=work_dir, input=self.data_src_cfg)
+        runner = AutoRunner(
+            work_dir=work_dir, input=self.data_src_cfg, templates_path_or_url=get_testing_algo_template_path()
+        )
         runner.set_training_params(train_param)  # 2 epochs
         runner.set_num_fold(1)
         with skip_if_downloading_fails():
@@ -116,7 +124,9 @@ def test_autorunner(self) -> None:
     @skip_if_no_cuda
     def test_autorunner_ensemble(self) -> None:
         work_dir = os.path.join(self.test_path, "work_dir")
-        runner = AutoRunner(work_dir=work_dir, input=self.data_src_cfg)
+        runner = AutoRunner(
+            work_dir=work_dir, input=self.data_src_cfg, templates_path_or_url=get_testing_algo_template_path()
+        )
         runner.set_training_params(train_param)  # 2 epochs
         runner.set_ensemble_method("AlgoEnsembleBestByFold")
         runner.set_num_fold(1)
@@ -126,7 +136,9 @@ def test_autorunner_ensemble(self) -> None:
     @skip_if_no_cuda
     def test_autorunner_gpu_customization(self) -> None:
         work_dir = os.path.join(self.test_path, "work_dir")
-        runner = AutoRunner(work_dir=work_dir, input=self.data_src_cfg)
+        runner = AutoRunner(
+            work_dir=work_dir, input=self.data_src_cfg, templates_path_or_url=get_testing_algo_template_path()
+        )
         gpu_customization_specs = {
             "universal": {"num_trials": 1, "range_num_images_per_batch": [1, 2], "range_num_sw_batch_size": [1, 2]}
         }
@@ -140,7 +152,13 @@ def test_autorunner_gpu_customization(self) -> None:
     @unittest.skipIf(not has_nni, "nni required")
     def test_autorunner_hpo(self) -> None:
         work_dir = os.path.join(self.test_path, "work_dir")
-        runner = AutoRunner(work_dir=work_dir, input=self.data_src_cfg, hpo=True, ensemble=False)
+        runner = AutoRunner(
+            work_dir=work_dir,
+            input=self.data_src_cfg,
+            hpo=True,
+            ensemble=False,
+            templates_path_or_url=get_testing_algo_template_path(),
+        )
         hpo_param = {
             "num_epochs_per_validation": train_param["num_epochs_per_validation"],
             "num_images_per_batch": train_param["num_images_per_batch"],
diff --git a/tests/test_integration_gpu_customization.py b/tests/test_integration_gpu_customization.py
--- a/tests/test_integration_gpu_customization.py
+++ b/tests/test_integration_gpu_customization.py
@@ -24,7 +24,13 @@
 from monai.data import create_test_image_3d
 from monai.utils import optional_import
 from monai.utils.enums import AlgoEnsembleKeys
-from tests.utils import SkipIfBeforePyTorchVersion, skip_if_downloading_fails, skip_if_no_cuda, skip_if_quick
+from tests.utils import (
+    SkipIfBeforePyTorchVersion,
+    get_testing_algo_template_path,
+    skip_if_downloading_fails,
+    skip_if_no_cuda,
+    skip_if_quick,
+)
 
 _, has_tb = optional_import("torch.utils.tensorboard", name="SummaryWriter")
 
@@ -118,7 +124,10 @@ def test_ensemble_gpu_customization(self) -> None:
 
         with skip_if_downloading_fails():
             bundle_generator = BundleGen(
-                algo_path=work_dir, data_stats_filename=da_output_yaml, data_src_cfg_name=data_src_cfg
+                algo_path=work_dir,
+                data_stats_filename=da_output_yaml,
+                data_src_cfg_name=data_src_cfg,
+                templates_path_or_url=get_testing_algo_template_path(),
             )
 
         gpu_customization_specs = {
diff --git a/tests/utils.py b/tests/utils.py
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -66,6 +66,16 @@ def testing_data_config(*keys):
     return reduce(operator.getitem, keys, _test_data_config)
 
 
+def get_testing_algo_template_path():
+    """
+    a local folder to the testing algorithm template or a url to the compressed template file.
+    Default to None, which effectively uses bundle_gen's ``default_algo_zip`` path.
+
+    https://github.com/Project-MONAI/MONAI/blob/1.1.0/monai/apps/auto3dseg/bundle_gen.py#L380-L381
+    """
+    return os.environ.get("MONAI_TESTING_ALGO_TEMPLATE", None)
+
+
 def clone(data: NdarrayTensor) -> NdarrayTensor:
     """
     Clone data independent of type.