Repository: Project-MONAI/MONAI

Issue:
test_integration_autorunner.py" -> test_autorunner_hpo(self) , test for HPO.  On my local machine it runs forever, and get stuck without any progress (no CPU or GPU usage)
 There is one integration test, I' unable to test in "test_integration_autorunner.py" -> test_autorunner_hpo(self) , test for HPO.  On my local machine it runs forever, and get stuck without any progress (no CPU or GPU usage).  Same behavior on old dev branch and on current (so it's unrelated to this PR).   So I've test everything except for HPO test there.

_Originally posted by @myron in https://github.com/Project-MONAI/MONAI/issues/5562#issuecomment-1324370473_
   


I also saw this issue multiple times with github runners, for example https://github.com/Project-MONAI/MONAI/actions/runs/3472091346/jobs/5802451135



Discussion:
hi @mingxin-zheng @Nic-Ma, do you have time to simplify `test_autorunner_hpo`? the way it creates nni server and monitors the file system changes is not reliable and difficult to debug.
Hi @wyli , yes, the autorunner test needs improvements. I would like to find out what is not needed and should be fixed first, and here are a few feasible options for us to gauge:
- avoid having a web service in the integration test
- finding a way to get trial information and fail the test appropiately
- avoid running hpo with training. For example, just checking if the trial command is correct
thanks, maybe it's good enough to go for option 3 for now? 

I have spent 5+ hours on option 2 but it's not easy, especially when we want to ensure the same testing behaviour across different testing environments
I support option 3, to avoid running hpo,  since it's rather slow (perhaps we could run it only once a month, or not running it at all)

on top of being slow there is a small issue there,  that on 8gpus in silently hangs (because the current toy dataset is small). so it's confirmed to work on 2gpus, but it's not clear if it works well on more than 2 gpus.

Resolving patch:
diff --git a/monai/apps/auto3dseg/auto_runner.py b/monai/apps/auto3dseg/auto_runner.py
--- a/monai/apps/auto3dseg/auto_runner.py
+++ b/monai/apps/auto3dseg/auto_runner.py
@@ -410,9 +410,15 @@ def set_hpo_params(self, params: Optional[Dict[str, Any]] = None):
             - "tuner"
             - "trainingService"
 
+        and (3) enable the dry-run mode if the user would generate the NNI configs without starting the NNI service.
+
         Args:
             params: a dict that defines the overriding key-value pairs during instantiation of the algo. For
                 BundleAlgo, it will override the template config filling.
+
+        Notes:
+            Users can set ``nni_dry_run`` to ``True`` in the ``params`` to enable the dry-run mode for the NNI backend.
+
         """
         if params is None:
             self.hpo_params = self.train_params
@@ -538,6 +544,7 @@ def _train_algo_in_nni(self, history):
         }
 
         last_total_tasks = len(import_bundle_algo_history(self.work_dir, only_trained=True))
+        mode_dry_run = self.hpo_params.pop("nni_dry_run", False)
         for task in history:
             for name, algo in task.items():
                 nni_gen = NNIGen(algo=algo, params=self.hpo_params)
@@ -551,11 +558,16 @@ def _train_algo_in_nni(self, history):
                 nni_config.update({"search_space": self.search_space})
                 trial_cmd = "python -m monai.apps.auto3dseg NNIGen run_algo " + obj_filename + " " + self.work_dir
                 nni_config.update({"trialCommand": trial_cmd})
-                nni_config_filename = os.path.abspath(os.path.join(self.work_dir, "nni_config.yaml"))
+                nni_config_filename = os.path.abspath(os.path.join(self.work_dir, f"{name}_nni_config.yaml"))
                 ConfigParser.export_config_file(nni_config, nni_config_filename, fmt="yaml", default_flow_style=None)
 
                 max_trial = min(self.hpo_tasks, default_nni_config["maxTrialNumber"])
                 cmd = "nnictl create --config " + nni_config_filename + " --port 8088"
+
+                if mode_dry_run:
+                    logger.info(f"AutoRunner HPO is in dry-run mode. Please manually launch: {cmd}")
+                    continue
+
                 subprocess.run(cmd.split(), check=True)
 
                 n_trainings = len(import_bundle_algo_history(self.work_dir, only_trained=True))
diff --git a/monai/apps/auto3dseg/hpo_gen.py b/monai/apps/auto3dseg/hpo_gen.py
--- a/monai/apps/auto3dseg/hpo_gen.py
+++ b/monai/apps/auto3dseg/hpo_gen.py
@@ -91,6 +91,15 @@ class NNIGen(HPOGen):
             ├── model_fold0
             └── scripts
 
+        .. code-block:: python
+            # Bundle Algorithms are already generated by BundleGen in work_dir
+            import_bundle_algo_history(work_dir, only_trained=False)
+            algo_dict = self.history[0]  # pick the first algorithm
+            algo_name = list(algo_dict.keys())[0]
+            onealgo = algo_dict[algo_name]
+            nni_gen = NNIGen(algo=onealgo)
+            nni_gen.print_bundle_algo_instruction()
+
     Notes:
         The NNIGen will prepare the algorithms in a folder and suggest a command to replace trialCommand in the experiment
         config. However, NNIGen will not trigger NNI. User needs to write their NNI experiment configs, and then run the
@@ -118,7 +127,6 @@ def __init__(self, algo: Optional[Algo] = None, params=None):
 
             if isinstance(self.algo, BundleAlgo):
                 self.obj_filename = algo_to_pickle(self.algo, template_path=self.algo.template_path)
-                self.print_bundle_algo_instruction()
             else:
                 self.obj_filename = algo_to_pickle(self.algo)
                 # nni instruction unknown


Test patch:
diff --git a/tests/test_integration_autorunner.py b/tests/test_integration_autorunner.py
--- a/tests/test_integration_autorunner.py
+++ b/tests/test_integration_autorunner.py
@@ -121,7 +121,7 @@ def test_autorunner_ensemble(self) -> None:
     @unittest.skipIf(not has_nni, "nni required")
     def test_autorunner_hpo(self) -> None:
         work_dir = os.path.join(self.test_path, "work_dir")
-        runner = AutoRunner(work_dir=work_dir, input=self.data_src_cfg, hpo=True)
+        runner = AutoRunner(work_dir=work_dir, input=self.data_src_cfg, hpo=True, ensemble=False)
         hpo_param = {
             "num_iterations": 8,
             "num_iterations_per_validation": 4,
@@ -139,6 +139,7 @@ def test_autorunner_hpo(self) -> None:
             "searching#num_images_per_batch": 2,
             "searching#num_epochs": 2,
             "searching#num_warmup_iterations": 4,
+            "nni_dry_run": True,
         }
         search_space = {"learning_rate": {"_type": "choice", "_value": [0.0001, 0.001, 0.01, 0.1]}}
         runner.set_num_fold(1)